You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by ni...@apache.org on 2026/10/19 02:46:55 UTC
(paimon-cpp) branch main updated: feat(realtime): support user-defined-key deduplication for append tables (#393)
This is an automated email from the ASF dual-hosted git repository.
SteNicholas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 40a7d05e feat(realtime): support user-defined-key deduplication for append tables (#393)
40a7d05e is described below
commit 40a7d05ed9402c3cf2a206b995c64addbbfa2d9e
Author: lxy <38...@users.noreply.github.com>
AuthorDate: Mon Oct 19 10:46:50 2026 +0800
feat(realtime): support user-defined-key deduplication for append tables (#393)
---
include/paimon/defs.h | 4 +
include/paimon/file_store_commit.h | 4 +
include/paimon/file_store_write.h | 16 +-
.../paimon/realtime/arrow_realtime_store_factory.h | 3 +-
include/paimon/realtime/realtime_commit_progress.h | 5 +
include/paimon/realtime/realtime_store.h | 27 +-
src/paimon/CMakeLists.txt | 11 +
src/paimon/common/defs.cpp | 1 +
src/paimon/common/global_index/CMakeLists.txt | 1 -
src/paimon/common/global_index/key_serializer.h | 3 +-
src/paimon/common/table/special_fields.h | 4 +-
src/paimon/common/table/special_fields_test.cpp | 2 +-
src/paimon/core/core_options.cpp | 8 +
src/paimon/core/core_options.h | 2 +
src/paimon/core/core_options_test.cpp | 4 +
.../core/operation/abstract_file_store_write.cpp | 48 +-
.../core/operation/abstract_file_store_write.h | 4 +
src/paimon/core/operation/abstract_split_read.cpp | 7 +
.../operation/append_only_file_store_write.cpp | 36 +-
.../operation/commit/commit_changes_provider.h | 3 +
src/paimon/core/operation/file_store_commit.cpp | 2 +
.../core/operation/file_store_commit_impl.cpp | 197 +++-
src/paimon/core/operation/file_store_commit_impl.h | 7 +
.../core/operation/file_store_commit_impl_test.cpp | 5 +-
src/paimon/core/operation/file_store_write.cpp | 41 +-
.../core/operation/key_value_file_store_write.cpp | 3 +-
.../operation/key_value_file_store_write_test.cpp | 50 +-
src/paimon/core/operation/raw_file_split_read.cpp | 16 +-
src/paimon/core/operation/raw_file_split_read.h | 7 +
.../realtime/arrow_deduplicate_realtime_store.cpp | 542 +++++++++++
.../realtime/arrow_deduplicate_realtime_store.h | 142 +++
.../arrow_deduplicate_realtime_store_test.cpp | 247 +++++
.../core/realtime/arrow_realtime_store_factory.cpp | 2 +
.../core/realtime/arrow_realtime_store_test.cpp | 20 +-
src/paimon/core/realtime/realtime_context_impl.cpp | 105 +-
src/paimon/core/realtime/realtime_context_impl.h | 19 +-
src/paimon/core/realtime/realtime_context_test.cpp | 145 ++-
.../core/realtime/realtime_deduplicate_state.cpp | 86 ++
.../core/realtime/realtime_deduplicate_state.h | 54 ++
.../realtime/realtime_deduplicate_state_test.cpp | 139 +++
.../core/realtime/realtime_deduplicate_writer.cpp | 353 +++++++
.../core/realtime/realtime_deduplicate_writer.h | 105 ++
.../core/realtime/realtime_key_offset_index.cpp | 319 +++++++
.../core/realtime/realtime_key_offset_index.h | 158 +++
.../realtime/realtime_key_offset_index_test.cpp | 230 +++++
.../core/realtime/realtime_offset_batch_reader.cpp | 14 +-
.../core/realtime/realtime_offset_batch_reader.h | 3 +-
.../realtime/realtime_offset_batch_reader_test.cpp | 10 +-
.../realtime/realtime_offset_file_index_lookup.cpp | 183 ++++
.../realtime/realtime_offset_file_index_lookup.h | 89 ++
.../realtime_offset_file_index_lookup_test.cpp | 228 +++++
.../core/realtime/realtime_schema_layout.cpp | 24 +-
src/paimon/core/realtime/realtime_schema_layout.h | 9 +-
.../core/realtime/realtime_schema_layout_test.cpp | 32 +
.../core/realtime/realtime_store_read_pipeline.cpp | 20 +-
src/paimon/core/realtime/realtime_utils.h | 116 +++
src/paimon/core/realtime/realtime_utils_test.cpp | 121 +++
src/paimon/core/schema/schema_validation.cpp | 49 +
src/paimon/core/schema/schema_validation.h | 5 +-
src/paimon/core/schema/schema_validation_test.cpp | 121 ++-
.../core/table/source/append_only_table_read.cpp | 154 ++-
.../core/table/source/append_only_table_read.h | 5 +
.../core/table/source/realtime_table_scan.cpp | 2 +-
src/paimon/core/table/source/table_read.cpp | 2 +
src/paimon/core/table/source/table_scan.cpp | 2 +
src/paimon/core/utils/commit_increment.h | 12 +
test/inte/realtime_write_inte_test.cpp | 1006 +++++++++++++++++++-
67 files changed, 5240 insertions(+), 154 deletions(-)
diff --git a/include/paimon/defs.h b/include/paimon/defs.h
index 88cb78bd..8c8e3235 100644
--- a/include/paimon/defs.h
+++ b/include/paimon/defs.h
@@ -645,6 +645,10 @@ struct PAIMON_EXPORT Options {
/// Supported values are "none" and "full". Default value is "none".
static const char REALTIME_STORE_STATS_MODE[];
+ /// "realtime.deduplicate-key-fields" - Comma-separated user-defined deduplicate key fields
+ /// used for last-write-wins deduplication on an append table. No default value.
+ static const char REALTIME_DEDUPLICATE_KEY_FIELDS[];
+
/// "scan.timestamp" - Optional timestamp string used in case of "from-timestamp" scan mode,
/// as an alternative to "scan.timestamp-millis".
/// It will be automatically converted to timestamp in unix milliseconds, using local time zone.
diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h
index 776ad52a..fcb094e0 100644
--- a/include/paimon/file_store_commit.h
+++ b/include/paimon/file_store_commit.h
@@ -98,6 +98,10 @@ class PAIMON_EXPORT FileStoreCommit {
/// Concurrent rollback or partition deletion must be fenced by the upstream coordinator.
/// Configure catalog writers with `WriteContextBuilder::WithCatalog()` too.
///
+ /// In deduplicate mode each commit attempt resolves logical deleted offsets against the
+ /// latest snapshot and merges them with its deletion vectors. Keep the complete progress
+ /// when retrying; serializing only CommitMessage discards logical deletions.
+ ///
/// @param realtime_commits Commit messages and left-closed, right-open offset ranges to
/// commit.
/// @param commit_identifier Identifier of the streaming commit operation.
diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h
index 71ca910b..8859e4f3 100644
--- a/include/paimon/file_store_write.h
+++ b/include/paimon/file_store_write.h
@@ -52,12 +52,16 @@ class PAIMON_EXPORT FileStoreWrite {
virtual ~FileStoreWrite() = default;
/// Support write an input `RecordBatch` to internal buffer or file.
- /// @note Real-time writers require a non-nullable int64 `_REALTIME_OFFSET` field before the
- /// table write fields. Its values must be strictly increasing within each batch and
- /// monotonically increasing for each partition-bucket across batches; gaps are allowed.
- /// The field is used for snapshot progress and is not written to data files.
+ /// @note Real-time writers require a non-nullable int64 `_REALTIME_OFFSET`. Its values must be
+ /// strictly increasing within each batch and monotonically increasing for each
+ /// partition-bucket across batches; gaps are allowed. Append-only and primary-key modes
+ /// receive it as an internal prefix and do not write it to data files. Deduplicate mode
+ /// requires it in the table schema and persists it as a normal data-file column.
/// @note If a field in table schema is marked as non-nullable (`nullable = false`),
/// the corresponding array in `batch` must have zero null entries.
+ /// @note A failure while writing or flushing a real-time batch invalidates the writer and its
+ /// `RealtimeContext`. Recreate both and let upstream replay input from the durable
+ /// recovery offset persisted in the snapshot; retrying the failed writer is unsupported.
virtual Status Write(std::unique_ptr<RecordBatch>&& batch) = 0;
/// Slices the current in-memory real-time data into a sealed segment so it can be reclaimed
@@ -111,7 +115,9 @@ class PAIMON_EXPORT FileStoreWrite {
/// Each range is returned atomically with the commit message generated from the same sealed
/// segments. Repeated calls return incremental progress. The upstream coordinator must retain
/// every result until it is committed and include all earlier prepared-but-uncommitted
- /// progress when a later checkpoint subsumes it.
+ /// progress when a later checkpoint subsumes it. In deduplicate mode the result also retains
+ /// an immutable deleted-offset bitmap; physical deletion vectors are generated at commit,
+ /// not at prepare. Preserve the complete progress, not just its CommitMessage.
///
/// @param commit_identifier Identifier of this prepare-commit operation in streaming mode.
/// @return Real-time commit messages with their partition-bucket offset ranges.
diff --git a/include/paimon/realtime/arrow_realtime_store_factory.h b/include/paimon/realtime/arrow_realtime_store_factory.h
index b3fa630d..235671ac 100644
--- a/include/paimon/realtime/arrow_realtime_store_factory.h
+++ b/include/paimon/realtime/arrow_realtime_store_factory.h
@@ -26,7 +26,8 @@ namespace paimon {
/// Factory for Paimon's default Arrow-backed `RealtimeStore`.
class PAIMON_EXPORT ArrowRealtimeStoreFactory : public RealtimeStoreFactory {
public:
- /// Creates an Arrow-backed store for one partition and bucket.
+ /// Creates an Arrow-backed append or primary-key store for one partition and bucket.
+ /// The framework uses the append implementation as the delegate of its DEDUPLICATE wrapper.
Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override;
};
diff --git a/include/paimon/realtime/realtime_commit_progress.h b/include/paimon/realtime/realtime_commit_progress.h
index bf19695c..c492280c 100644
--- a/include/paimon/realtime/realtime_commit_progress.h
+++ b/include/paimon/realtime/realtime_commit_progress.h
@@ -24,6 +24,7 @@
#include "paimon/commit_message.h"
#include "paimon/realtime/offset_range.h"
#include "paimon/realtime/realtime_context.h"
+#include "paimon/utils/roaring_bitmap64.h"
#include "paimon/visibility.h"
namespace paimon {
@@ -41,6 +42,10 @@ struct PAIMON_EXPORT RealtimeCommitProgress {
RealtimePartitionBucket partition_bucket;
/// Left-closed, right-open offset range represented by the commit message.
OffsetRange offset_range;
+ /// Immutable logical deletions for deduplicate mode; null for other modes. Commit resolves
+ /// these offsets against the latest snapshot and rebuilds physical DVs on every retry.
+ /// This bitmap is not embedded in CommitMessage serialization.
+ std::shared_ptr<const RoaringBitmap64> deleted_offsets = nullptr;
};
} // namespace paimon
diff --git a/include/paimon/realtime/realtime_store.h b/include/paimon/realtime/realtime_store.h
index 1d9ab0c6..f31c5c97 100644
--- a/include/paimon/realtime/realtime_store.h
+++ b/include/paimon/realtime/realtime_store.h
@@ -46,20 +46,20 @@ class Predicate;
enum class PAIMON_EXPORT RealtimeStoreMode {
APPEND_ONLY,
PRIMARY_KEY,
+ /// Experimental mode whose persisted data and index formats may change in future releases.
+ DEDUPLICATE,
};
-/// Parameters used by a `RealtimeStoreFactory` to create a store.
+/// Parameters used to create a real-time store.
struct PAIMON_EXPORT RealtimeStoreCreateRequest {
- /// Schema whose ownership is transferred to the factory. Append mode receives the complete
- /// append store-write schema: [_REALTIME_OFFSET, table write fields]. Primary-key mode receives
- /// the real-time primary-key store-write schema:
- /// [_SEQUENCE_NUMBER, _VALUE_KIND, _REALTIME_OFFSET, table write fields].
+ /// Physical write schema whose ownership is transferred to the consumer. Its layout is
+ /// mode-specific and contains a non-null int64 `_REALTIME_OFFSET` field.
std::unique_ptr<::ArrowSchema> write_schema;
/// Table options available to the store implementation.
std::map<std::string, std::string> options;
/// Memory pool for allocations retained by the store.
std::shared_ptr<MemoryPool> memory_pool;
- /// Table mode implemented by the store.
+ /// Requested store mode.
RealtimeStoreMode mode = RealtimeStoreMode::APPEND_ONLY;
/// Statistics collected by the store for query pruning.
StatisticsMode statistics_mode = StatisticsMode::NONE;
@@ -69,6 +69,9 @@ struct PAIMON_EXPORT RealtimeStoreCreateRequest {
/// File system used to access `temp_directory`. The default store retains shared ownership
/// for its lifetime. Custom stores may ignore this hint.
std::shared_ptr<FileSystem> file_system = nullptr;
+ /// User-defined deduplicate key field names when key-based deduplication is configured; empty
+ /// otherwise.
+ std::vector<std::string> deduplicate_key_fields = {};
};
/// A record batch and its application-assigned offset bounds.
@@ -77,9 +80,9 @@ struct PAIMON_EXPORT RealtimeStoreCreateRequest {
/// [_REALTIME_OFFSET, table write fields], and offsets are strictly increasing before the batch
/// enters the store. Primary-key batches use the real-time primary-key store-write schema, are
/// sorted by full primary key then sequence number, and retain the original offset in
-/// `_REALTIME_OFFSET`. `offset_range` is the left-closed, right-open envelope from the first
-/// application offset through one past the last; offsets may have gaps, so its count is not the
-/// batch row count.
+/// `_REALTIME_OFFSET`.
+/// `offset_range` is the left-closed, right-open envelope from the first application offset through
+/// one past the last; offsets may have gaps, so its count is not the batch row count.
struct PAIMON_EXPORT RealtimeWriteBatch {
/// Input batch whose ownership is transferred to `RealtimeStore::Write`.
std::unique_ptr<RecordBatch> batch;
@@ -170,9 +173,8 @@ class PAIMON_EXPORT RealtimeStore {
/// Creates readers that expose all rows in a sealed segment for Paimon file writing.
///
/// The returned readers collectively expose every sealed row exactly once. Append-mode readers
- /// preserve write order and contain `_REALTIME_OFFSET` followed by the table write fields.
- /// Primary-key readers contain the real-time primary-key store fields; each reader's complete
- /// stream is sorted by full primary key then sequence number.
+ /// preserve write order. Primary-key reader streams are sorted by full primary key then
+ /// sequence number. Returned batches match the mode-specific physical write schema.
/// Returned readers have independent mutable read state and may be operated concurrently with
/// one another without external synchronization.
virtual Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
@@ -215,7 +217,6 @@ class PAIMON_EXPORT RealtimeStoreFactory {
virtual ~RealtimeStoreFactory() = default;
/// Creates a store configured with the supplied schema, statistics, options, and memory pool.
- /// Creates a store for the requested table mode.
/// The factory consumes `request`, including ownership of `request.write_schema`.
virtual Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) = 0;
};
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index ba89aa82..489aaced 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -86,6 +86,7 @@ set(PAIMON_COMMON_SRCS
common/global_index/bitmap_global_index_result.cpp
common/global_index/global_index_result.cpp
common/global_index/global_indexer_factory.cpp
+ common/global_index/key_serializer.cpp
common/io/buffered_input_stream.cpp
common/io/byte_array_input_stream.cpp
common/io/byte_array_output_stream.cpp
@@ -402,7 +403,12 @@ set(PAIMON_CORE_SRCS
core/operation/write_context.cpp
core/operation/write_restore.cpp
core/realtime/arrow_realtime_store.cpp
+ core/realtime/arrow_deduplicate_realtime_store.cpp
core/realtime/arrow_realtime_store_factory.cpp
+ core/realtime/realtime_key_offset_index.cpp
+ core/realtime/realtime_offset_file_index_lookup.cpp
+ core/realtime/realtime_deduplicate_state.cpp
+ core/realtime/realtime_deduplicate_writer.cpp
core/realtime/realtime_offset_batch_reader.cpp
core/realtime/realtime_primary_key_reader.cpp
core/realtime/realtime_append_only_writer.cpp
@@ -850,14 +856,19 @@ if(PAIMON_BUILD_TESTS)
core/manifest/index_manifest_entry_serializer_test.cpp
core/manifest/index_manifest_file_handler_test.cpp
core/memory/writer_memory_manager_test.cpp
+ core/realtime/arrow_deduplicate_realtime_store_test.cpp
core/realtime/arrow_realtime_store_test.cpp
core/realtime/primary_key_realtime_store_test.cpp
+ core/realtime/realtime_deduplicate_state_test.cpp
+ core/realtime/realtime_key_offset_index_test.cpp
core/realtime/realtime_offset_batch_reader_test.cpp
+ core/realtime/realtime_offset_file_index_lookup_test.cpp
core/realtime/realtime_store_read_pipeline_test.cpp
core/realtime/realtime_schema_layout_test.cpp
core/realtime/realtime_primary_key_reader_test.cpp
core/realtime/realtime_context_test.cpp
core/realtime/realtime_reader_test.cpp
+ core/realtime/realtime_utils_test.cpp
core/mergetree/levels_test.cpp
core/mergetree/lookup_file_test.cpp
core/mergetree/lookup_levels_test.cpp
diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp
index aa59ed5b..951394e5 100644
--- a/src/paimon/common/defs.cpp
+++ b/src/paimon/common/defs.cpp
@@ -170,6 +170,7 @@ const char Options::REALTIME_ENABLED[] = "realtime.enabled";
const char Options::REALTIME_SPILL_ENABLED[] = "realtime.spill-enabled";
const char Options::REALTIME_READ_VIEW_TTL[] = "realtime.read-view-ttl";
const char Options::REALTIME_STORE_STATS_MODE[] = "realtime.store.stats-mode";
+const char Options::REALTIME_DEDUPLICATE_KEY_FIELDS[] = "realtime.deduplicate-key-fields";
const char Options::SCAN_TIMESTAMP[] = "scan.timestamp";
const char Options::SCAN_TAG_NAME[] = "scan.tag-name";
const char Options::WRITE_ONLY[] = "write-only";
diff --git a/src/paimon/common/global_index/CMakeLists.txt b/src/paimon/common/global_index/CMakeLists.txt
index d344495f..aa1fe82b 100644
--- a/src/paimon/common/global_index/CMakeLists.txt
+++ b/src/paimon/common/global_index/CMakeLists.txt
@@ -30,7 +30,6 @@ set(PAIMON_GLOBAL_INDEX_SRC
sorted_file_meta_selector.cpp
sorted_index_file_meta.cpp
btree/lazy_filtered_btree_reader.cpp
- key_serializer.cpp
rangebitmap/range_bitmap_global_index.cpp
rangebitmap/range_bitmap_global_index_factory.cpp
offset_global_index_reader.cpp
diff --git a/src/paimon/common/global_index/key_serializer.h b/src/paimon/common/global_index/key_serializer.h
index 09b5298d..f104a6d3 100644
--- a/src/paimon/common/global_index/key_serializer.h
+++ b/src/paimon/common/global_index/key_serializer.h
@@ -26,10 +26,11 @@
#include "paimon/memory/bytes.h"
#include "paimon/memory/memory_pool.h"
#include "paimon/predicate/literal.h"
+#include "paimon/visibility.h"
namespace paimon {
/// Provides core methods to serialize, deserialize, and compare global index keys.
-class KeySerializer {
+class PAIMON_EXPORT KeySerializer {
public:
~KeySerializer() = default;
diff --git a/src/paimon/common/table/special_fields.h b/src/paimon/common/table/special_fields.h
index e810fc1a..f3e64fcc 100644
--- a/src/paimon/common/table/special_fields.h
+++ b/src/paimon/common/table/special_fields.h
@@ -85,13 +85,15 @@ struct SpecialFields {
depth;
}
+ // `_REALTIME_OFFSET` is persisted as a regular TableSchema field in realtime deduplicate
+ // mode, so it is not a synthetic system field.
static bool IsSystemField(const std::string& field_name) {
if (StringUtils::StartsWith(field_name, KEY_FIELD_PREFIX)) {
return true;
}
return field_name == SequenceNumber().Name() || field_name == ValueKind().Name() ||
field_name == RowKind().Name() || field_name == RowId().Name() ||
- field_name == IndexScore().Name() || field_name == RealtimeOffset().Name();
+ field_name == IndexScore().Name();
}
// TODO(xinyu.lxy): add a func to complete row-tracking fields
diff --git a/src/paimon/common/table/special_fields_test.cpp b/src/paimon/common/table/special_fields_test.cpp
index a985e2b8..05c0808e 100644
--- a/src/paimon/common/table/special_fields_test.cpp
+++ b/src/paimon/common/table/special_fields_test.cpp
@@ -80,7 +80,7 @@ TEST(SpecialFieldsTest, TestIsSystemField) {
ASSERT_TRUE(SpecialFields::IsSystemField("rowkind"));
ASSERT_TRUE(SpecialFields::IsSystemField("_ROW_ID"));
ASSERT_TRUE(SpecialFields::IsSystemField("_INDEX_SCORE"));
- ASSERT_TRUE(SpecialFields::IsSystemField("_REALTIME_OFFSET"));
+ ASSERT_FALSE(SpecialFields::IsSystemField("_REALTIME_OFFSET"));
ASSERT_TRUE(SpecialFields::IsSystemField("_KEY_0"));
}
diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp
index 28af1914..92afe6ea 100644
--- a/src/paimon/core/core_options.cpp
+++ b/src/paimon/core/core_options.cpp
@@ -375,6 +375,7 @@ struct CoreOptions::Impl {
std::vector<std::string> blob_fields;
std::vector<std::string> blob_descriptor_fields;
std::vector<std::string> blob_view_fields;
+ std::vector<std::string> realtime_deduplicate_key_fields;
std::string partition_default_name = "__DEFAULT_PARTITION__";
StartupMode startup_mode = StartupMode::Default();
@@ -858,6 +859,9 @@ struct CoreOptions::Impl {
PAIMON_RETURN_NOT_OK(parser.Parse<bool>(Options::REALTIME_ENABLED, &realtime_enabled));
PAIMON_RETURN_NOT_OK(
parser.Parse<bool>(Options::REALTIME_SPILL_ENABLED, &realtime_spill_enabled));
+ PAIMON_RETURN_NOT_OK(parser.ParseList<std::string>(
+ Options::REALTIME_DEDUPLICATE_KEY_FIELDS, Options::FIELDS_SEPARATOR,
+ &realtime_deduplicate_key_fields, /*need_trim=*/true));
PAIMON_RETURN_NOT_OK(parser.ParseTimeDuration(Options::REALTIME_READ_VIEW_TTL,
&realtime_read_view_ttl_millis));
if (realtime_read_view_ttl_millis <= 0) {
@@ -1209,6 +1213,10 @@ StatisticsMode CoreOptions::GetRealtimeStoreStatisticsMode() const {
return impl_->realtime_store_statistics_mode;
}
+const std::vector<std::string>& CoreOptions::GetRealtimeDeduplicateKeyFields() const {
+ return impl_->realtime_deduplicate_key_fields;
+}
+
int32_t CoreOptions::GetScanManifestEntryCacheMaxSnapshots() const {
return impl_->scan_manifest_entry_cache_max_snapshots;
}
diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h
index a16f75e4..a16f5e67 100644
--- a/src/paimon/core/core_options.h
+++ b/src/paimon/core/core_options.h
@@ -112,6 +112,8 @@ class PAIMON_EXPORT CoreOptions {
int64_t GetRealtimeReadViewTtlMillis() const;
/// Returns the statistics mode used by the real-time store.
StatisticsMode GetRealtimeStoreStatisticsMode() const;
+ /// Returns append-table user-defined deduplicate key fields used by real-time deduplication.
+ const std::vector<std::string>& GetRealtimeDeduplicateKeyFields() const;
int32_t GetScanManifestEntryCacheMaxSnapshots() const;
bool ScanManifestEntryLazyDecodeEnabled() const;
diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp
index 0d8ab296..074105cd 100644
--- a/src/paimon/core/core_options_test.cpp
+++ b/src/paimon/core/core_options_test.cpp
@@ -72,6 +72,7 @@ TEST(CoreOptionsTest, TestDefaultValue) {
ASSERT_FALSE(core_options.RealtimeEnabled());
ASSERT_FALSE(core_options.RealtimeSpillEnabled());
ASSERT_EQ(StatisticsMode::NONE, core_options.GetRealtimeStoreStatisticsMode());
+ ASSERT_TRUE(core_options.GetRealtimeDeduplicateKeyFields().empty());
ASSERT_EQ("zstd", core_options.GetFileCompression());
ASSERT_EQ(std::nullopt, core_options.GetChangelogFileCompression());
ASSERT_EQ("zstd", core_options.GetWriteFileCompression(0));
@@ -341,6 +342,7 @@ TEST(CoreOptionsTest, TestFromMap) {
{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"},
{Options::REALTIME_ENABLED, "true"},
{Options::REALTIME_SPILL_ENABLED, "true"},
+ {Options::REALTIME_DEDUPLICATE_KEY_FIELDS, " id, tenant_id "},
{Options::BUCKET_FUNCTION_TYPE, "mod"},
{"fields.metrics.map.storage-layout", "shared-shredding"},
{"fields.metrics.map.shared-shredding.max-columns", "128"},
@@ -512,6 +514,8 @@ TEST(CoreOptionsTest, TestFromMap) {
ASSERT_TRUE(core_options.KeyValueSequenceNumberEnabled());
ASSERT_TRUE(core_options.RealtimeEnabled());
ASSERT_TRUE(core_options.RealtimeSpillEnabled());
+ ASSERT_EQ((std::vector<std::string>{"id", "tenant_id"}),
+ core_options.GetRealtimeDeduplicateKeyFields());
ASSERT_TRUE(core_options.LookupRemoteFileEnabled());
ASSERT_EQ(core_options.GetLookupRemoteLevelThreshold(), 2);
ASSERT_EQ(BucketFunctionType::MOD, core_options.GetBucketFunctionType());
diff --git a/src/paimon/core/operation/abstract_file_store_write.cpp b/src/paimon/core/operation/abstract_file_store_write.cpp
index 8df19007..8f134f31 100644
--- a/src/paimon/core/operation/abstract_file_store_write.cpp
+++ b/src/paimon/core/operation/abstract_file_store_write.cpp
@@ -101,6 +101,11 @@ Status AbstractFileStoreWrite::Write(std::unique_ptr<RecordBatch>&& batch) {
if (PAIMON_UNLIKELY(batch == nullptr)) {
return Status::Invalid("batch is null pointer");
}
+ std::shared_lock<std::shared_mutex> write_lock(realtime_write_completion_mutex_,
+ std::defer_lock);
+ if (IsRealtimeWrite()) {
+ write_lock.lock();
+ }
PAIMON_RETURN_NOT_OK(CheckRealtimeWriteUsable());
// in FileStoreWrite::Create() we have checked the table kind and bucket mode, here we only
// check the bucket id in batch
@@ -146,8 +151,16 @@ Status AbstractFileStoreWrite::Write(std::unique_ptr<RecordBatch>&& batch) {
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<BatchWriter> writer,
GetWriter(partition, batch->GetBucket()));
assert(writer);
- PAIMON_RETURN_NOT_OK(writer->Write(std::move(batch)));
- return writer_memory_manager_->OnWriteCompleted(writer.get());
+ Status status = writer->Write(std::move(batch));
+ if (status.ok()) {
+ status = writer_memory_manager_->OnWriteCompleted(writer.get());
+ }
+ if (!status.ok() && IsRealtimeWrite()) {
+ // Bucket writes and memory flushing can fail after changing buffered data or indexes.
+ // Fence the shared context so no retry or prepare can publish that partial state.
+ return PoisonRealtimeWrite(status);
+ }
+ return status;
}
Status AbstractFileStoreWrite::Seal() {
@@ -172,7 +185,8 @@ Status AbstractFileStoreWrite::Seal() {
return PoisonRealtimeWrite(status);
}
}
- return Status::OK();
+ std::unique_lock<std::shared_mutex> completion_lock(realtime_write_completion_mutex_);
+ return CheckRealtimeWriteUsable();
}
Status AbstractFileStoreWrite::Compact(const std::map<std::string, std::string>& partition,
@@ -346,6 +360,10 @@ Result<std::vector<RealtimeCommitProgress>> AbstractFileStoreWrite::PrepareCommi
// Prepare may have drained an earlier bucket or partially written the failing bucket.
return PoisonRealtimeWrite(result.status());
}
+ // A write may have changed a sealed checkpoint before failing. Wait until every in-flight
+ // write has published success or poisoned the context before exposing any commit messages.
+ std::unique_lock<std::shared_mutex> completion_lock(realtime_write_completion_mutex_);
+ PAIMON_RETURN_NOT_OK(CheckRealtimeWriteUsable());
return result;
}
@@ -372,18 +390,28 @@ Result<std::vector<RealtimeCommitProgress>> AbstractFileStoreWrite::PrepareRealt
PAIMON_ASSIGN_OR_RAISE(CommitIncrement increment,
snapshot.writer->PrepareCommit(/*wait_compaction=*/false));
writer_memory_manager_->RefreshWriterMemory(snapshot.writer.get());
+ std::shared_ptr<CompactDeletionFile> compact_deletion_file =
+ increment.GetCompactDeletionFile();
+ CompactIncrement& compact_increment = increment.GetCompactIncrement();
+ if (compact_deletion_file) {
+ PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<IndexFileMeta>> dv_index_file_meta,
+ compact_deletion_file->GetOrCompute());
+ if (dv_index_file_meta) {
+ compact_increment.AddNewIndexFiles({dv_index_file_meta.value()});
+ }
+ }
auto committable = std::make_shared<CommitMessageImpl>(
snapshot.partition, snapshot.bucket, snapshot.total_buckets,
- increment.GetNewFilesIncrement(), increment.GetCompactIncrement());
+ increment.GetNewFilesIncrement(), compact_increment);
if (!increment.GetRealtimeOffsetRange()) {
if (!committable->IsEmpty()) {
return Status::Invalid("real-time commit message does not have an offset range");
}
continue;
}
- if (committable->IsEmpty()) {
- return Status::Invalid("sealed real-time segment produced an empty commit message");
- }
+ // A real-time range may contain only changes that cancel in memory, for example an insert
+ // followed by a delete before either row is persisted. Keep the empty message so commit
+ // can durably advance the offset without manufacturing a data or deletion-vector file.
std::vector<std::pair<std::string, std::string>> partition_values;
PAIMON_ASSIGN_OR_RAISE(partition_values, file_store_path_factory_->GeneratePartitionVector(
snapshot.partition));
@@ -391,7 +419,7 @@ Result<std::vector<RealtimeCommitProgress>> AbstractFileStoreWrite::PrepareRealt
partition_values.end());
result.push_back(RealtimeCommitProgress{
committable, RealtimePartitionBucket(std::move(partition), snapshot.bucket),
- increment.GetRealtimeOffsetRange().value()});
+ increment.GetRealtimeOffsetRange().value(), increment.GetRealtimeDeletedOffsets()});
}
{
std::lock_guard<std::mutex> lock(realtime_metrics_mutex_);
@@ -466,8 +494,8 @@ std::shared_ptr<Metrics> AbstractFileStoreWrite::GetMetrics() const {
Status AbstractFileStoreWrite::CheckRealtimeWriteUsable() const {
if (IsRealtimeWrite() && realtime_write_poisoned_.load()) {
return Status::Invalid(
- "real-time writer cannot be reused after Seal or PrepareCommitWithProgress failed, "
- "or after it closed with unprepared data; recreate RealtimeContext and "
+ "real-time writer cannot be reused after Write, Seal or PrepareCommitWithProgress "
+ "failed, or after it closed with unprepared data; recreate RealtimeContext and "
"FileStoreWrite, then let upstream recover input from the durable recovery offset "
"persisted in the snapshot");
}
diff --git a/src/paimon/core/operation/abstract_file_store_write.h b/src/paimon/core/operation/abstract_file_store_write.h
index 1ac918b8..d98cc870 100644
--- a/src/paimon/core/operation/abstract_file_store_write.h
+++ b/src/paimon/core/operation/abstract_file_store_write.h
@@ -23,6 +23,7 @@
#include <limits>
#include <memory>
#include <mutex>
+#include <shared_mutex>
#include <string>
#include <unordered_map>
#include <vector>
@@ -180,6 +181,9 @@ class AbstractFileStoreWrite : public FileStoreWrite {
writers_;
std::mutex writers_mutex_;
std::mutex realtime_seal_prepare_mutex_;
+ // Seal/prepare may flush concurrently with writes, but cannot return before an in-flight
+ // write has published its outcome, including invalidating the context on failure.
+ std::shared_mutex realtime_write_completion_mutex_;
std::mutex realtime_metrics_mutex_;
bool ignore_previous_files_ = false;
bool is_streaming_mode_ = false;
diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp
index c6669023..d28a4ed1 100644
--- a/src/paimon/core/operation/abstract_split_read.cpp
+++ b/src/paimon/core/operation/abstract_split_read.cpp
@@ -191,6 +191,13 @@ Result<std::unique_ptr<FileBatchReader>> AbstractSplitRead::ApplyIndexAndDvReade
if (dv_factory) {
PAIMON_ASSIGN_OR_RAISE(deletion_vector, dv_factory(file->file_name));
}
+ // Existing files can still contain DV64 after the table option is changed to false.
+ if (context_->GetRealtimeContext() && !options_.GetRealtimeDeduplicateKeyFields().empty() &&
+ dynamic_cast<Bitmap64DeletionVector*>(deletion_vector.get()) != nullptr) {
+ return Status::Invalid(
+ "real-time deduplicate mode does not support bitmap64 deletion "
+ "vectors in existing data files");
+ }
std::shared_ptr<FileIndexResult> file_index_result;
if (options_.FileIndexReadEnabled() &&
dynamic_cast<Bitmap64DeletionVector*>(deletion_vector.get()) == nullptr) {
diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp
index 62e82310..15b89ba9 100644
--- a/src/paimon/core/operation/append_only_file_store_write.cpp
+++ b/src/paimon/core/operation/append_only_file_store_write.cpp
@@ -53,6 +53,7 @@
#include "paimon/core/operation/restore_files.h"
#include "paimon/core/realtime/realtime_append_only_writer.h"
#include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_deduplicate_writer.h"
#include "paimon/core/schema/table_schema.h"
#include "paimon/core/snapshot.h"
#include "paimon/core/utils/file_store_path_factory.h"
@@ -132,14 +133,29 @@ Status AppendOnlyFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) {
PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, latest && latest->Id() == snapshot_id
? Result<Snapshot>(latest.value())
: snapshot_manager_->LoadSnapshot(snapshot_id));
- PAIMON_ASSIGN_OR_RAISE(
- RealtimeOffsetMap committed_offsets,
- RealtimeCommitProperties::ReadOffsets(std::optional<Snapshot>(std::move(snapshot)),
- options_.GetFileSystem()));
+ PAIMON_ASSIGN_OR_RAISE(RealtimeOffsetMap committed_offsets,
+ RealtimeCommitProperties::ReadOffsets(std::optional<Snapshot>(snapshot),
+ options_.GetFileSystem()));
+ auto scan_filter = std::make_shared<ScanFilter>(
+ /*predicate=*/nullptr, std::vector<std::map<std::string, std::string>>{},
+ /*bucket_filter=*/std::nullopt);
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreScan> scan, CreateFileStoreScan(scan_filter));
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileStoreScan::RawPlan> plan,
+ scan->WithSnapshot(snapshot)->CreatePlan());
+ RealtimeDataFileMap committed_data_files;
+ for (const ManifestEntry& entry : plan->Files()) {
+ std::vector<std::pair<std::string, std::string>> partition_values;
+ PAIMON_ASSIGN_OR_RAISE(
+ partition_values, file_store_path_factory_->GeneratePartitionVector(entry.Partition()));
+ std::map<std::string, std::string> partition(partition_values.begin(),
+ partition_values.end());
+ committed_data_files[RealtimePartitionBucket(std::move(partition), entry.Bucket())]
+ .push_back(entry.File());
+ }
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
RealtimeContextImpl::Cast(realtime_context_));
- PAIMON_RETURN_NOT_OK(
- realtime_context_impl->AdvanceCommittedProgress(snapshot_id, committed_offsets));
+ PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress(
+ snapshot_id, committed_offsets, committed_data_files));
return Status::OK();
}
@@ -301,6 +317,14 @@ Result<std::shared_ptr<BatchWriter>> AppendOnlyFileStoreWrite::CreateWriter(
file_store_path_factory_->GeneratePartitionVector(partition));
std::map<std::string, std::string> partition_map(partition_values.begin(),
partition_values.end());
+ const std::vector<std::string>& deduplicate_key_fields =
+ options_.GetRealtimeDeduplicateKeyFields();
+ if (!deduplicate_key_fields.empty()) {
+ return RealtimeDeduplicateWriter::Create(
+ partition_map, bucket, realtime_context_, writer, realtime_schema_layout_,
+ table_schema_->Id(), deduplicate_key_fields, restore_data_files, data_file_path_factory,
+ options_, io_manager_ ? io_manager_->GetTempDir() : "", pool_);
+ }
return RealtimeAppendOnlyWriter::Create(partition_map, bucket, realtime_context_, writer,
realtime_schema_layout_, options_,
io_manager_ ? io_manager_->GetTempDir() : "", pool_);
diff --git a/src/paimon/core/operation/commit/commit_changes_provider.h b/src/paimon/core/operation/commit/commit_changes_provider.h
index ef23eb0d..603f59f9 100644
--- a/src/paimon/core/operation/commit/commit_changes_provider.h
+++ b/src/paimon/core/operation/commit/commit_changes_provider.h
@@ -22,6 +22,7 @@
#include <functional>
#include <memory>
#include <optional>
+#include <string>
#include <vector>
#include "paimon/core/manifest/index_manifest_entry.h"
@@ -44,6 +45,8 @@ struct CommitChanges {
std::vector<ManifestEntry> delta_files;
std::vector<ManifestEntry> changelog_files;
std::vector<IndexManifestEntry> index_entries;
+ // Unpublished files owned by this attempt; retain on uncertain publication outcomes.
+ std::vector<std::string> temporary_files;
};
class CommitChangesProvider {
diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp
index c628a7ce..457b48dd 100644
--- a/src/paimon/core/operation/file_store_commit.cpp
+++ b/src/paimon/core/operation/file_store_commit.cpp
@@ -46,6 +46,7 @@
#include "paimon/core/operation/file_store_scan.h"
#include "paimon/core/operation/key_value_file_store_scan.h"
#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/schema_validation.h"
#include "paimon/core/schema/table_schema.h"
#include "paimon/core/table/format/format_table_file_store_commit.h"
#include "paimon/core/table/format/format_table_loader.h"
@@ -274,6 +275,7 @@ Result<std::unique_ptr<FileStoreCommit>> FileStoreCommit::Create(
std::shared_ptr<arrow::Schema> arrow_schema =
DataField::ConvertDataFieldsToArrowSchema(schema->Fields());
PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, specific_fs));
+ PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options));
assert(options.GetFileSystem());
assert(options.GetFileFormat());
PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options));
diff --git a/src/paimon/core/operation/file_store_commit_impl.cpp b/src/paimon/core/operation/file_store_commit_impl.cpp
index 41692854..4d6422c0 100644
--- a/src/paimon/core/operation/file_store_commit_impl.cpp
+++ b/src/paimon/core/operation/file_store_commit_impl.cpp
@@ -46,6 +46,7 @@
#include "paimon/common/utils/path_util.h"
#include "paimon/common/utils/scope_guard.h"
#include "paimon/core/catalog/snapshot_commit.h"
+#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h"
#include "paimon/core/deletionvectors/deletion_vectors_index_file.h"
#include "paimon/core/index/index_file_meta.h"
#include "paimon/core/io/compact_increment.h"
@@ -73,6 +74,7 @@
#include "paimon/core/operation/metrics/commit_metrics.h"
#include "paimon/core/operation/metrics/commit_stats.h"
#include "paimon/core/partition/partition_statistics.h"
+#include "paimon/core/realtime/realtime_offset_file_index_lookup.h"
#include "paimon/core/schema/schema_manager.h"
#include "paimon/core/schema/table_schema.h"
#include "paimon/core/table/bucket_mode.h"
@@ -96,6 +98,23 @@ namespace {
constexpr const char* kCommitStrictModeLastSafeSnapshot = "commit.strict-mode.last-safe-snapshot";
constexpr const char* kSequenceSnapshotOrdering = "sequence.snapshot-ordering";
+class RealtimeCommitChangesProvider final : public CommitChangesProvider {
+ public:
+ using Materializer =
+ std::function<Result<std::shared_ptr<CommitChanges>>(const std::optional<Snapshot>&)>;
+
+ explicit RealtimeCommitChangesProvider(Materializer materializer)
+ : materializer_(std::move(materializer)) {}
+
+ Result<std::shared_ptr<CommitChanges>> Provide(
+ const std::optional<Snapshot>& latest_snapshot) const override {
+ return materializer_(latest_snapshot);
+ }
+
+ private:
+ Materializer materializer_;
+};
+
} // namespace
Status FileStoreCommitImpl::ValidateCommitOptions(const CoreOptions& options) {
@@ -879,7 +898,9 @@ Status FileStoreCommitImpl::Commit(
ReportCommit(changes, duration.Get(), generated_snapshot, attempt);
});
- if (!ignore_empty_commit_ || changes.HasAppendChanges()) {
+ // A real-time commit may carry only durable offset progress, for example when an insert and
+ // delete cancel before either row is persisted. Such a commit still needs a snapshot.
+ if (!ignore_empty_commit_ || changes.HasAppendChanges() || !realtime_ranges.empty()) {
Snapshot::CommitKind commit_kind = Snapshot::CommitKind::Append();
if (append_commit_check_conflict_) {
check_append_files = true;
@@ -951,10 +972,16 @@ Result<int64_t> FileStoreCommitImpl::CommitWithProgress(
std::vector<std::shared_ptr<CommitMessage>> commit_messages;
commit_messages.reserve(ordered_commits.size());
std::map<RealtimePartitionBucket, OffsetRange> realtime_ranges;
+ std::map<RealtimePartitionBucket, RoaringBitmap64> deleted_offsets;
for (const RealtimeCommitProgress& realtime_commit : ordered_commits) {
if (!realtime_commit.commit_message) {
return Status::Invalid("real-time commit message is null");
}
+ if (!options_.GetRealtimeDeduplicateKeyFields().empty() &&
+ !realtime_commit.deleted_offsets) {
+ return Status::Invalid(
+ "realtime deduplicate progress requires a deleted-offset bitmap");
+ }
auto commit_message =
std::dynamic_pointer_cast<CommitMessageImpl>(realtime_commit.commit_message);
if (!commit_message) {
@@ -979,6 +1006,9 @@ Result<int64_t> FileStoreCommitImpl::CommitWithProgress(
OffsetRange(previous_range.begin, realtime_commit.offset_range.end);
}
commit_messages.push_back(realtime_commit.commit_message);
+ if (realtime_commit.deleted_offsets) {
+ deleted_offsets[realtime_commit.partition_bucket] |= *realtime_commit.deleted_offsets;
+ }
}
std::shared_ptr<ManifestCommittable> committable =
@@ -993,14 +1023,163 @@ Result<int64_t> FileStoreCommitImpl::CommitWithProgress(
PAIMON_RETURN_NOT_OK(CheckFilesExistence({committable}));
last_committed_snapshot_id_ = -1;
- PAIMON_RETURN_NOT_OK(Commit(committable, /*check_append_files=*/false,
- /*retry_on_conflict=*/true, realtime_ranges));
+ if (options_.GetRealtimeDeduplicateKeyFields().empty()) {
+ if (!deleted_offsets.empty()) {
+ return Status::Invalid("logical realtime deletions require deduplicate mode");
+ }
+ PAIMON_RETURN_NOT_OK(Commit(committable, /*check_append_files=*/false,
+ /*retry_on_conflict=*/true, realtime_ranges));
+ } else {
+ PAIMON_ASSIGN_OR_RAISE(ManifestEntryChanges changes, CollectChanges(commit_messages));
+ if (changes.HasCompactChanges()) {
+ return Status::Invalid(
+ "realtime deduplicate progress must not contain physical compaction changes");
+ }
+ auto provider = std::make_shared<RealtimeCommitChangesProvider>(
+ [this, changes, deleted_offsets](const std::optional<Snapshot>& snapshot) {
+ return MaterializeRealtimeDeletions(changes, deleted_offsets, snapshot);
+ });
+ const bool has_deletions =
+ std::any_of(deleted_offsets.begin(), deleted_offsets.end(),
+ [](const auto& entry) { return !entry.second.IsEmpty(); });
+ const bool overwrite =
+ has_deletions || conflict_detection_.ShouldBeOverwriteCommit(
+ changes.append_table_files, changes.append_index_files);
+ const bool detect_conflicts = overwrite || append_commit_check_conflict_ ||
+ conflict_detection_.HasRowIdCheckFromSnapshot() ||
+ changes.HasGlobalIndexFileAdditions();
+ Duration duration;
+ // Data files, offset progress and freshly materialized DVs must publish atomically.
+ PAIMON_ASSIGN_OR_RAISE(
+ TryCommitResult result,
+ TryCommit(
+ provider, identifier, watermark, committable->Properties(), realtime_ranges,
+ overwrite ? Snapshot::CommitKind::Overwrite() : Snapshot::CommitKind::Append(),
+ /*reset_all_realtime_progress=*/false,
+ /*removed_realtime_partitions=*/{}, detect_conflicts,
+ /*retry_on_conflict=*/true));
+ ReportCommit(changes, duration.Get(), result.already_committed ? 0 : 1, result.attempts);
+ }
if (last_committed_snapshot_id_ < Snapshot::FIRST_SNAPSHOT_ID) {
return Status::Invalid("real-time commit did not produce a snapshot");
}
return last_committed_snapshot_id_;
}
+Result<std::shared_ptr<CommitChanges>> FileStoreCommitImpl::MaterializeRealtimeDeletions(
+ const ManifestEntryChanges& changes,
+ const std::map<RealtimePartitionBucket, RoaringBitmap64>& deleted_offsets,
+ const std::optional<Snapshot>& latest_snapshot) const {
+ auto result = std::make_shared<CommitChanges>(
+ changes.append_table_files, changes.append_changelog, changes.append_index_files);
+ // Nothing generated here can be referenced by a snapshot until Provide succeeds.
+ ScopeGuard cleanup([&]() {
+ for (const std::string& path : result->temporary_files) {
+ [[maybe_unused]] Status status = fs_->Delete(path, /*recursive=*/false);
+ }
+ });
+ std::vector<std::map<std::string, std::string>> partitions;
+ for (const auto& [partition_bucket, offsets] : deleted_offsets) {
+ if (!offsets.IsEmpty()) {
+ partitions.push_back(partition_bucket.partition);
+ }
+ }
+ if (partitions.empty()) {
+ cleanup.Release();
+ return result;
+ }
+ std::vector<ManifestEntry> committed_files;
+ std::vector<IndexManifestEntry> committed_indexes;
+ if (latest_snapshot) {
+ PAIMON_ASSIGN_OR_RAISE(committed_files, commit_scanner_->ReadAllEntriesFromPartitions(
+ latest_snapshot.value(), partitions));
+ PAIMON_ASSIGN_OR_RAISE(committed_indexes,
+ commit_scanner_->ReadAllIndexEntriesFromPartitions(
+ latest_snapshot.value(), partitions));
+ }
+ for (const auto& [partition_bucket, offsets] : deleted_offsets) {
+ if (offsets.IsEmpty()) {
+ continue;
+ }
+ PAIMON_ASSIGN_OR_RAISE(BinaryRow partition,
+ path_factory_->ToBinaryRow(partition_bucket.partition));
+ const int32_t bucket = partition_bucket.bucket;
+ std::map<std::string, std::shared_ptr<DataFileMeta>> active_files;
+ for (const ManifestEntry& entry : committed_files) {
+ if (entry.Partition() == partition && entry.Bucket() == bucket) {
+ active_files[entry.File()->file_name] = entry.File();
+ }
+ }
+ // All earlier prepared-but-uncommitted progress is supplied with this commit. Include
+ // its new files as well, so handoff and multiple prepares need no writer-local directory.
+ for (const ManifestEntry& entry : result->delta_files) {
+ if (!(entry.Partition() == partition) || entry.Bucket() != bucket) {
+ continue;
+ }
+ if (entry.Kind() == FileKind::Add()) {
+ active_files[entry.File()->file_name] = entry.File();
+ } else {
+ active_files.erase(entry.File()->file_name);
+ }
+ }
+ std::vector<std::shared_ptr<DataFileMeta>> data_files;
+ for (const auto& [_, file] : active_files) {
+ data_files.push_back(file);
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<DataFilePathFactory> data_path_factory,
+ path_factory_->CreateDataFilePathFactory(partition, bucket));
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<RealtimeOffsetFileIndexLookup> lookup,
+ RealtimeOffsetFileIndexLookup::Create(
+ schema_, table_schema_->Id(),
+ schema_->GetFieldByName(options_.GetRealtimeDeduplicateKeyFields().front()),
+ data_files, data_path_factory, fs_, memory_pool_, options_));
+ using FilePositions = std::map<std::string, RoaringBitmap32>;
+ PAIMON_ASSIGN_OR_RAISE(FilePositions positions, lookup->LookupFilePositions(offsets));
+ if (positions.empty()) {
+ continue;
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<IndexPathFactory> index_path_factory,
+ path_factory_->CreateIndexFileFactory(partition, bucket));
+ auto dv_file = std::make_shared<DeletionVectorsIndexFile>(fs_, index_path_factory,
+ /*bitmap64=*/false, memory_pool_);
+ std::vector<std::shared_ptr<IndexFileMeta>> baseline_files;
+ for (const IndexManifestEntry& entry : committed_indexes) {
+ if (entry.partition == partition && entry.bucket == bucket &&
+ entry.index_file->IndexType() == DeletionVectorsIndexFile::DELETION_VECTORS_INDEX) {
+ baseline_files.push_back(entry.index_file);
+ }
+ }
+ using DeletionVectors = std::map<std::string, std::shared_ptr<DeletionVector>>;
+ PAIMON_ASSIGN_OR_RAISE(DeletionVectors baseline,
+ dv_file->ReadAllDeletionVectors(baseline_files));
+ // Compaction can retire old files and move their rows. Never carry old physical row
+ // positions across snapshots: resolve logical offsets in the current active files above.
+ for (auto iter = baseline.begin(); iter != baseline.end();) {
+ if (active_files.count(iter->first) == 0) {
+ iter = baseline.erase(iter);
+ } else {
+ ++iter;
+ }
+ }
+ BucketedDvMaintainer maintainer(dv_file, baseline);
+ for (const auto& [file_name, bitmap] : positions) {
+ for (RoaringBitmap32::Iterator position = bitmap.Begin(); position != bitmap.End();
+ ++position) {
+ PAIMON_RETURN_NOT_OK(maintainer.NotifyNewDeletion(file_name, *position));
+ }
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<IndexFileMeta>> index,
+ maintainer.WriteDeletionVectorsIndex());
+ if (index) {
+ result->temporary_files.push_back(dv_file->Path(index.value()));
+ result->index_entries.emplace_back(FileKind::Add(), partition, bucket, index.value());
+ }
+ }
+ cleanup.Release();
+ return result;
+}
+
Result<FileStoreCommitImpl::TryCommitResult> FileStoreCommitImpl::TryCommit(
const std::vector<ManifestEntry>& delta_files,
const std::vector<ManifestEntry>& changelog_files,
@@ -1040,6 +1219,11 @@ Result<FileStoreCommitImpl::TryCommitResult> FileStoreCommitImpl::TryCommit(
}
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<CommitChanges> commit_changes,
changes_provider->Provide(latest_snapshot));
+ ScopeGuard unpublished_cleanup([&]() {
+ for (const std::string& path : commit_changes->temporary_files) {
+ [[maybe_unused]] Status status = fs_->Delete(path, /*recursive=*/false);
+ }
+ });
using SnapshotProperties = std::map<std::string, std::string>;
PAIMON_ASSIGN_OR_RAISE(
SnapshotProperties snapshot_properties,
@@ -1047,6 +1231,9 @@ Result<FileStoreCommitImpl::TryCommitResult> FileStoreCommitImpl::TryCommit(
reset_all_realtime_progress,
removed_realtime_partitions, *partition_computer_, fs_,
root_path_, snapshot_manager_->Branch()));
+ // From publication onward an error may have an uncertain outcome. Keep generated DVs
+ // unless the publisher explicitly reports that this attempt lost the snapshot race.
+ unpublished_cleanup.Release();
PAIMON_ASSIGN_OR_RAISE(
bool commit_success,
TryCommitOnce(commit_changes->delta_files, commit_changes->changelog_files,
@@ -1055,6 +1242,10 @@ Result<FileStoreCommitImpl::TryCommitResult> FileStoreCommitImpl::TryCommit(
if (commit_success) {
break;
}
+ for (const std::string& temporary_file : commit_changes->temporary_files) {
+ [[maybe_unused]] Status cleanup_status =
+ fs_->Delete(temporary_file, /*recursive=*/false);
+ }
if (!realtime_ranges.empty()) {
auto offsets_iter = snapshot_properties.find(RealtimeCommitProperties::kOffsetsKey);
if (offsets_iter != snapshot_properties.end()) {
diff --git a/src/paimon/core/operation/file_store_commit_impl.h b/src/paimon/core/operation/file_store_commit_impl.h
index 94f00bef..65d6c78e 100644
--- a/src/paimon/core/operation/file_store_commit_impl.h
+++ b/src/paimon/core/operation/file_store_commit_impl.h
@@ -70,6 +70,8 @@ class SchemaManager;
class TableSchema;
class BinaryRowPartitionComputer;
class CommitChangesProvider;
+struct CommitChanges;
+class RoaringBitmap64;
class CommitMessage;
class Executor;
class FileSystem;
@@ -204,6 +206,11 @@ class FileStoreCommitImpl : public FileStoreCommit {
void ReportCommit(const ManifestEntryChanges& changes, int64_t commit_duration,
int32_t generated_snapshot, int32_t attempt);
+ Result<std::shared_ptr<CommitChanges>> MaterializeRealtimeDeletions(
+ const ManifestEntryChanges& changes,
+ const std::map<RealtimePartitionBucket, RoaringBitmap64>& deleted_offsets,
+ const std::optional<Snapshot>& latest_snapshot) const;
+
Result<TryCommitResult> TryCommit(
const std::vector<ManifestEntry>& delta_files,
const std::vector<ManifestEntry>& changelog_files,
diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp
index 02ce10c4..9600ce20 100644
--- a/src/paimon/core/operation/file_store_commit_impl_test.cpp
+++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp
@@ -1476,7 +1476,7 @@ TEST_F(FileStoreCommitImplTest, TestRealtimeCommitIsIdempotentThroughTheCatalog)
CatalogCommitSpec spec;
spec.catalog = catalog;
spec.commit_user = "realtime_commit_user";
- spec.options = {{Options::REALTIME_ENABLED, "true"}};
+ spec.options = {{Options::REALTIME_ENABLED, "true"}, {Options::BUCKET, "2"}};
ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit, CreateCatalogCommit(spec));
auto commit_impl = std::dynamic_pointer_cast<FileStoreCommitImpl>(
std::shared_ptr<FileStoreCommit>(std::move(commit)));
@@ -1504,7 +1504,7 @@ TEST_F(FileStoreCommitImplTest, TestRealtimeCommitReportsAnUnpublishedHistory) {
CatalogCommitSpec spec;
spec.catalog = catalog;
spec.commit_user = "realtime_commit_user";
- spec.options = {{Options::REALTIME_ENABLED, "true"}};
+ spec.options = {{Options::REALTIME_ENABLED, "true"}, {Options::BUCKET, "2"}};
ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit, CreateCatalogCommit(spec));
auto commit_impl = std::dynamic_pointer_cast<FileStoreCommitImpl>(
std::shared_ptr<FileStoreCommit>(std::move(commit)));
@@ -1797,6 +1797,7 @@ TEST_F(FileStoreCommitImplTest, TestRealtimeCommitRebasesAfterSnapshotConflict)
CommitContextBuilder realtime_builder(table_path_, "realtime_commit_user");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> realtime_context,
realtime_builder.AddOption(Options::REALTIME_ENABLED, "true")
+ .AddOption(Options::BUCKET, "2")
.AddOption(Options::COMMIT_MAX_RETRIES, "1")
.AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms")
.AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms")
diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index 2ae664f8..244a5c48 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -49,6 +49,7 @@
#include "paimon/core/realtime/realtime_context_impl.h"
#include "paimon/core/realtime/realtime_schema_layout.h"
#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/schema_validation.h"
#include "paimon/core/schema/table_schema.h"
#include "paimon/core/table/bucket_mode.h"
#include "paimon/core/table/format/format_table_file_store_write.h"
@@ -89,7 +90,7 @@ Status RestoreRealtimeCommittedProgress(const std::shared_ptr<RealtimeContext>&
RealtimeOffsetMap realtime_committed_offsets,
RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem()));
PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress(
- latest_snapshot->Id(), realtime_committed_offsets));
+ latest_snapshot->Id(), realtime_committed_offsets, /*committed_data_files=*/{}));
}
return Status::OK();
}
@@ -248,6 +249,7 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
PAIMON_ASSIGN_OR_RAISE(
CoreOptions options,
CoreOptions::FromMap(opts, specific_fs, ctx->GetFileSystemSchemeToIdentifierMap()));
+ PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options));
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> partition_schema,
FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys()));
@@ -284,8 +286,14 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
if (ctx->GetRealtimeContext() && !options.RealtimeEnabled()) {
return Status::Invalid("real-time write requires realtime.enabled=true");
}
+ if (ctx->GetRealtimeContext()) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
+ RealtimeContextImpl::Cast(ctx->GetRealtimeContext()));
+ PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable());
+ }
if (schema->PrimaryKeys().empty()) {
// append table
+ const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty();
bool need_dv_maintainer_factory = options.DeletionVectorsEnabled();
if (options.GetBucket() == -1) {
need_dv_maintainer_factory = false;
@@ -295,17 +303,15 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
fmt::format("not support bucket {} in append table", options.GetBucket()));
}
if (ctx->GetRealtimeContext()) {
- if (options.GetBucket() <= 0) {
- return Status::Invalid("real-time append write requires fixed bucket mode");
- }
- if (options.DataEvolutionEnabled() || !ctx->GetWriteSchema().empty()) {
- return Status::Invalid("real-time append write does not support data evolution");
+ if (!ctx->GetWriteSchema().empty()) {
+ return Status::Invalid("real-time append write does not support a custom schema");
}
- if (options.DeletionVectorsEnabled()) {
- return Status::Invalid("real-time append write does not support deletion vectors");
+ if (!realtime_deduplicate) {
+ PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(),
+ snapshot_manager, options));
}
- PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(),
- snapshot_manager, options));
+ } else if (realtime_deduplicate) {
+ return Status::Invalid("real-time deduplicate write requires a real-time context");
}
std::shared_ptr<arrow::Schema> write_schema = arrow_schema;
const auto& write_field_names = ctx->GetWriteSchema();
@@ -326,9 +332,10 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
std::shared_ptr<RealtimeSchemaLayout> realtime_schema_layout;
if (ctx->GetRealtimeContext()) {
- PAIMON_ASSIGN_OR_RAISE(
- std::unique_ptr<RealtimeSchemaLayout> layout,
- RealtimeSchemaLayout::Create(RealtimeStoreMode::APPEND_ONLY, write_schema));
+ const RealtimeStoreMode mode = realtime_deduplicate ? RealtimeStoreMode::DEDUPLICATE
+ : RealtimeStoreMode::APPEND_ONLY;
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RealtimeSchemaLayout> layout,
+ RealtimeSchemaLayout::Create(mode, write_schema));
realtime_schema_layout = std::move(layout);
}
@@ -353,6 +360,14 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
partition_schema, dv_maintainer_factory, io_manager, options, ignore_previous_files,
ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->GetRealtimeContext(),
ctx->GetExecutor(), ctx->GetMemoryPool());
+ if (ctx->GetRealtimeContext() && realtime_deduplicate) {
+ PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
+ snapshot_manager->LatestSnapshot());
+ if (latest_snapshot) {
+ PAIMON_RETURN_NOT_OK(
+ file_store_write->RefreshCommittedSnapshot(latest_snapshot->Id()));
+ }
+ }
return std::unique_ptr<FileStoreWrite>(std::move(file_store_write));
} else {
// pk table
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index 35a703b2..be72ad91 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -195,7 +195,8 @@ Status KeyValueFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) {
options_.GetFileSystem()));
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
RealtimeContextImpl::Cast(realtime_context_));
- return realtime_context_impl->AdvanceCommittedProgress(snapshot_id, committed_offsets);
+ return realtime_context_impl->AdvanceCommittedProgress(snapshot_id, committed_offsets,
+ /*committed_data_files=*/{});
}
Status KeyValueFileStoreWrite::Close() {
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index 988fb3f5..c0e94e23 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -461,12 +461,6 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
RecordBatch::RowKind::UPDATE_AFTER});
ASSERT_OK(writer->Write(std::move(batch)));
ASSERT_NOK(writer->Write(MakeBatch(schema, R"([[3, "missing-offset"]])")));
- ASSERT_NOK_WITH_MSG(
- writer->Write(
- MakeBatch(realtime_schema, R"([[31, 3, "duplicate-a"], [31, 4, "duplicate-b"]])")),
- "offsets must be strictly increasing");
- ASSERT_NOK_WITH_MSG(writer->Write(MakeBatch(realtime_schema, R"([[30, 3, "backwards"]])")),
- "offset moved backwards or was duplicated");
ASSERT_NOK(writer->Write(MakeBatch(realtime_schema, R"([[null, 3, "null-offset"]])")));
std::shared_ptr<Metrics> building_metrics = writer->GetMetrics();
ASSERT_OK_AND_ASSIGN(double building_rows,
@@ -501,6 +495,35 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeWrite) {
ASSERT_EQ(2, row_count);
ASSERT_EQ(0, TestHelper::CountChannelFiles(dir->GetFileSystem(), dir->Str()));
ASSERT_OK(writer->Close());
+
+ // Each bucket-writer failure invalidates its context, so isolate the invalid batches.
+ const std::vector<std::pair<std::string, std::string>> invalid_batches = {
+ {R"([[31, 3, "duplicate-a"], [31, 4, "duplicate-b"]])",
+ "offsets must be strictly increasing"},
+ {R"([[30, 3, "backwards"]])", "offset moved backwards or was duplicated"}};
+ for (const auto& [json, expected_error] : invalid_batches) {
+ SCOPED_TRACE(expected_error);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> invalid_context,
+ RealtimeContext::Create());
+ WriteContextBuilder invalid_builder(table_path, "invalid");
+ invalid_builder.SetOptions(options)
+ .WithStreamingMode(true)
+ .WithRealtimeContext(invalid_context)
+ .WithTempDirectory(dir->Str());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> invalid_write_context,
+ invalid_builder.Finish());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> invalid_writer,
+ FileStoreWrite::Create(std::move(invalid_write_context)));
+ ASSERT_OK(invalid_writer->Write(MakeBatch(realtime_schema, R"([[30, 1, "existing"]])")));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> prepared,
+ invalid_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+ ASSERT_EQ(1, prepared.size());
+ ASSERT_NOK_WITH_MSG(invalid_writer->Write(MakeBatch(realtime_schema, json)),
+ expected_error);
+ ASSERT_NOK_WITH_MSG(ReadRealtimePrimaryKeyStoreRows(invalid_context),
+ "real-time context cannot be reused");
+ ASSERT_OK(invalid_writer->Close());
+ }
}
TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
@@ -559,8 +582,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimePool) {
ASSERT_NOK_WITH_MSG(rejecting_writer->Write(MakeBatch(realtime_schema, R"([[0, 2, "two"]])")),
"Out of memory");
ASSERT_GT(rejecting_pool->allocation_count, rejecting_allocations_before_write);
- ASSERT_OK_AND_ASSIGN(auto rejected_rows, ReadRealtimePrimaryKeyStoreRows(rejecting_context));
- ASSERT_TRUE(rejected_rows.empty());
+ ASSERT_NOK_WITH_MSG(ReadRealtimePrimaryKeyStoreRows(rejecting_context),
+ "real-time context cannot be reused");
ASSERT_OK(rejecting_writer->Close());
}
@@ -621,11 +644,6 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) {
ASSERT_OK_AND_ASSIGN(auto store_rows, ReadRealtimePrimaryKeyStoreRows(realtime_context));
ASSERT_EQ((decltype(store_rows){{0, 1, "legal", max - 1, max - 1}}), store_rows);
- ASSERT_NOK_WITH_MSG(
- writer->Write(MakeBatch(realtime_schema, fmt::format(R"([[{}, 2, "overflow"]])", max))),
- "real-time offset range exceeds INT64_MAX");
- ASSERT_OK_AND_ASSIGN(store_rows, ReadRealtimePrimaryKeyStoreRows(realtime_context));
- ASSERT_EQ((decltype(store_rows){{0, 1, "legal", max - 1, max - 1}}), store_rows);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context_impl,
RealtimeContextImpl::Cast(realtime_context));
ASSERT_OK_AND_ASSIGN(RealtimeReadState read_state, context_impl->AcquireReadState());
@@ -635,6 +653,12 @@ TEST_F(KeyValueFileStoreWriteTest, TestRealtimeLimits) {
ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> boundary_progress,
writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
ASSERT_EQ(1, boundary_progress.size());
+ ASSERT_EQ(OffsetRange(max - 1, max), boundary_progress[0].offset_range);
+ ASSERT_NOK_WITH_MSG(
+ writer->Write(MakeBatch(realtime_schema, fmt::format(R"([[{}, 2, "overflow"]])", max))),
+ "real-time offset range exceeds INT64_MAX");
+ ASSERT_NOK_WITH_MSG(ReadRealtimePrimaryKeyStoreRows(realtime_context),
+ "real-time context cannot be reused");
ASSERT_OK(writer->Close());
ASSERT_GE(snapshot_id, 1);
}
diff --git a/src/paimon/core/operation/raw_file_split_read.cpp b/src/paimon/core/operation/raw_file_split_read.cpp
index 5f75ea25..58a19fc6 100644
--- a/src/paimon/core/operation/raw_file_split_read.cpp
+++ b/src/paimon/core/operation/raw_file_split_read.cpp
@@ -138,14 +138,26 @@ Result<std::unique_ptr<BatchReader>> RawFileSplitRead::CreateReader(
const BinaryRow& partition, int32_t bucket,
const std::vector<std::shared_ptr<DataFileMeta>>& data_files,
DeletionVector::Factory dv_factory, const std::optional<std::vector<Range>>& local_row_ranges) {
+ return CreateReader(partition, bucket, data_files, std::move(dv_factory), local_row_ranges,
+ raw_read_schema_, context_->GetPredicate());
+}
+
+Result<std::unique_ptr<BatchReader>> RawFileSplitRead::CreateReader(
+ const BinaryRow& partition, int32_t bucket,
+ const std::vector<std::shared_ptr<DataFileMeta>>& data_files,
+ DeletionVector::Factory dv_factory, const std::optional<std::vector<Range>>& local_row_ranges,
+ const std::shared_ptr<arrow::Schema>& read_schema,
+ const std::shared_ptr<Predicate>& predicate) {
PAIMON_RETURN_NOT_OK(ValidateFileLocalRowRanges(data_files, local_row_ranges));
- const auto& predicate = context_->GetPredicate();
+ if (!read_schema) {
+ return Status::Invalid("raw file read schema is null");
+ }
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<DataFilePathFactory> data_file_path_factory,
path_factory_->CreateDataFilePathFactory(partition, bucket));
PAIMON_ASSIGN_OR_RAISE(
std::vector<std::unique_ptr<FileBatchReader>> raw_file_readers,
- CreateRawFileReaders(partition, data_files, raw_read_schema_, predicate, dv_factory,
+ CreateRawFileReaders(partition, data_files, read_schema, predicate, dv_factory,
local_row_ranges, data_file_path_factory,
/*extra_format_options=*/{}));
diff --git a/src/paimon/core/operation/raw_file_split_read.h b/src/paimon/core/operation/raw_file_split_read.h
index 0ffe4582..6e900768 100644
--- a/src/paimon/core/operation/raw_file_split_read.h
+++ b/src/paimon/core/operation/raw_file_split_read.h
@@ -85,6 +85,13 @@ class RawFileSplitRead : public AbstractSplitRead {
const std::vector<std::shared_ptr<DataFileMeta>>& files, DeletionVector::Factory dv_factory,
const std::optional<std::vector<Range>>& local_row_ranges);
+ Result<std::unique_ptr<BatchReader>> CreateReader(
+ const BinaryRow& partition, int32_t bucket,
+ const std::vector<std::shared_ptr<DataFileMeta>>& files, DeletionVector::Factory dv_factory,
+ const std::optional<std::vector<Range>>& local_row_ranges,
+ const std::shared_ptr<arrow::Schema>& read_schema,
+ const std::shared_ptr<Predicate>& predicate);
+
Result<bool> Match(const std::shared_ptr<Split>& split, bool force_keep_delete) const override;
Result<std::unique_ptr<FileBatchReader>> ApplyIndexAndDvReaderIfNeeded(
std::unique_ptr<FileBatchReader>&& file_reader, const std::shared_ptr<DataFileMeta>& file,
diff --git a/src/paimon/core/realtime/arrow_deduplicate_realtime_store.cpp b/src/paimon/core/realtime/arrow_deduplicate_realtime_store.cpp
new file mode 100644
index 00000000..9d44f8e6
--- /dev/null
+++ b/src/paimon/core/realtime/arrow_deduplicate_realtime_store.cpp
@@ -0,0 +1,542 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/realtime/arrow_deduplicate_realtime_store.h"
+
+#include <algorithm>
+#include <utility>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "arrow/c/helpers.h"
+#include "arrow/compute/api.h"
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/core/realtime/realtime_utils.h"
+#include "paimon/macros.h"
+#include "paimon/metrics.h"
+#include "paimon/record_batch.h"
+
+namespace paimon {
+namespace {
+
+Result<std::shared_ptr<arrow::StructArray>> ProjectByName(
+ const std::shared_ptr<arrow::StructArray>& source,
+ const std::shared_ptr<arrow::Schema>& target_schema) {
+ arrow::ArrayVector fields;
+ fields.reserve(target_schema->num_fields());
+ for (const std::shared_ptr<arrow::Field>& field : target_schema->fields()) {
+ std::shared_ptr<arrow::Array> array = source->GetFieldByName(field->name());
+ if (!array || !array->type()->Equals(field->type())) {
+ return Status::Invalid("real-time deduplicate reader cannot project field: ",
+ field->name());
+ }
+ fields.push_back(std::move(array));
+ }
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray> result,
+ arrow::StructArray::Make(fields, target_schema->fields()));
+ return result;
+}
+
+Result<BatchReader::ReadBatch> ExportBatch(const std::shared_ptr<arrow::StructArray>& data,
+ const std::shared_ptr<arrow::MemoryPool>& pool) {
+ auto array = std::make_unique<ArrowArray>();
+ auto schema = std::make_unique<ArrowSchema>();
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*data, array.get(), schema.get()));
+ PAIMON_RETURN_NOT_OK(AddArrowArrayLifetime(array.get(), schema.get(), pool));
+ return std::make_pair(std::move(array), std::move(schema));
+}
+
+Result<std::shared_ptr<arrow::StructArray>> FilterDeletedOffsets(
+ const std::shared_ptr<arrow::StructArray>& source,
+ const std::shared_ptr<const RoaringBitmap64>& deleted_offsets,
+ const std::shared_ptr<arrow::MemoryPool>& pool) {
+ if (!deleted_offsets || deleted_offsets->IsEmpty() || source->length() == 0) {
+ return source;
+ }
+ std::shared_ptr<arrow::Array> offset_array =
+ source->GetFieldByName(SpecialFields::RealtimeOffset().Name());
+ if (!offset_array || offset_array->type_id() != arrow::Type::INT64 ||
+ offset_array->null_count() != 0) {
+ return Status::Invalid(
+ "real-time deduplicate data must contain non-null int64 _REALTIME_OFFSET");
+ }
+ std::shared_ptr<arrow::Int64Array> offsets =
+ checked_pointer_cast<arrow::Int64Array>(offset_array);
+ arrow::BooleanBuilder selection(pool.get());
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(selection.Reserve(source->length()));
+ // TODO(xinyu.lxy): Merge-scan the ordered offsets with a RoaringBitmap64 iterator instead of
+ // calling Contains for every row.
+ for (int64_t row = 0; row < source->length(); ++row) {
+ selection.UnsafeAppend(!deleted_offsets->Contains(offsets->Value(row)));
+ }
+ std::shared_ptr<arrow::Array> selection_array;
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(selection.Finish(&selection_array));
+ arrow::compute::ExecContext exec_context(pool.get());
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ arrow::Datum filtered,
+ arrow::compute::Filter(source, selection_array, arrow::compute::FilterOptions::Defaults(),
+ &exec_context));
+ return checked_pointer_cast<arrow::StructArray>(filtered.make_array());
+}
+
+} // namespace
+
+class ArrowDeduplicateRealtimeStore::SegmentHandle final : public RealtimeSegmentHandle {
+ public:
+ SegmentHandle(std::shared_ptr<RealtimeSegmentHandle> delegate, OffsetRange offset_range,
+ int64_t row_count, std::shared_ptr<const RoaringBitmap64> offset_deletions)
+ : delegate_(std::move(delegate)),
+ offset_range_(offset_range),
+ row_count_(row_count),
+ offset_deletions_(std::move(offset_deletions)) {}
+
+ OffsetRange GetOffsetRange() const override {
+ return offset_range_;
+ }
+
+ int64_t GetRowCount() const override {
+ return row_count_;
+ }
+
+ const std::shared_ptr<RealtimeSegmentHandle>& Delegate() const {
+ return delegate_;
+ }
+
+ const std::shared_ptr<const RoaringBitmap64>& OffsetDeletions() const {
+ return offset_deletions_;
+ }
+
+ private:
+ std::shared_ptr<RealtimeSegmentHandle> delegate_;
+ OffsetRange offset_range_;
+ int64_t row_count_;
+ std::shared_ptr<const RoaringBitmap64> offset_deletions_;
+};
+
+class ArrowDeduplicateRealtimeStore::ReadView final : public RealtimeReadView {
+ public:
+ ReadView(std::shared_ptr<RealtimeReadView> delegate, std::optional<OffsetRange> offset_range,
+ std::shared_ptr<const RoaringBitmap64> offset_deletions)
+ : delegate_(std::move(delegate)),
+ offset_range_(offset_range),
+ offset_deletions_(std::move(offset_deletions)) {}
+
+ std::optional<OffsetRange> GetOffsetRange() const override {
+ return offset_range_;
+ }
+
+ const std::shared_ptr<RealtimeReadView>& Delegate() const {
+ return delegate_;
+ }
+
+ const std::shared_ptr<const RoaringBitmap64>& OffsetDeletions() const {
+ return offset_deletions_;
+ }
+
+ private:
+ std::shared_ptr<RealtimeReadView> delegate_;
+ std::optional<OffsetRange> offset_range_;
+ std::shared_ptr<const RoaringBitmap64> offset_deletions_;
+};
+
+class ArrowDeduplicateRealtimeStore::OffsetFilteringBatchReader final : public BatchReader {
+ public:
+ OffsetFilteringBatchReader(std::unique_ptr<BatchReader> inner,
+ std::shared_ptr<arrow::Schema> output_schema,
+ std::shared_ptr<const RoaringBitmap64> offset_deletions,
+ std::shared_ptr<arrow::MemoryPool> arrow_pool)
+ : inner_(std::move(inner)),
+ output_schema_(std::move(output_schema)),
+ offset_deletions_(std::move(offset_deletions)),
+ arrow_pool_(std::move(arrow_pool)) {}
+
+ // TODO(xinyu.lxy): Override NextBatchWithBitmap to compose the inner selection with offset
+ // deletions and avoid materializing an intermediate filtered array.
+ Result<ReadBatch> NextBatch() override {
+ while (true) {
+ PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, inner_->NextBatch());
+ if (BatchReader::IsEofBatch(batch)) {
+ return batch;
+ }
+ auto& [c_array, c_schema] = batch;
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> imported,
+ arrow::ImportArray(c_array.get(), c_schema.get()));
+ if (!imported || imported->type_id() != arrow::Type::STRUCT) {
+ return Status::Invalid("real-time deduplicate reader returned a non-StructArray");
+ }
+ std::shared_ptr<arrow::StructArray> source =
+ checked_pointer_cast<arrow::StructArray>(imported);
+ PAIMON_ASSIGN_OR_RAISE(source,
+ FilterDeletedOffsets(source, offset_deletions_, arrow_pool_));
+ if (source->length() == 0) {
+ continue;
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::StructArray> projected,
+ ProjectByName(source, output_schema_));
+ return ExportBatch(projected, arrow_pool_);
+ }
+ }
+
+ std::shared_ptr<Metrics> GetReaderMetrics() const override {
+ return inner_ ? inner_->GetReaderMetrics() : std::make_shared<MetricsImpl>();
+ }
+
+ void Close() override {
+ if (inner_) {
+ inner_->Close();
+ inner_.reset();
+ }
+ }
+
+ private:
+ std::unique_ptr<BatchReader> inner_;
+ std::shared_ptr<arrow::Schema> output_schema_;
+ std::shared_ptr<const RoaringBitmap64> offset_deletions_;
+ std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+};
+
+Result<std::shared_ptr<ArrowDeduplicateRealtimeStore>> ArrowDeduplicateRealtimeStore::Create(
+ const std::shared_ptr<arrow::Schema>& write_schema,
+ const std::vector<std::string>& deduplicate_key_fields,
+ const std::shared_ptr<RealtimeStore>& delegate, const std::shared_ptr<MemoryPool>& memory_pool,
+ const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
+ if (!write_schema || !delegate || !memory_pool || !arrow_pool) {
+ return Status::Invalid("real-time deduplicate store is missing a required dependency");
+ }
+ PAIMON_ASSIGN_OR_RAISE(int32_t key_position, RealtimeUtils::GetDeduplicateKeyPosition(
+ write_schema, deduplicate_key_fields));
+ PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateOffsetField(write_schema));
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<MutableKeyOffsetIndex> building_key_index,
+ MutableKeyOffsetIndex::Create(write_schema->field(key_position), memory_pool));
+ return std::shared_ptr<ArrowDeduplicateRealtimeStore>(new ArrowDeduplicateRealtimeStore(
+ write_schema, key_position, write_schema->field(key_position),
+ std::move(building_key_index), delegate, memory_pool, arrow_pool));
+}
+
+ArrowDeduplicateRealtimeStore::ArrowDeduplicateRealtimeStore(
+ const std::shared_ptr<arrow::Schema>& write_schema, int32_t key_field_position,
+ const std::shared_ptr<arrow::Field>& key_field,
+ std::shared_ptr<MutableKeyOffsetIndex> building_key_index,
+ const std::shared_ptr<RealtimeStore>& delegate, const std::shared_ptr<MemoryPool>& memory_pool,
+ const std::shared_ptr<arrow::MemoryPool>& arrow_pool)
+ : write_schema_(write_schema),
+ key_field_position_(key_field_position),
+ key_field_(key_field),
+ delegate_(delegate),
+ memory_pool_(memory_pool),
+ arrow_pool_(arrow_pool),
+ building_key_index_(std::move(building_key_index)),
+ pending_offset_deletions_(std::make_shared<RoaringBitmap64>()) {}
+
+Status ArrowDeduplicateRealtimeStore::Write(RealtimeWriteBatch&&) {
+ return Status::Invalid(
+ "deduplicate store Write requires key-offset lookup; use RealtimeDeduplicateWriter");
+}
+
+Result<std::shared_ptr<arrow::StructArray>> ArrowDeduplicateRealtimeStore::ExtractKeys(
+ const std::shared_ptr<arrow::StructArray>& data) const {
+ if (!data || !data->type()->Equals(arrow::struct_(write_schema_->fields()))) {
+ return Status::Invalid("deduplicate write batch does not match store schema");
+ }
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::StructArray> keys,
+ arrow::StructArray::Make({data->field(key_field_position_)}, {key_field_}));
+ return keys;
+}
+
+Result<std::shared_ptr<const KeyOffsetLookup>>
+ArrowDeduplicateRealtimeStore::AcquireKeyOffsetLookup() {
+ std::lock_guard<std::mutex> lock(mutex_);
+ if (!building_key_index_) {
+ return Status::Invalid("deduplicate building key-offset index is not initialized");
+ }
+ std::vector<std::shared_ptr<const KeyOffsetLookup>> lookups;
+ if (committed_key_lookup_) {
+ lookups.push_back(committed_key_lookup_);
+ }
+ lookups.reserve(lookups.size() + sealed_states_.size() + 1);
+ for (const SealedState& sealed : sealed_states_) {
+ lookups.push_back(sealed.key_lookup);
+ }
+ lookups.push_back(building_key_index_->Snapshot());
+ return std::shared_ptr<const KeyOffsetLookup>(new CompositeKeyOffsetLookup(std::move(lookups)));
+}
+
+Status ArrowDeduplicateRealtimeStore::EnsureMutableDeletionBitmap() {
+ if (!pending_offset_deletions_.unique()) {
+ pending_offset_deletions_ = std::make_shared<RoaringBitmap64>(*pending_offset_deletions_);
+ }
+ return Status::OK();
+}
+
+Status ArrowDeduplicateRealtimeStore::WriteAndLookup(
+ const std::shared_ptr<arrow::StructArray>& data,
+ const std::vector<RecordBatch::RowKind>& row_kinds, const OffsetRange& offset_range,
+ const RoaringBitmap64& previous_offsets) {
+ if (!data || data->length() <= 0 || static_cast<int64_t>(row_kinds.size()) != data->length()) {
+ return Status::Invalid("deduplicate write data and row kinds are not aligned");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::StructArray> keys, ExtractKeys(data));
+ std::shared_ptr<arrow::Array> raw_offsets =
+ data->GetFieldByName(SpecialFields::RealtimeOffset().Name());
+ if (!raw_offsets || raw_offsets->type_id() != arrow::Type::INT64 ||
+ raw_offsets->null_count() != 0) {
+ return Status::Invalid("deduplicate write requires non-null int64 offsets");
+ }
+ std::shared_ptr<arrow::Int64Array> offsets =
+ checked_pointer_cast<arrow::Int64Array>(raw_offsets);
+
+ std::lock_guard<std::mutex> lock(mutex_);
+ if (!building_key_index_) {
+ return Status::Invalid("deduplicate building key-offset index is not initialized");
+ }
+ if (building_offset_range_ && offset_range.begin < building_offset_range_->end) {
+ return Status::Invalid("real-time offset ranges must be ordered and non-overlapping");
+ }
+ PAIMON_RETURN_NOT_OK(EnsureMutableDeletionBitmap());
+ *pending_offset_deletions_ |= previous_offsets;
+
+ arrow::BooleanBuilder add_selection(arrow_pool_.get());
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(add_selection.Reserve(data->length()));
+ int64_t add_count = 0;
+ for (int64_t row = 0; row < data->length(); ++row) {
+ std::shared_ptr<arrow::StructArray> one_key =
+ checked_pointer_cast<arrow::StructArray>(keys->Slice(row, 1));
+ PAIMON_ASSIGN_OR_RAISE(RoaringBitmap64 earlier_in_memory,
+ building_key_index_->LookupOffsets(one_key));
+ *pending_offset_deletions_ |= earlier_in_memory;
+
+ const RecordBatch::RowKind row_kind = row_kinds[row];
+ const bool is_add = row_kind == RecordBatch::RowKind::INSERT ||
+ row_kind == RecordBatch::RowKind::UPDATE_AFTER;
+ const bool is_delete = row_kind == RecordBatch::RowKind::DELETE ||
+ row_kind == RecordBatch::RowKind::UPDATE_BEFORE;
+ if (!is_add && !is_delete) {
+ return Status::Invalid("unknown row kind in real-time deduplicate write");
+ }
+ add_selection.UnsafeAppend(is_add);
+ if (!is_add) {
+ PAIMON_RETURN_NOT_OK(building_key_index_->Erase(one_key));
+ continue;
+ }
+ std::shared_ptr<arrow::Int64Array> one_offset =
+ checked_pointer_cast<arrow::Int64Array>(offsets->Slice(row, 1));
+ PAIMON_RETURN_NOT_OK(building_key_index_->Add(one_key, one_offset));
+ building_row_offsets_.Add(offsets->Value(row));
+ ++add_count;
+ }
+
+ if (add_count > 0) {
+ std::shared_ptr<arrow::Array> selection;
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(add_selection.Finish(&selection));
+ arrow::compute::ExecContext exec_context(arrow_pool_.get());
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ arrow::Datum filtered,
+ arrow::compute::Filter(data, selection, arrow::compute::FilterOptions::Defaults(),
+ &exec_context));
+ std::shared_ptr<arrow::StructArray> additions =
+ checked_pointer_cast<arrow::StructArray>(filtered.make_array());
+ auto c_array = std::make_unique<ArrowArray>();
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*additions, c_array.get()));
+ PAIMON_RETURN_NOT_OK(AddArrowArrayLifetime(c_array.get(), /*schema=*/nullptr, arrow_pool_));
+ RecordBatchBuilder builder(c_array.get());
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> append_batch, builder.Finish());
+ PAIMON_RETURN_NOT_OK(
+ delegate_->Write(RealtimeWriteBatch{std::move(append_batch), offset_range}));
+ }
+ if (!building_offset_range_) {
+ building_offset_range_ = offset_range;
+ } else {
+ building_offset_range_ = OffsetRange(building_offset_range_->begin, offset_range.end);
+ }
+ return Status::OK();
+}
+
+Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>>
+ArrowDeduplicateRealtimeStore::SealForCommit() {
+ std::lock_guard<std::mutex> lock(mutex_);
+ if (!building_offset_range_) {
+ return std::optional<std::shared_ptr<RealtimeSegmentHandle>>();
+ }
+ // Delegate rotation and the key/deletion state rotation must be atomic. Deduplicate mode
+ // currently rejects real-time spill, so the delegate only performs its short in-memory
+ // rotation here.
+ // TODO(xinyu.lxy): Introduce two-phase store sealing before allowing deduplicate spill, so
+ // delegate spill I/O can complete without holding this mutex.
+ PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<RealtimeSegmentHandle>> delegate_segment,
+ delegate_->SealForCommit());
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<MutableKeyOffsetIndex> next_building_index,
+ MutableKeyOffsetIndex::Create(key_field_, memory_pool_));
+ std::shared_ptr<const RoaringBitmap64> checkpoint = pending_offset_deletions_;
+ RoaringBitmap64 live_rows =
+ RoaringBitmap64::AndNot(building_row_offsets_, *pending_offset_deletions_);
+ auto handle = std::make_shared<SegmentHandle>(delegate_segment.value_or(nullptr),
+ building_offset_range_.value(),
+ live_rows.Cardinality(), checkpoint);
+ sealed_states_.push_back(
+ SealedState{building_offset_range_.value(), building_key_index_->Snapshot(), checkpoint});
+ building_key_index_ = std::move(next_building_index);
+ building_row_offsets_ = RoaringBitmap64();
+ building_offset_range_.reset();
+ return std::optional<std::shared_ptr<RealtimeSegmentHandle>>(std::move(handle));
+}
+
+Result<std::shared_ptr<const RoaringBitmap64>> ArrowDeduplicateRealtimeStore::OffsetDeletionsOf(
+ const std::shared_ptr<RealtimeSegmentHandle>& segment) {
+ std::shared_ptr<SegmentHandle> handle = std::dynamic_pointer_cast<SegmentHandle>(segment);
+ if (!handle) {
+ return Status::Invalid("segment was not created by the deduplicate real-time store");
+ }
+ return handle->OffsetDeletions();
+}
+
+Result<std::shared_ptr<const RoaringBitmap64>> ArrowDeduplicateRealtimeStore::OffsetDeletionsOf(
+ const std::shared_ptr<RealtimeReadView>& view) {
+ std::shared_ptr<ReadView> deduplicate_view = std::dynamic_pointer_cast<ReadView>(view);
+ if (!deduplicate_view) {
+ return Status::Invalid("read view was not created by the deduplicate real-time store");
+ }
+ return deduplicate_view->OffsetDeletions();
+}
+
+Result<std::vector<std::unique_ptr<BatchReader>>>
+ArrowDeduplicateRealtimeStore::CreateCommitReaders(
+ const std::shared_ptr<RealtimeSegmentHandle>& segment) {
+ std::shared_ptr<SegmentHandle> handle = std::dynamic_pointer_cast<SegmentHandle>(segment);
+ if (!handle) {
+ return Status::Invalid("segment was not created by the deduplicate real-time store");
+ }
+ if (!handle->Delegate()) {
+ return std::vector<std::unique_ptr<BatchReader>>();
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> inner,
+ delegate_->CreateCommitReaders(handle->Delegate()));
+ std::vector<std::unique_ptr<BatchReader>> readers;
+ readers.reserve(inner.size());
+ for (std::unique_ptr<BatchReader>& reader : inner) {
+ readers.push_back(std::make_unique<OffsetFilteringBatchReader>(
+ std::move(reader), write_schema_, handle->OffsetDeletions(), arrow_pool_));
+ }
+ return readers;
+}
+
+Result<std::shared_ptr<RealtimeReadView>> ArrowDeduplicateRealtimeStore::AcquireReadView() {
+ std::lock_guard<std::mutex> lock(mutex_);
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> delegate_view,
+ delegate_->AcquireReadView());
+ std::optional<OffsetRange> range;
+ if (!sealed_states_.empty()) {
+ range = OffsetRange(sealed_states_.front().offset_range.begin,
+ sealed_states_.back().offset_range.end);
+ }
+ if (building_offset_range_) {
+ range =
+ range
+ ? std::optional<OffsetRange>(OffsetRange(range->begin, building_offset_range_->end))
+ : building_offset_range_;
+ }
+ return std::shared_ptr<RealtimeReadView>(
+ new ReadView(std::move(delegate_view), range, pending_offset_deletions_));
+}
+
+Result<std::vector<std::unique_ptr<BatchReader>>> ArrowDeduplicateRealtimeStore::CreateQueryReaders(
+ const std::shared_ptr<RealtimeReadView>& view, const RealtimeQueryContext& context) {
+ std::shared_ptr<ReadView> deduplicate_view = std::dynamic_pointer_cast<ReadView>(view);
+ if (!deduplicate_view) {
+ return Status::Invalid("read view was not created by the deduplicate real-time store");
+ }
+ if (!context.read_schema || !context.read_schema->release) {
+ return Status::Invalid("deduplicate query read schema is null");
+ }
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> output_schema,
+ arrow::ImportSchema(context.read_schema));
+ PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateOffsetField(output_schema));
+ ArrowSchema inner_c_schema;
+ ArrowSchemaMarkReleased(&inner_c_schema);
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*output_schema, &inner_c_schema));
+ RealtimeQueryContext inner_context{&inner_c_schema, context.predicate, context.read_batch_size};
+ PAIMON_ASSIGN_OR_RAISE(
+ std::vector<std::unique_ptr<BatchReader>> inner,
+ delegate_->CreateQueryReaders(deduplicate_view->Delegate(), inner_context));
+ std::vector<std::unique_ptr<BatchReader>> readers;
+ readers.reserve(inner.size());
+ for (std::unique_ptr<BatchReader>& reader : inner) {
+ readers.push_back(std::make_unique<OffsetFilteringBatchReader>(
+ std::move(reader), output_schema, deduplicate_view->OffsetDeletions(), arrow_pool_));
+ }
+ return readers;
+}
+
+Status ArrowDeduplicateRealtimeStore::AdvanceCommittedOffset(int64_t) {
+ return Status::Invalid(
+ "deduplicate committed advancement requires an atomic key-offset lookup replacement");
+}
+
+Status ArrowDeduplicateRealtimeStore::AdvanceCommittedOffsetAndLookup(
+ int64_t committed_end_offset, const std::shared_ptr<const KeyOffsetLookup>& committed_lookup) {
+ if (committed_end_offset < 0) {
+ return Status::Invalid("committed real-time offset must not be negative");
+ }
+ if (!committed_lookup) {
+ return Status::Invalid("committed key-offset lookup must not be null");
+ }
+ std::lock_guard<std::mutex> lock(mutex_);
+ PAIMON_RETURN_NOT_OK(delegate_->AdvanceCommittedOffset(committed_end_offset));
+ std::shared_ptr<const RoaringBitmap64> applied_checkpoint;
+ size_t reclaimed_count = 0;
+ while (reclaimed_count < sealed_states_.size() &&
+ sealed_states_[reclaimed_count].offset_range.end <= committed_end_offset) {
+ applied_checkpoint = sealed_states_[reclaimed_count].offset_deletions;
+ ++reclaimed_count;
+ }
+ if (reclaimed_count > 0) {
+ sealed_states_.erase(sealed_states_.begin(), sealed_states_.begin() + reclaimed_count);
+ }
+ if (applied_checkpoint) {
+ PAIMON_RETURN_NOT_OK(EnsureMutableDeletionBitmap());
+ *pending_offset_deletions_ -= *applied_checkpoint;
+ }
+ committed_key_lookup_ = committed_lookup;
+ return Status::OK();
+}
+
+Status ArrowDeduplicateRealtimeStore::AttachCommittedKeyOffsetLookup(
+ const std::shared_ptr<const KeyOffsetLookup>& committed_lookup) {
+ std::lock_guard<std::mutex> lock(mutex_);
+ if (committed_key_lookup_ && committed_key_lookup_ != committed_lookup) {
+ return Status::Invalid(
+ "committed key-offset lookup is already attached to deduplicate store");
+ }
+ committed_key_lookup_ = committed_lookup;
+ return Status::OK();
+}
+
+RealtimeStoreDataUsage ArrowDeduplicateRealtimeStore::GetDataUsage() const {
+ return delegate_->GetDataUsage();
+}
+
+uint64_t ArrowDeduplicateRealtimeStore::GetMemoryUsage() const {
+ return delegate_->GetMemoryUsage();
+}
+
+} // namespace paimon
diff --git a/src/paimon/core/realtime/arrow_deduplicate_realtime_store.h b/src/paimon/core/realtime/arrow_deduplicate_realtime_store.h
new file mode 100644
index 00000000..191b20ab
--- /dev/null
+++ b/src/paimon/core/realtime/arrow_deduplicate_realtime_store.h
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "paimon/core/realtime/realtime_key_offset_index.h"
+#include "paimon/realtime/realtime_store.h"
+#include "paimon/utils/roaring_bitmap64.h"
+
+namespace arrow {
+class MemoryPool;
+class Schema;
+class StructArray;
+class Field;
+} // namespace arrow
+
+namespace paimon {
+
+class MemoryPool;
+
+/// Framework-owned Arrow deduplication semantics over a pluggable append-store delegate.
+///
+/// The user-defined deduplicate key is an append-table key, not a Paimon primary key. Every add row
+/// keeps its `_REALTIME_OFFSET` in memory and in the eventual data file. A read view pins one
+/// immutable bitmap, so later writes cannot change the set of rows visible to that query. The
+/// delegate owns physical buffering, readers, and spill; this wrapper always owns key indexes and
+/// offset DVs.
+class ArrowDeduplicateRealtimeStore final : public RealtimeStore {
+ public:
+ static Result<std::shared_ptr<ArrowDeduplicateRealtimeStore>> Create(
+ const std::shared_ptr<arrow::Schema>& write_schema,
+ const std::vector<std::string>& deduplicate_key_fields,
+ const std::shared_ptr<RealtimeStore>& delegate,
+ const std::shared_ptr<MemoryPool>& memory_pool,
+ const std::shared_ptr<arrow::MemoryPool>& arrow_pool);
+
+ /// The generic API cannot perform the committed `.offset` lookup. Deduplicate writers must
+ /// call `AcquireKeyOffsetLookup`, perform lookup outside the store lock, then call
+ /// `WriteAndLookup`.
+ Status Write(RealtimeWriteBatch&& write_batch) override;
+
+ /// Pins the exact committed, sealed and building key indexes used by one write. File I/O is
+ /// intentionally performed by the caller after this short critical section.
+ Result<std::shared_ptr<const KeyOffsetLookup>> AcquireKeyOffsetLookup();
+
+ /// Applies last-write-wins semantics. `previous_offsets` is the result from the pinned lookup;
+ /// duplicates inside this batch are resolved again in row order.
+ Status WriteAndLookup(const std::shared_ptr<arrow::StructArray>& data,
+ const std::vector<RecordBatch::RowKind>& row_kinds,
+ const OffsetRange& offset_range, const RoaringBitmap64& previous_offsets);
+
+ Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override;
+
+ Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
+ const std::shared_ptr<RealtimeSegmentHandle>& segment) override;
+
+ Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override;
+
+ Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
+ const std::shared_ptr<RealtimeReadView>& view,
+ const RealtimeQueryContext& context) override;
+
+ /// DEDUPLICATE advancement also has to atomically replace the committed `.offset` lookup.
+ /// Calling the base notification would create a query/write cutover with mismatched state.
+ Status AdvanceCommittedOffset(int64_t committed_end_offset) override;
+
+ Status AdvanceCommittedOffsetAndLookup(
+ int64_t committed_end_offset,
+ const std::shared_ptr<const KeyOffsetLookup>& committed_lookup);
+
+ Status AttachCommittedKeyOffsetLookup(
+ const std::shared_ptr<const KeyOffsetLookup>& committed_lookup);
+
+ RealtimeStoreDataUsage GetDataUsage() const override;
+ uint64_t GetMemoryUsage() const override;
+
+ static Result<std::shared_ptr<const RoaringBitmap64>> OffsetDeletionsOf(
+ const std::shared_ptr<RealtimeSegmentHandle>& segment);
+ static Result<std::shared_ptr<const RoaringBitmap64>> OffsetDeletionsOf(
+ const std::shared_ptr<RealtimeReadView>& view);
+
+ private:
+ struct SealedState {
+ OffsetRange offset_range;
+ std::shared_ptr<const KeyOffsetLookup> key_lookup;
+ std::shared_ptr<const RoaringBitmap64> offset_deletions;
+ };
+
+ class SegmentHandle;
+ class ReadView;
+ class OffsetFilteringBatchReader;
+
+ ArrowDeduplicateRealtimeStore(const std::shared_ptr<arrow::Schema>& write_schema,
+ int32_t key_field_position,
+ const std::shared_ptr<arrow::Field>& key_field,
+ std::shared_ptr<MutableKeyOffsetIndex> building_key_index,
+ const std::shared_ptr<RealtimeStore>& delegate,
+ const std::shared_ptr<MemoryPool>& memory_pool,
+ const std::shared_ptr<arrow::MemoryPool>& arrow_pool);
+
+ Result<std::shared_ptr<arrow::StructArray>> ExtractKeys(
+ const std::shared_ptr<arrow::StructArray>& data) const;
+ Status EnsureMutableDeletionBitmap();
+
+ std::shared_ptr<arrow::Schema> write_schema_;
+ int32_t key_field_position_;
+ std::shared_ptr<arrow::Field> key_field_;
+ std::shared_ptr<RealtimeStore> delegate_;
+ std::shared_ptr<MemoryPool> memory_pool_;
+ std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+
+ mutable std::mutex mutex_;
+ std::shared_ptr<MutableKeyOffsetIndex> building_key_index_;
+ std::shared_ptr<const KeyOffsetLookup> committed_key_lookup_;
+ std::vector<SealedState> sealed_states_;
+ std::shared_ptr<RoaringBitmap64> pending_offset_deletions_;
+ RoaringBitmap64 building_row_offsets_;
+ std::optional<OffsetRange> building_offset_range_;
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/realtime/arrow_deduplicate_realtime_store_test.cpp b/src/paimon/core/realtime/arrow_deduplicate_realtime_store_test.cpp
new file mode 100644
index 00000000..e671b695
--- /dev/null
+++ b/src/paimon/core/realtime/arrow_deduplicate_realtime_store_test.cpp
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/realtime/arrow_deduplicate_realtime_store.h"
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "arrow/ipc/json_simple.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/core/realtime/arrow_realtime_store.h"
+#include "paimon/macros.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+class EmptyKeyOffsetLookup final : public KeyOffsetLookup {
+ public:
+ Result<RoaringBitmap64> LookupOffsets(
+ const std::shared_ptr<arrow::StructArray>&) const override {
+ return RoaringBitmap64();
+ }
+};
+
+class ArrowDeduplicateRealtimeStoreTest : public testing::Test {
+ public:
+ void SetUp() override {
+ schema_ = arrow::schema({
+ DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
+ arrow::field("id", arrow::int64()),
+ arrow::field("value", arrow::utf8()),
+ });
+ pool_ = GetDefaultPool();
+ arrow_pool_ = GetArrowPool(pool_);
+ delegate_ = std::make_shared<ArrowRealtimeStore>(
+ schema_, RealtimeStoreMode::APPEND_ONLY, StatisticsMode::NONE,
+ /*temp_directory=*/"", /*spill_file_system=*/nullptr,
+ /*spill_compression=*/"zstd", /*spill_compression_level=*/1, pool_, arrow_pool_);
+ ASSERT_OK_AND_ASSIGN(store_, ArrowDeduplicateRealtimeStore::Create(
+ schema_, {"id"}, delegate_, pool_, arrow_pool_));
+ empty_lookup_ = std::make_shared<EmptyKeyOffsetLookup>();
+ ASSERT_OK(store_->AttachCommittedKeyOffsetLookup(empty_lookup_));
+ }
+
+ std::shared_ptr<arrow::StructArray> MakeBatch(const std::string& json) const {
+ return checked_pointer_cast<arrow::StructArray>(
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema_->fields()), json)
+ .ValueOrDie());
+ }
+
+ std::shared_ptr<arrow::StructArray> MakeKeys(const std::string& json) const {
+ return checked_pointer_cast<arrow::StructArray>(
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({schema_->field(1)}), json)
+ .ValueOrDie());
+ }
+
+ void WriteAndSeal(const std::string& json, const OffsetRange& offset_range,
+ const RoaringBitmap64& previous_offsets) {
+ ASSERT_OK(store_->WriteAndLookup(MakeBatch(json), {RecordBatch::RowKind::INSERT},
+ offset_range, previous_offsets));
+ ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+ store_->SealForCommit());
+ ASSERT_TRUE(segment.has_value());
+ }
+
+ Result<std::vector<int64_t>> ReadOffsets(const std::shared_ptr<RealtimeReadView>& view) const {
+ auto c_schema = std::make_unique<ArrowSchema>();
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema_, c_schema.get()));
+ RealtimeQueryContext context{c_schema.get(), /*predicate=*/nullptr};
+ PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+ store_->CreateQueryReaders(view, context));
+ std::vector<int64_t> result;
+ for (std::unique_ptr<BatchReader>& reader : readers) {
+ while (true) {
+ PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch());
+ if (BatchReader::IsEofBatch(batch)) {
+ break;
+ }
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::Array> imported,
+ arrow::ImportArray(batch.first.get(), batch.second.get()));
+ std::shared_ptr<arrow::StructArray> rows =
+ checked_pointer_cast<arrow::StructArray>(imported);
+ std::shared_ptr<arrow::Int64Array> offsets =
+ checked_pointer_cast<arrow::Int64Array>(
+ rows->GetFieldByName(SpecialFields::RealtimeOffset().Name()));
+ for (int64_t row = 0; row < offsets->length(); ++row) {
+ result.push_back(offsets->Value(row));
+ }
+ }
+ }
+ return result;
+ }
+
+ protected:
+ std::shared_ptr<arrow::Schema> schema_;
+ std::shared_ptr<MemoryPool> pool_;
+ std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+ std::shared_ptr<ArrowRealtimeStore> delegate_;
+ std::shared_ptr<ArrowDeduplicateRealtimeStore> store_;
+ std::shared_ptr<const KeyOffsetLookup> empty_lookup_;
+};
+
+TEST_F(ArrowDeduplicateRealtimeStoreTest, TestReclaimsOnlyCompleteSegmentPrefix) {
+ WriteAndSeal(R"([[2, 1, "old"]])", OffsetRange(0, 10), RoaringBitmap64());
+ RoaringBitmap64 overwritten_offsets;
+ overwritten_offsets.Add(2);
+ WriteAndSeal(R"([[12, 1, "new"]])", OffsetRange(10, 20), overwritten_offsets);
+
+ RealtimeStoreDataUsage usage = store_->GetDataUsage();
+ ASSERT_EQ(2, usage.sealed_row_count);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> initial_view, store_->AcquireReadView());
+ ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(0, 20)), initial_view->GetOffsetRange());
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<const RoaringBitmap64> initial_deletions,
+ ArrowDeduplicateRealtimeStore::OffsetDeletionsOf(initial_view));
+ ASSERT_EQ("{2}", initial_deletions->ToString());
+
+ ASSERT_OK(store_->AdvanceCommittedOffsetAndLookup(/*committed_end_offset=*/5, empty_lookup_));
+ usage = store_->GetDataUsage();
+ ASSERT_EQ(2, usage.sealed_row_count);
+
+ ASSERT_OK(store_->AdvanceCommittedOffsetAndLookup(/*committed_end_offset=*/10, empty_lookup_));
+ usage = store_->GetDataUsage();
+ ASSERT_EQ(1, usage.sealed_row_count);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> retained_view,
+ store_->AcquireReadView());
+ ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(10, 20)), retained_view->GetOffsetRange());
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<const RoaringBitmap64> retained_deletions,
+ ArrowDeduplicateRealtimeStore::OffsetDeletionsOf(retained_view));
+ ASSERT_EQ("{2}", retained_deletions->ToString());
+
+ ASSERT_OK(store_->AdvanceCommittedOffsetAndLookup(/*committed_end_offset=*/15, empty_lookup_));
+ usage = store_->GetDataUsage();
+ ASSERT_EQ(1, usage.sealed_row_count);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> crossing_view,
+ store_->AcquireReadView());
+ ASSERT_EQ(std::optional<OffsetRange>(OffsetRange(10, 20)), crossing_view->GetOffsetRange());
+
+ ASSERT_OK(store_->AdvanceCommittedOffsetAndLookup(/*committed_end_offset=*/20, empty_lookup_));
+ usage = store_->GetDataUsage();
+ ASSERT_EQ(0, usage.sealed_row_count);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> reclaimed_view,
+ store_->AcquireReadView());
+ ASSERT_FALSE(reclaimed_view->GetOffsetRange().has_value());
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<const RoaringBitmap64> reclaimed_deletions,
+ ArrowDeduplicateRealtimeStore::OffsetDeletionsOf(reclaimed_view));
+ ASSERT_TRUE(reclaimed_deletions->IsEmpty());
+}
+
+TEST_F(ArrowDeduplicateRealtimeStoreTest, TestLastWriteWinsAndReadViewIsSnapshot) {
+ ASSERT_OK(store_->WriteAndLookup(
+ MakeBatch(R"([[1, 1, "old"], [2, 2, "keep"], [3, 1, "new"]])"),
+ {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::INSERT, RecordBatch::RowKind::INSERT},
+ OffsetRange(1, 4), RoaringBitmap64()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> pinned_view, store_->AcquireReadView());
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<const RoaringBitmap64> pinned_deletions,
+ ArrowDeduplicateRealtimeStore::OffsetDeletionsOf(pinned_view));
+ ASSERT_EQ("{1}", pinned_deletions->ToString());
+
+ ASSERT_OK(store_->WriteAndLookup(MakeBatch(R"([[4, 2, "latest"]])"),
+ {RecordBatch::RowKind::UPDATE_AFTER}, OffsetRange(4, 5),
+ RoaringBitmap64()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeReadView> current_view, store_->AcquireReadView());
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<const RoaringBitmap64> current_deletions,
+ ArrowDeduplicateRealtimeStore::OffsetDeletionsOf(current_view));
+ ASSERT_EQ("{1,2}", current_deletions->ToString());
+ ASSERT_EQ("{1}", pinned_deletions->ToString());
+
+ ASSERT_OK_AND_ASSIGN(std::vector<int64_t> pinned_offsets, ReadOffsets(pinned_view));
+ ASSERT_EQ((std::vector<int64_t>{2, 3}), pinned_offsets);
+ ASSERT_OK_AND_ASSIGN(std::vector<int64_t> current_offsets, ReadOffsets(current_view));
+ ASSERT_EQ((std::vector<int64_t>{3, 4}), current_offsets);
+
+ ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+ store_->SealForCommit());
+ ASSERT_TRUE(segment.has_value());
+ ASSERT_EQ(2, segment.value()->GetRowCount());
+ ASSERT_EQ(OffsetRange(1, 5), segment.value()->GetOffsetRange());
+}
+
+TEST_F(ArrowDeduplicateRealtimeStoreTest, TestDeleteProducesNoLiveRows) {
+ ASSERT_OK(store_->WriteAndLookup(MakeBatch(R"([[1, 1, "value"]])"),
+ {RecordBatch::RowKind::INSERT}, OffsetRange(1, 2),
+ RoaringBitmap64()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<const KeyOffsetLookup> lookup,
+ store_->AcquireKeyOffsetLookup());
+ ASSERT_OK_AND_ASSIGN(RoaringBitmap64 previous, lookup->LookupOffsets(MakeKeys(R"([[1]])")));
+ ASSERT_EQ("{1}", previous.ToString());
+ ASSERT_OK(store_->WriteAndLookup(MakeBatch(R"([[2, 1, "ignored"]])"),
+ {RecordBatch::RowKind::DELETE}, OffsetRange(2, 3), previous));
+
+ ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+ store_->SealForCommit());
+ ASSERT_TRUE(segment.has_value());
+ ASSERT_EQ(0, segment.value()->GetRowCount());
+ ASSERT_OK_AND_ASSIGN(std::vector<std::unique_ptr<BatchReader>> readers,
+ store_->CreateCommitReaders(segment.value()));
+ ASSERT_EQ(1, readers.size());
+ ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, readers[0]->NextBatch());
+ ASSERT_TRUE(BatchReader::IsEofBatch(batch));
+}
+
+TEST_F(ArrowDeduplicateRealtimeStoreTest, TestValidationAndSpecializedApis) {
+ ASSERT_NOK_WITH_MSG(
+ ArrowDeduplicateRealtimeStore::Create(schema_, {}, delegate_, pool_, arrow_pool_),
+ "requires exactly one deduplicate key field");
+ ASSERT_NOK_WITH_MSG(
+ ArrowDeduplicateRealtimeStore::Create(arrow::schema({arrow::field("id", arrow::int64())}),
+ {"id"}, delegate_, pool_, arrow_pool_),
+ "requires non-null int64 _REALTIME_OFFSET");
+ ASSERT_NOK_WITH_MSG(store_->Write(RealtimeWriteBatch{nullptr, OffsetRange(0, 1)}),
+ "Write requires key-offset lookup");
+ ASSERT_NOK_WITH_MSG(store_->AdvanceCommittedOffset(1),
+ "requires an atomic key-offset lookup replacement");
+ ASSERT_NOK_WITH_MSG(store_->AdvanceCommittedOffsetAndLookup(-1, empty_lookup_),
+ "must not be negative");
+ ASSERT_NOK_WITH_MSG(store_->AdvanceCommittedOffsetAndLookup(1, nullptr), "must not be null");
+ ASSERT_NOK_WITH_MSG(
+ store_->AttachCommittedKeyOffsetLookup(std::make_shared<EmptyKeyOffsetLookup>()),
+ "already attached");
+}
+
+} // namespace paimon::test
diff --git a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
index 4777ee35..07c1ba24 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_factory.cpp
@@ -28,6 +28,7 @@
#include "paimon/common/utils/scope_guard.h"
#include "paimon/core/core_options.h"
#include "paimon/core/realtime/arrow_realtime_store.h"
+#include "paimon/core/realtime/realtime_utils.h"
#include "paimon/macros.h"
namespace paimon {
@@ -49,6 +50,7 @@ Result<std::shared_ptr<RealtimeStore>> ArrowRealtimeStoreFactory::Create(
return Status::Invalid("invalid real-time store mode: ",
static_cast<int32_t>(request.mode));
}
+ PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateOffsetField(imported_schema));
std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(request.memory_pool);
PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(request.options));
const CompressOptions& spill_compression = options.GetSpillCompressOptions();
diff --git a/src/paimon/core/realtime/arrow_realtime_store_test.cpp b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
index c8d988e3..60ea9f98 100644
--- a/src/paimon/core/realtime/arrow_realtime_store_test.cpp
+++ b/src/paimon/core/realtime/arrow_realtime_store_test.cpp
@@ -131,6 +131,8 @@ class ArrowRealtimeStoreTest : public testing::Test {
std::shared_ptr<ArrowRealtimeStore> store_;
};
+} // namespace
+
TEST_F(ArrowRealtimeStoreTest, TestWriteValidationAndSeal) {
ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<RealtimeSegmentHandle>> empty_segment,
store_->SealForCommit());
@@ -239,7 +241,8 @@ TEST_F(ArrowRealtimeStoreTest, TestQueryReaderClipsCommittedOffsetWithBitmap) {
store_->CreateQueryReaders(view, context));
ASSERT_EQ(1, readers.size());
readers[0] =
- std::make_unique<RealtimeOffsetBatchReader>(std::move(readers[0]), OffsetRange(12, 15));
+ std::make_unique<RealtimeOffsetBatchReader>(std::move(readers[0]), OffsetRange(12, 15),
+ /*keep_offset=*/false);
ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap first,
readers[0]->NextBatchWithBitmap());
@@ -271,7 +274,8 @@ TEST_F(ArrowRealtimeStoreTest, TestQueryReaderClipsCommittedOffsetWithBitmap) {
store_->CreateQueryReaders(view, context));
ASSERT_EQ(1, readers.size());
readers[0] =
- std::make_unique<RealtimeOffsetBatchReader>(std::move(readers[0]), OffsetRange(15, 15));
+ std::make_unique<RealtimeOffsetBatchReader>(std::move(readers[0]), OffsetRange(15, 15),
+ /*keep_offset=*/false);
ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap eof, readers[0]->NextBatchWithBitmap());
ASSERT_TRUE(BatchReader::IsEofBatch(eof));
}
@@ -401,6 +405,14 @@ TEST_F(ArrowRealtimeStoreTest, TestFactoryRejectsSpillWithoutTempDirectory) {
"realtime.spill-enabled requires a non-empty temporary directory");
}
+TEST_F(ArrowRealtimeStoreTest, TestFactoryRejectsFrameworkOwnedMode) {
+ ArrowRealtimeStoreFactory factory;
+ std::unique_ptr<ArrowSchema> write_schema = MakeReadSchema(schema_);
+ RealtimeStoreCreateRequest request{std::move(write_schema), /*options=*/{}, pool_,
+ RealtimeStoreMode::DEDUPLICATE, StatisticsMode::NONE};
+ ASSERT_NOK_WITH_MSG(factory.Create(std::move(request)), "invalid real-time store mode");
+}
+
TEST_F(ArrowRealtimeStoreTest, TestFullStatisticsPrunesNonMatchingBatch) {
ArrowRealtimeStoreFactory factory;
std::unique_ptr<ArrowSchema> write_schema = MakeReadSchema(schema_);
@@ -426,7 +438,8 @@ TEST_F(ArrowRealtimeStoreTest, TestFullStatisticsPrunesNonMatchingBatch) {
store->CreateQueryReaders(view, context));
ASSERT_EQ(1, readers.size());
readers[0] =
- std::make_unique<RealtimeOffsetBatchReader>(std::move(readers[0]), OffsetRange(3, 4));
+ std::make_unique<RealtimeOffsetBatchReader>(std::move(readers[0]), OffsetRange(3, 4),
+ /*keep_offset=*/false);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> filtered,
ReadResultCollector::CollectResult(std::move(readers[0])));
@@ -478,5 +491,4 @@ TEST_F(ArrowRealtimeStoreTest, TestRejectsHandlesFromAnotherStoreImplementation)
ASSERT_NOK_WITH_MSG(store_->CreateQueryReaders(view, context), "mem query read schema is null");
}
-} // namespace
} // namespace paimon::test
diff --git a/src/paimon/core/realtime/realtime_context_impl.cpp b/src/paimon/core/realtime/realtime_context_impl.cpp
index 6b476617..fc09f304 100644
--- a/src/paimon/core/realtime/realtime_context_impl.cpp
+++ b/src/paimon/core/realtime/realtime_context_impl.cpp
@@ -40,9 +40,12 @@
#include "fmt/format.h"
#include "paimon/arrow/abi.h"
#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
#include "paimon/common/utils/arrow/status_utils.h"
#include "paimon/common/utils/scope_guard.h"
#include "paimon/common/utils/uuid.h"
+#include "paimon/core/realtime/arrow_deduplicate_realtime_store.h"
+#include "paimon/core/realtime/realtime_deduplicate_state.h"
#include "paimon/macros.h"
#include "paimon/realtime/realtime_store.h"
#include "paimon/status.h"
@@ -172,6 +175,7 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
}
if (iter != stores_.end()) {
if (iter->second.mode != request.mode ||
+ iter->second.deduplicate_key_fields != request.deduplicate_key_fields ||
!iter->second.write_schema->Equals(*requested_schema, /*check_metadata=*/true)) {
return Status::Invalid(fmt::format(
"real-time store schema or mode mismatch for partition {}, bucket {}; recreate "
@@ -200,7 +204,8 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
initial_offset = memory_range->end;
}
}
- return RealtimeStoreState{iter->second.store, initial_offset};
+ return RealtimeStoreState{iter->second.store, initial_offset,
+ iter->second.deduplicate_state};
}
if (!request.memory_pool) {
return Status::Invalid("real-time store memory pool is null");
@@ -209,17 +214,35 @@ Result<RealtimeStoreState> RealtimeContextImpl::GetOrCreateRealtimeStore(
arrow::ExportSchema(*requested_schema, request.write_schema.get()));
RealtimeStoreMode mode = request.mode;
std::string temp_directory = request.temp_directory;
- PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> store,
+ std::vector<std::string> deduplicate_key_fields = request.deduplicate_key_fields;
+ std::shared_ptr<MemoryPool> memory_pool = request.memory_pool;
+ std::shared_ptr<RealtimeDeduplicateState> deduplicate_state;
+ if (mode == RealtimeStoreMode::DEDUPLICATE) {
+ deduplicate_state = std::make_shared<RealtimeDeduplicateState>();
+ // DEDUPLICATE semantics belong to the framework wrapper. The pluggable factory only
+ // supplies the append storage delegate and cannot bypass key-index or offset-DV handling.
+ request.mode = RealtimeStoreMode::APPEND_ONLY;
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
factory_->Create(std::move(request)));
- if (!store) {
+ if (!delegate) {
return Status::Invalid("real-time store factory returned a null store");
}
+ std::shared_ptr<RealtimeStore> store = delegate;
+ if (mode == RealtimeStoreMode::DEDUPLICATE) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ArrowDeduplicateRealtimeStore> deduplicate_store,
+ ArrowDeduplicateRealtimeStore::Create(
+ requested_schema, deduplicate_key_fields, delegate, memory_pool,
+ GetArrowPool(memory_pool)));
+ store = std::move(deduplicate_store);
+ }
stores_.emplace(partition_bucket,
- StoreEntry{store, requested_schema, mode, std::move(temp_directory)});
+ StoreEntry{store, requested_schema, mode, std::move(temp_directory),
+ std::move(deduplicate_key_fields), deduplicate_state});
if (offset_iter != committed_offsets_.end()) {
reclaimed_offsets_.emplace(partition_bucket, offset_iter->second);
}
- return RealtimeStoreState{std::move(store), initial_offset};
+ return RealtimeStoreState{std::move(store), initial_offset, std::move(deduplicate_state)};
}
Result<int64_t> RealtimeContextImpl::AdvanceMaterializedMaxSequenceNumber(
@@ -249,11 +272,17 @@ Result<RealtimeReadState> RealtimeContextImpl::AcquireReadState() {
for (const auto& [partition_bucket, store] : stores_) {
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> read_view,
store.store->AcquireReadView());
+ std::shared_ptr<const RoaringBitmap64> offset_deletions;
+ if (store.deduplicate_state) {
+ PAIMON_ASSIGN_OR_RAISE(offset_deletions,
+ ArrowDeduplicateRealtimeStore::OffsetDeletionsOf(read_view));
+ }
if (!read_view) {
return Status::Invalid("real-time store returned a null read view");
}
- result.views.push_back(
- RealtimePartitionBucketView{partition_bucket, store.store, std::move(read_view)});
+ result.views.push_back(RealtimePartitionBucketView{partition_bucket, store.store,
+ std::move(read_view), store.mode,
+ std::move(offset_deletions)});
}
return result;
}
@@ -333,8 +362,9 @@ Status RealtimeContextImpl::ReleaseReadView(const std::string& opaque_ticket) {
return Status::OK();
}
-Status RealtimeContextImpl::AdvanceCommittedProgress(int64_t snapshot_id,
- const RealtimeOffsetMap& committed_offsets) {
+Status RealtimeContextImpl::AdvanceCommittedProgress(
+ int64_t snapshot_id, const RealtimeOffsetMap& committed_offsets,
+ const RealtimeDataFileMap& committed_data_files) {
PAIMON_RETURN_NOT_OK(CheckUsable());
if (snapshot_id < 0) {
return Status::Invalid("real-time refresh snapshot id must not be negative");
@@ -373,32 +403,65 @@ Status RealtimeContextImpl::AdvanceCommittedProgress(int64_t snapshot_id,
}
}
committed_offsets_ = committed_offsets;
+ committed_data_files_ = committed_data_files;
last_refreshed_snapshot_id_ = snapshot_id;
}
- std::vector<std::tuple<RealtimePartitionBucket, std::shared_ptr<RealtimeStore>, int64_t>>
- notifications;
+ struct Notification {
+ RealtimePartitionBucket partition_bucket;
+ std::shared_ptr<RealtimeStore> store;
+ std::shared_ptr<RealtimeDeduplicateState> deduplicate_state;
+ int64_t committed_end_offset;
+ std::vector<std::shared_ptr<DataFileMeta>> active_data_files;
+ };
+ std::vector<Notification> notifications;
{
std::lock_guard<std::mutex> registry_lock(mutex_);
for (const auto& [partition_bucket, committed_end_offset] : committed_offsets_) {
- auto reclaimed_iter = reclaimed_offsets_.find(partition_bucket);
- if (reclaimed_iter != reclaimed_offsets_.end() &&
- reclaimed_iter->second >= committed_end_offset) {
+ auto store_iter = stores_.find(partition_bucket);
+ if (store_iter == stores_.end()) {
continue;
}
- auto store_iter = stores_.find(partition_bucket);
- if (store_iter != stores_.end()) {
- notifications.emplace_back(partition_bucket, store_iter->second.store,
- committed_end_offset);
+ const std::shared_ptr<RealtimeDeduplicateState>& deduplicate_state =
+ store_iter->second.deduplicate_state;
+ if (deduplicate_state) {
+ auto installed_iter = installed_snapshot_ids_.find(partition_bucket);
+ if (installed_iter != installed_snapshot_ids_.end() &&
+ installed_iter->second >= last_refreshed_snapshot_id_.value()) {
+ continue;
+ }
+ } else {
+ auto reclaimed_iter = reclaimed_offsets_.find(partition_bucket);
+ if (reclaimed_iter != reclaimed_offsets_.end() &&
+ reclaimed_iter->second >= committed_end_offset) {
+ continue;
+ }
}
+ auto files_iter = committed_data_files_.find(partition_bucket);
+ std::vector<std::shared_ptr<DataFileMeta>> active_data_files =
+ files_iter == committed_data_files_.end()
+ ? std::vector<std::shared_ptr<DataFileMeta>>()
+ : files_iter->second;
+ notifications.push_back(Notification{partition_bucket, store_iter->second.store,
+ deduplicate_state, committed_end_offset,
+ std::move(active_data_files)});
}
}
// Reclaim independent stores on a best-effort basis, then report the first error.
Status first_error = Status::OK();
- for (const auto& [partition_bucket, store, committed_end_offset] : notifications) {
- Status status = store->AdvanceCommittedOffset(committed_end_offset);
+ for (const Notification& notification : notifications) {
+ Status status =
+ notification.deduplicate_state
+ ? notification.deduplicate_state->InstallCommittedSnapshot(
+ notification.committed_end_offset, notification.active_data_files,
+ notification.store)
+ : notification.store->AdvanceCommittedOffset(notification.committed_end_offset);
if (status.ok()) {
- reclaimed_offsets_[partition_bucket] = committed_end_offset;
+ reclaimed_offsets_[notification.partition_bucket] = notification.committed_end_offset;
+ if (notification.deduplicate_state) {
+ installed_snapshot_ids_[notification.partition_bucket] =
+ last_refreshed_snapshot_id_.value();
+ }
} else if (first_error.ok()) {
first_error = std::move(status);
}
diff --git a/src/paimon/core/realtime/realtime_context_impl.h b/src/paimon/core/realtime/realtime_context_impl.h
index 2286925f..7204b383 100644
--- a/src/paimon/core/realtime/realtime_context_impl.h
+++ b/src/paimon/core/realtime/realtime_context_impl.h
@@ -47,17 +47,26 @@ namespace paimon {
class RealtimeStore;
class RealtimeReadView;
+class RealtimeDeduplicateState;
+struct DataFileMeta;
class MemoryPool;
+class RoaringBitmap64;
+
+using RealtimeDataFileMap =
+ std::map<RealtimePartitionBucket, std::vector<std::shared_ptr<DataFileMeta>>>;
struct RealtimeStoreState {
std::shared_ptr<RealtimeStore> store;
int64_t initial_offset;
+ std::shared_ptr<RealtimeDeduplicateState> deduplicate_state;
};
struct RealtimePartitionBucketView {
RealtimePartitionBucket partition_bucket;
std::shared_ptr<RealtimeStore> store;
std::shared_ptr<RealtimeReadView> read_view;
+ RealtimeStoreMode mode = RealtimeStoreMode::APPEND_ONLY;
+ std::shared_ptr<const RoaringBitmap64> offset_deletions;
};
struct RealtimeReadState {
@@ -99,8 +108,8 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
// Returns an error requiring a new context if a newer snapshot removes or moves committed
// progress backwards for a store created by this context. Progress for inactive stores is
// only reference state and can be replaced in place.
- Status AdvanceCommittedProgress(int64_t snapshot_id,
- const RealtimeOffsetMap& committed_offsets);
+ Status AdvanceCommittedProgress(int64_t snapshot_id, const RealtimeOffsetMap& committed_offsets,
+ const RealtimeDataFileMap& committed_data_files);
private:
static constexpr std::chrono::milliseconds kReadViewReleaseCheckInterval{100};
@@ -115,6 +124,8 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
std::shared_ptr<arrow::Schema> write_schema;
RealtimeStoreMode mode;
std::string temp_directory;
+ std::vector<std::string> deduplicate_key_fields;
+ std::shared_ptr<RealtimeDeduplicateState> deduplicate_state;
int64_t materialized_max_sequence_number = -1;
};
@@ -130,8 +141,12 @@ class PAIMON_EXPORT RealtimeContextImpl final : public RealtimeContext {
std::map<RealtimePartitionBucket, StoreEntry> stores_;
// Full-table progress used as the initial offset when a store is created lazily.
RealtimeOffsetMap committed_offsets_;
+ // Active files from the same Snapshot as `committed_offsets_`.
+ RealtimeDataFileMap committed_data_files_;
// Progress already reflected in stores owned by this context.
RealtimeOffsetMap reclaimed_offsets_;
+ // DEDUPLICATE lookups have to follow Snapshot changes even when the offset does not advance.
+ std::map<RealtimePartitionBucket, int64_t> installed_snapshot_ids_;
std::optional<int64_t> last_refreshed_snapshot_id_;
std::mutex read_views_mutex_;
std::condition_variable read_views_cv_;
diff --git a/src/paimon/core/realtime/realtime_context_test.cpp b/src/paimon/core/realtime/realtime_context_test.cpp
index c691fd51..155bb586 100644
--- a/src/paimon/core/realtime/realtime_context_test.cpp
+++ b/src/paimon/core/realtime/realtime_context_test.cpp
@@ -28,8 +28,16 @@
#include "arrow/api.h"
#include "arrow/c/bridge.h"
#include "arrow/c/helpers.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/io/data_file_path_factory.h"
+#include "paimon/core/realtime/arrow_deduplicate_realtime_store.h"
#include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_deduplicate_state.h"
+#include "paimon/core/realtime/realtime_offset_file_index_lookup.h"
+#include "paimon/fs/local/local_file_system.h"
#include "paimon/memory/memory_pool.h"
+#include "paimon/realtime/arrow_realtime_store_factory.h"
#include "paimon/testing/utils/testharness.h"
namespace paimon::test {
@@ -95,6 +103,8 @@ class TestingRealtimeStoreFactory : public RealtimeStoreFactory {
if (!request.write_schema || !request.write_schema->release) {
return Status::Invalid("testing write schema is null");
}
+ requested_modes.push_back(request.mode);
+ requested_deduplicate_key_fields.push_back(request.deduplicate_key_fields);
ArrowSchemaRelease(request.write_schema.get());
if (return_null_store) {
return std::shared_ptr<RealtimeStore>();
@@ -106,6 +116,8 @@ class TestingRealtimeStoreFactory : public RealtimeStoreFactory {
bool return_null_store = false;
std::vector<std::shared_ptr<TestingRealtimeStore>> stores;
+ std::vector<RealtimeStoreMode> requested_modes;
+ std::vector<std::vector<std::string>> requested_deduplicate_key_fields;
};
std::unique_ptr<ArrowSchema> MakeWriteSchema(
@@ -118,6 +130,20 @@ std::unique_ptr<ArrowSchema> MakeWriteSchema(
return schema;
}
+std::shared_ptr<arrow::Schema> MakeDeduplicateSchema() {
+ return arrow::schema({
+ DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
+ arrow::field("id", arrow::int64()),
+ arrow::field("value", arrow::utf8()),
+ });
+}
+
+std::unique_ptr<ArrowSchema> ExportSchema(const std::shared_ptr<arrow::Schema>& schema) {
+ auto exported = std::make_unique<ArrowSchema>();
+ EXPECT_TRUE(arrow::ExportSchema(*schema, exported.get()).ok());
+ return exported;
+}
+
Result<std::shared_ptr<RealtimeContextImpl>> CreateContext(
const std::shared_ptr<RealtimeStoreFactory>& factory) {
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContext> context,
@@ -138,6 +164,8 @@ Result<RealtimeStoreState> GetOrCreateAppendStore(
RealtimePartitionBucket(partition, bucket));
}
+} // namespace
+
TEST(RealtimeContextTest, TestReusesStoreAndCapturesRegisteredViews) {
auto factory = std::make_shared<TestingRealtimeStoreFactory>();
ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
@@ -301,17 +329,20 @@ TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, GetDefaultPool()));
ASSERT_EQ(2, factory->stores.size());
- ASSERT_NOK_WITH_MSG(context->AdvanceCommittedProgress(-1, {}),
+ ASSERT_NOK_WITH_MSG(context->AdvanceCommittedProgress(-1, {}, /*committed_data_files=*/{}),
"snapshot id must not be negative");
ASSERT_NOK_WITH_MSG(context->AdvanceCommittedProgress(
- 4, {{RealtimePartitionBucket(partition, /*bucket=*/-1), /*offset=*/3}}),
+ 4, {{RealtimePartitionBucket(partition, /*bucket=*/-1), /*offset=*/3}},
+ /*committed_data_files=*/{}),
"invalid partition-bucket committed offset");
ASSERT_TRUE(factory->stores[0]->committed_offsets.empty());
ASSERT_TRUE(factory->stores[1]->committed_offsets.empty());
ASSERT_OK(context->AdvanceCommittedProgress(
- 5, {{RealtimePartitionBucket(partition, /*bucket=*/0), /*offset=*/7},
- {RealtimePartitionBucket({{"dt", "unknown"}}, /*bucket=*/0), /*offset=*/9}}));
+ 5,
+ {{RealtimePartitionBucket(partition, /*bucket=*/0), /*offset=*/7},
+ {RealtimePartitionBucket({{"dt", "unknown"}}, /*bucket=*/0), /*offset=*/9}},
+ /*committed_data_files=*/{}));
ASSERT_EQ(std::vector<int64_t>({7}), factory->stores[0]->committed_offsets);
ASSERT_TRUE(factory->stores[1]->committed_offsets.empty());
@@ -321,19 +352,89 @@ TEST(RealtimeContextTest, TestCommittedProgressIsMonotonicAndSelective) {
ASSERT_EQ(9, restored_state.initial_offset);
ASSERT_OK(context->AdvanceCommittedProgress(
- 5, {{RealtimePartitionBucket(partition, /*bucket=*/0), /*offset=*/10}}));
+ 5, {{RealtimePartitionBucket(partition, /*bucket=*/0), /*offset=*/10}},
+ /*committed_data_files=*/{}));
ASSERT_EQ(std::vector<int64_t>({7}), factory->stores[0]->committed_offsets);
- ASSERT_NOK_WITH_MSG(context->AdvanceCommittedProgress(4, {}),
+ ASSERT_NOK_WITH_MSG(context->AdvanceCommittedProgress(4, {}, /*committed_data_files=*/{}),
"committed snapshot cannot move backwards");
ASSERT_OK(context->AdvanceCommittedProgress(
- 6, {{RealtimePartitionBucket(partition, /*bucket=*/0), /*offset=*/7},
- {RealtimePartitionBucket(partition, /*bucket=*/1), /*offset=*/8},
- {RealtimePartitionBucket({{"dt", "unknown"}}, /*bucket=*/0), /*offset=*/9}}));
+ 6,
+ {{RealtimePartitionBucket(partition, /*bucket=*/0), /*offset=*/7},
+ {RealtimePartitionBucket(partition, /*bucket=*/1), /*offset=*/8},
+ {RealtimePartitionBucket({{"dt", "unknown"}}, /*bucket=*/0), /*offset=*/9}},
+ /*committed_data_files=*/{}));
ASSERT_EQ(std::vector<int64_t>({7}), factory->stores[0]->committed_offsets);
ASSERT_EQ(std::vector<int64_t>({8}), factory->stores[1]->committed_offsets);
}
+TEST(RealtimeContextTest, TestDeduplicateStoreWrapsAppendDelegateFromFactory) {
+ auto factory = std::make_shared<TestingRealtimeStoreFactory>();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+ RealtimeStoreCreateRequest request{ExportSchema(MakeDeduplicateSchema()), /*options=*/{},
+ GetDefaultPool(), RealtimeStoreMode::DEDUPLICATE};
+ request.deduplicate_key_fields = {"id"};
+
+ ASSERT_OK_AND_ASSIGN(
+ RealtimeStoreState state,
+ context->GetOrCreateRealtimeStore(std::move(request),
+ RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
+ ASSERT_NE(nullptr, std::dynamic_pointer_cast<ArrowDeduplicateRealtimeStore>(state.store));
+ ASSERT_NE(nullptr, state.deduplicate_state);
+ ASSERT_EQ(1, factory->stores.size());
+ ASSERT_EQ((std::vector<RealtimeStoreMode>{RealtimeStoreMode::APPEND_ONLY}),
+ factory->requested_modes);
+ ASSERT_EQ((std::vector<std::vector<std::string>>{{"id"}}),
+ factory->requested_deduplicate_key_fields);
+
+ ASSERT_OK_AND_ASSIGN(RealtimeReadState read_state, context->AcquireReadState());
+ ASSERT_EQ(1, read_state.views.size());
+ ASSERT_EQ(RealtimeStoreMode::DEDUPLICATE, read_state.views[0].mode);
+ ASSERT_NE(nullptr, read_state.views[0].offset_deletions);
+}
+
+TEST(RealtimeContextTest, TestNewDeduplicateSnapshotRefreshesLookupAtSameOffset) {
+ auto factory = std::make_shared<ArrowRealtimeStoreFactory>();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
+ std::shared_ptr<arrow::Schema> schema = MakeDeduplicateSchema();
+ const std::map<std::string, std::string> options = {
+ {"file-index.bitmap.columns", SpecialFields::RealtimeOffset().Name()}};
+ RealtimeStoreCreateRequest request{ExportSchema(schema), options, GetDefaultPool(),
+ RealtimeStoreMode::DEDUPLICATE};
+ request.deduplicate_key_fields = {"id"};
+ const RealtimePartitionBucket partition_bucket(/*partition=*/{}, /*bucket=*/0);
+ ASSERT_OK_AND_ASSIGN(RealtimeStoreState store_state,
+ context->GetOrCreateRealtimeStore(std::move(request), partition_bucket));
+ ASSERT_TRUE(store_state.deduplicate_state);
+
+ ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options));
+ auto path_factory = std::make_shared<DataFilePathFactory>();
+ auto file_system = std::make_shared<LocalFileSystem>();
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<RealtimeOffsetFileIndexLookup> initial_lookup,
+ RealtimeOffsetFileIndexLookup::Create(
+ schema, /*data_schema_id=*/0, schema->GetFieldByName("id"),
+ /*data_files=*/{}, path_factory, file_system, GetDefaultPool(), core_options));
+ ASSERT_OK(
+ store_state.deduplicate_state->AttachFileIndexLookup(initial_lookup, store_state.store));
+
+ const RealtimeOffsetMap committed_offsets = {{partition_bucket, /*offset=*/10}};
+ ASSERT_OK(context->AdvanceCommittedProgress(/*snapshot_id=*/5, committed_offsets,
+ /*committed_data_files=*/{}));
+ const std::shared_ptr<RealtimeOffsetFileIndexLookup> snapshot_5_lookup =
+ store_state.deduplicate_state->committed_file_lookup_;
+ ASSERT_NE(nullptr, snapshot_5_lookup);
+ ASSERT_NE(initial_lookup, snapshot_5_lookup);
+
+ ASSERT_OK(context->AdvanceCommittedProgress(/*snapshot_id=*/5, committed_offsets,
+ /*committed_data_files=*/{}));
+ ASSERT_EQ(snapshot_5_lookup, store_state.deduplicate_state->committed_file_lookup_);
+
+ ASSERT_OK(context->AdvanceCommittedProgress(/*snapshot_id=*/6, committed_offsets,
+ /*committed_data_files=*/{}));
+ ASSERT_NE(snapshot_5_lookup, store_state.deduplicate_state->committed_file_lookup_);
+}
+
TEST(RealtimeContextTest, TestRemovedInactivePartitionDoesNotRequireReopen) {
auto factory = std::make_shared<TestingRealtimeStoreFactory>();
ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> context, CreateContext(factory));
@@ -343,13 +444,15 @@ TEST(RealtimeContextTest, TestRemovedInactivePartitionDoesNotRequireReopen) {
const RealtimePartitionBucket inactive_partition_bucket(inactive_partition, /*bucket=*/0);
ASSERT_OK(context->AdvanceCommittedProgress(
- 5, {{active_partition_bucket, /*offset=*/7}, {inactive_partition_bucket, /*offset=*/9}}));
+ 5, {{active_partition_bucket, /*offset=*/7}, {inactive_partition_bucket, /*offset=*/9}},
+ /*committed_data_files=*/{}));
ASSERT_OK_AND_ASSIGN(RealtimeStoreState active_state,
GetOrCreateAppendStore(context, active_partition, 0, MakeWriteSchema(), {},
GetDefaultPool()));
ASSERT_EQ(7, active_state.initial_offset);
- ASSERT_OK(context->AdvanceCommittedProgress(6, {{active_partition_bucket, /*offset=*/7}}));
+ ASSERT_OK(context->AdvanceCommittedProgress(6, {{active_partition_bucket, /*offset=*/7}},
+ /*committed_data_files=*/{}));
ASSERT_OK_AND_ASSIGN(RealtimeStoreState inactive_state,
GetOrCreateAppendStore(context, inactive_partition, 0, MakeWriteSchema(),
{}, GetDefaultPool()));
@@ -374,8 +477,9 @@ TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
{RealtimePartitionBucket(partition, /*bucket=*/0), /*offset=*/7},
{RealtimePartitionBucket(partition, /*bucket=*/1), /*offset=*/8},
{RealtimePartitionBucket(partition, /*bucket=*/2), /*offset=*/9}};
- ASSERT_NOK_WITH_MSG(context->AdvanceCommittedProgress(5, committed_offsets),
- "injected committed offset failure");
+ ASSERT_NOK_WITH_MSG(
+ context->AdvanceCommittedProgress(5, committed_offsets, /*committed_data_files=*/{}),
+ "injected committed offset failure");
ASSERT_EQ(std::vector<int64_t>({7}), factory->stores[0]->committed_offsets);
ASSERT_TRUE(factory->stores[1]->committed_offsets.empty());
ASSERT_EQ(std::vector<int64_t>({9}), factory->stores[2]->committed_offsets);
@@ -385,7 +489,7 @@ TEST(RealtimeContextTest, TestRetriesOnlyIncompleteReclamation) {
GetOrCreateAppendStore(context, partition, 1, MakeWriteSchema(), {}, GetDefaultPool()));
ASSERT_EQ(8, failed_store_state.initial_offset);
- ASSERT_OK(context->AdvanceCommittedProgress(5, committed_offsets));
+ ASSERT_OK(context->AdvanceCommittedProgress(5, committed_offsets, /*committed_data_files=*/{}));
ASSERT_EQ(1, factory->stores[0]->advance_count);
ASSERT_EQ(2, factory->stores[1]->advance_count);
ASSERT_EQ(1, factory->stores[2]->advance_count);
@@ -405,22 +509,26 @@ TEST(RealtimeContextTest, TestRequiresReopenWhenCommittedProgressMovesBackwards)
ASSERT_OK(GetOrCreateAppendStore(context, second_partition, 0, MakeWriteSchema(), {},
GetDefaultPool()));
ASSERT_OK(context->AdvanceCommittedProgress(
- 5, {{first_partition_bucket, /*offset=*/7}, {second_partition_bucket, /*offset=*/9}}));
+ 5, {{first_partition_bucket, /*offset=*/7}, {second_partition_bucket, /*offset=*/9}},
+ /*committed_data_files=*/{}));
ASSERT_EQ(std::vector<int64_t>({7}), factory->stores[0]->committed_offsets);
ASSERT_EQ(std::vector<int64_t>({9}), factory->stores[1]->committed_offsets);
ASSERT_NOK_WITH_MSG(
context->AdvanceCommittedProgress(
- 6, {{first_partition_bucket, /*offset=*/6}, {second_partition_bucket, /*offset=*/10}}),
+ 6, {{first_partition_bucket, /*offset=*/6}, {second_partition_bucket, /*offset=*/10}},
+ /*committed_data_files=*/{}),
"recreate RealtimeContext");
ASSERT_NOK_WITH_MSG(
- context->AdvanceCommittedProgress(6, {{first_partition_bucket, /*offset=*/10}}),
+ context->AdvanceCommittedProgress(6, {{first_partition_bucket, /*offset=*/10}},
+ /*committed_data_files=*/{}),
"recreate RealtimeContext");
ASSERT_EQ(std::vector<int64_t>({7}), factory->stores[0]->committed_offsets);
ASSERT_EQ(std::vector<int64_t>({9}), factory->stores[1]->committed_offsets);
ASSERT_OK(context->AdvanceCommittedProgress(
- 6, {{first_partition_bucket, /*offset=*/10}, {second_partition_bucket, /*offset=*/11}}));
+ 6, {{first_partition_bucket, /*offset=*/10}, {second_partition_bucket, /*offset=*/11}},
+ /*committed_data_files=*/{}));
ASSERT_EQ(std::vector<int64_t>({7, 10}), factory->stores[0]->committed_offsets);
ASSERT_EQ(std::vector<int64_t>({9, 11}), factory->stores[1]->committed_offsets);
}
@@ -489,5 +597,4 @@ TEST(RealtimeContextTest, TestRejectsNullPluginResults) {
ASSERT_NOK_WITH_MSG(context->AcquireReadState(), "real-time store returned a null read view");
}
-} // namespace
} // namespace paimon::test
diff --git a/src/paimon/core/realtime/realtime_deduplicate_state.cpp b/src/paimon/core/realtime/realtime_deduplicate_state.cpp
new file mode 100644
index 00000000..d913b03a
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_deduplicate_state.cpp
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/realtime/realtime_deduplicate_state.h"
+
+#include <utility>
+
+#include "paimon/core/realtime/arrow_deduplicate_realtime_store.h"
+#include "paimon/core/realtime/realtime_offset_file_index_lookup.h"
+#include "paimon/macros.h"
+
+namespace paimon {
+
+Status RealtimeDeduplicateState::AttachFileIndexLookup(
+ const std::shared_ptr<RealtimeOffsetFileIndexLookup>& file_lookup,
+ const std::shared_ptr<RealtimeStore>& store) {
+ if (!file_lookup || !store) {
+ return Status::Invalid("cannot attach a null deduplicate file lookup or store");
+ }
+ std::shared_ptr<ArrowDeduplicateRealtimeStore> deduplicate_store =
+ std::dynamic_pointer_cast<ArrowDeduplicateRealtimeStore>(store);
+ if (!deduplicate_store) {
+ return Status::Invalid("deduplicate state requires ArrowDeduplicateRealtimeStore");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const KeyOffsetLookup> key_lookup,
+ file_lookup->CreateKeyOffsetLookup());
+ {
+ std::lock_guard<std::mutex> lock(mutex_);
+ if (committed_file_lookup_) {
+ return Status::OK();
+ }
+ PAIMON_RETURN_NOT_OK(deduplicate_store->AttachCommittedKeyOffsetLookup(key_lookup));
+ committed_file_lookup_ = file_lookup;
+ }
+ return Status::OK();
+}
+
+Status RealtimeDeduplicateState::InstallCommittedSnapshot(
+ int64_t committed_end_offset,
+ const std::vector<std::shared_ptr<DataFileMeta>>& active_data_files,
+ const std::shared_ptr<RealtimeStore>& store) {
+ if (committed_end_offset < 0 || !store) {
+ return Status::Invalid("invalid deduplicate committed advancement");
+ }
+ std::shared_ptr<ArrowDeduplicateRealtimeStore> deduplicate_store =
+ std::dynamic_pointer_cast<ArrowDeduplicateRealtimeStore>(store);
+ if (!deduplicate_store) {
+ return Status::Invalid("deduplicate state requires ArrowDeduplicateRealtimeStore");
+ }
+ std::shared_ptr<RealtimeOffsetFileIndexLookup> current_file_lookup;
+ {
+ std::lock_guard<std::mutex> lock(mutex_);
+ if (!committed_file_lookup_) {
+ return Status::Invalid("deduplicate file lookup is not attached");
+ }
+ current_file_lookup = committed_file_lookup_;
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeOffsetFileIndexLookup> next_file_lookup,
+ current_file_lookup->WithDataFiles(active_data_files));
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const KeyOffsetLookup> next_key_lookup,
+ next_file_lookup->CreateKeyOffsetLookup());
+ {
+ std::lock_guard<std::mutex> lock(mutex_);
+ PAIMON_RETURN_NOT_OK(deduplicate_store->AdvanceCommittedOffsetAndLookup(
+ committed_end_offset, next_key_lookup));
+ committed_file_lookup_ = std::move(next_file_lookup);
+ }
+ return Status::OK();
+}
+
+} // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_deduplicate_state.h b/src/paimon/core/realtime/realtime_deduplicate_state.h
new file mode 100644
index 00000000..50e703ee
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_deduplicate_state.h
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <mutex>
+#include <vector>
+
+#include "paimon/result.h"
+
+namespace paimon {
+
+class RealtimeStore;
+struct DataFileMeta;
+
+class RealtimeOffsetFileIndexLookup;
+
+/// Maintains the committed key lookup shared by successive writers of one partition-bucket.
+/// Refresh installs the new snapshot's key lookup together with committed offset advancement.
+/// Pending deletions stay as logical offsets in the store's immutable sealed checkpoints;
+/// RealtimeCommitProgress carries them to commit, which materializes snapshot-specific DVs.
+class RealtimeDeduplicateState {
+ public:
+ Status AttachFileIndexLookup(const std::shared_ptr<RealtimeOffsetFileIndexLookup>& file_lookup,
+ const std::shared_ptr<RealtimeStore>& store);
+
+ Status InstallCommittedSnapshot(
+ int64_t committed_end_offset,
+ const std::vector<std::shared_ptr<DataFileMeta>>& active_data_files,
+ const std::shared_ptr<RealtimeStore>& store);
+
+ private:
+ mutable std::mutex mutex_;
+ std::shared_ptr<RealtimeOffsetFileIndexLookup> committed_file_lookup_;
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_deduplicate_state_test.cpp b/src/paimon/core/realtime/realtime_deduplicate_state_test.cpp
new file mode 100644
index 00000000..45dde7bd
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_deduplicate_state_test.cpp
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/realtime/realtime_deduplicate_state.h"
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/ipc/json_simple.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/io/data_file_path_factory.h"
+#include "paimon/core/realtime/arrow_deduplicate_realtime_store.h"
+#include "paimon/core/realtime/arrow_realtime_store.h"
+#include "paimon/core/realtime/realtime_key_offset_index.h"
+#include "paimon/core/realtime/realtime_offset_file_index_lookup.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+TEST(RealtimeDeduplicateStateTest, TestAttachAndInstallSnapshot) {
+ const std::shared_ptr<arrow::Schema> schema = arrow::schema({
+ DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
+ arrow::field("id", arrow::int64()),
+ arrow::field("value", arrow::utf8()),
+ });
+ const std::shared_ptr<MemoryPool> pool = GetDefaultPool();
+ const std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(pool);
+ auto delegate = std::make_shared<ArrowRealtimeStore>(
+ schema, RealtimeStoreMode::APPEND_ONLY, StatisticsMode::NONE,
+ /*temp_directory=*/"", /*spill_file_system=*/nullptr,
+ /*spill_compression=*/"zstd", /*spill_compression_level=*/1, pool, arrow_pool);
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<ArrowDeduplicateRealtimeStore> store,
+ ArrowDeduplicateRealtimeStore::Create(schema, {"id"}, delegate, pool, arrow_pool));
+
+ std::unique_ptr<UniqueTestDirectory> directory = UniqueTestDirectory::Create();
+ ASSERT_NE(nullptr, directory);
+ auto path_factory = std::make_shared<DataFilePathFactory>();
+ ASSERT_OK(path_factory->Init(directory->Str(), /*format_identifier=*/"parquet",
+ /*data_file_prefix=*/"data-",
+ /*external_path_provider=*/nullptr));
+ ASSERT_OK_AND_ASSIGN(CoreOptions options,
+ CoreOptions::FromMap({{"file-index.bitmap.columns",
+ SpecialFields::RealtimeOffset().Name()}}));
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<RealtimeOffsetFileIndexLookup> initial_lookup,
+ RealtimeOffsetFileIndexLookup::Create(
+ schema, /*data_schema_id=*/0, schema->GetFieldByName("id"),
+ /*data_files=*/{}, path_factory, directory->GetFileSystem(), pool, options));
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<RealtimeOffsetFileIndexLookup> ignored_lookup,
+ RealtimeOffsetFileIndexLookup::Create(
+ schema, /*data_schema_id=*/0, schema->GetFieldByName("id"),
+ /*data_files=*/{}, path_factory, directory->GetFileSystem(), pool, options));
+
+ RealtimeDeduplicateState state;
+ ASSERT_EQ(nullptr, state.committed_file_lookup_);
+ ASSERT_NOK_WITH_MSG(state.AttachFileIndexLookup(nullptr, store), "cannot attach a null");
+ ASSERT_NOK_WITH_MSG(state.AttachFileIndexLookup(initial_lookup, delegate),
+ "requires ArrowDeduplicateRealtimeStore");
+ ASSERT_OK(state.AttachFileIndexLookup(initial_lookup, store));
+ ASSERT_EQ(initial_lookup, state.committed_file_lookup_);
+ ASSERT_OK(state.AttachFileIndexLookup(ignored_lookup, store));
+ ASSERT_EQ(initial_lookup, state.committed_file_lookup_);
+
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<DataFileMeta> first_file,
+ DataFileMeta::ForAppend("prepared-0.parquet", /*file_size=*/0, /*row_count=*/1,
+ SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+ /*max_sequence_number=*/0, /*schema_id=*/0,
+ /*file_source=*/std::nullopt, /*value_stats_cols=*/std::nullopt,
+ /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+ /*write_cols=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<DataFileKeyOffsetIndexWriter> sidecar_writer,
+ DataFileKeyOffsetIndexWriter::Create(schema->field(1), path_factory,
+ directory->GetFileSystem(), pool, options.ToMap()));
+ const std::shared_ptr<arrow::StructArray> keys = checked_pointer_cast<arrow::StructArray>(
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({schema->field(1)}), R"([[1]])")
+ .ValueOrDie());
+ const std::shared_ptr<arrow::Int64Array> offsets = checked_pointer_cast<arrow::Int64Array>(
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::int64(), "[0]").ValueOrDie());
+ ASSERT_OK(sidecar_writer->AddBatch(keys, offsets));
+ ASSERT_OK_AND_ASSIGN(std::string sidecar, sidecar_writer->Finish(first_file));
+ first_file = first_file->CopyWithExtraFiles({sidecar});
+
+ ASSERT_NOK_WITH_MSG(state.InstallCommittedSnapshot(-1, {}, store),
+ "invalid deduplicate committed advancement");
+ ASSERT_NOK_WITH_MSG(state.InstallCommittedSnapshot(1, {}, delegate),
+ "requires ArrowDeduplicateRealtimeStore");
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<const KeyOffsetLookup> old_keys,
+ store->AcquireKeyOffsetLookup());
+ ASSERT_OK_AND_ASSIGN(RoaringBitmap64 old_offsets, old_keys->LookupOffsets(keys));
+ ASSERT_TRUE(old_offsets.IsEmpty());
+
+ ASSERT_OK(state.InstallCommittedSnapshot(/*committed_end_offset=*/1, {first_file}, store));
+ const std::shared_ptr<RealtimeOffsetFileIndexLookup> installed = state.committed_file_lookup_;
+ ASSERT_NE(initial_lookup, installed);
+ ASSERT_EQ(std::vector<std::shared_ptr<DataFileMeta>>{first_file}, installed->DataFiles());
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<const KeyOffsetLookup> new_keys,
+ store->AcquireKeyOffsetLookup());
+ ASSERT_OK_AND_ASSIGN(RoaringBitmap64 new_offsets, new_keys->LookupOffsets(keys));
+ ASSERT_EQ("{0}", new_offsets.ToString());
+ ASSERT_OK_AND_ASSIGN(old_offsets, old_keys->LookupOffsets(keys));
+ ASSERT_TRUE(old_offsets.IsEmpty());
+
+ // A new snapshot may replace its active file set without advancing the durable offset.
+ ASSERT_OK(state.InstallCommittedSnapshot(/*committed_end_offset=*/1, {}, store));
+ ASSERT_TRUE(state.committed_file_lookup_->DataFiles().empty());
+ ASSERT_EQ(std::vector<std::shared_ptr<DataFileMeta>>{first_file}, installed->DataFiles());
+ ASSERT_OK_AND_ASSIGN(RoaringBitmap64 pinned_offsets, new_keys->LookupOffsets(keys));
+ ASSERT_EQ("{0}", pinned_offsets.ToString());
+}
+
+} // namespace paimon::test
diff --git a/src/paimon/core/realtime/realtime_deduplicate_writer.cpp b/src/paimon/core/realtime/realtime_deduplicate_writer.cpp
new file mode 100644
index 00000000..c9080546
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_deduplicate_writer.cpp
@@ -0,0 +1,353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/realtime/realtime_deduplicate_writer.h"
+
+#include <optional>
+#include <utility>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "arrow/c/helpers.h"
+#include "paimon/common/reader/concat_batch_reader.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/append/append_only_writer.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/index/index_file_meta.h"
+#include "paimon/core/io/compact_increment.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/io/data_file_path_factory.h"
+#include "paimon/core/io/data_increment.h"
+#include "paimon/core/realtime/arrow_deduplicate_realtime_store.h"
+#include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_deduplicate_state.h"
+#include "paimon/core/realtime/realtime_key_offset_index.h"
+#include "paimon/core/realtime/realtime_offset_file_index_lookup.h"
+#include "paimon/core/realtime/realtime_offset_utils.h"
+#include "paimon/core/realtime/realtime_schema_layout.h"
+#include "paimon/core/realtime/realtime_utils.h"
+#include "paimon/core/utils/commit_increment.h"
+#include "paimon/macros.h"
+#include "paimon/realtime/realtime_context.h"
+
+namespace paimon {
+namespace {
+
+template <typename T>
+void AppendAll(const std::vector<std::shared_ptr<T>>& source,
+ std::vector<std::shared_ptr<T>>* target) {
+ target->insert(target->end(), source.begin(), source.end());
+}
+
+} // namespace
+
+Result<std::shared_ptr<RealtimeDeduplicateWriter>> RealtimeDeduplicateWriter::Create(
+ const std::map<std::string, std::string>& partition, int32_t bucket,
+ const std::shared_ptr<RealtimeContext>& realtime_context,
+ const std::shared_ptr<AppendOnlyWriter>& file_writer,
+ const std::shared_ptr<RealtimeSchemaLayout>& schema_layout, int64_t schema_id,
+ const std::vector<std::string>& deduplicate_key_fields,
+ const std::vector<std::shared_ptr<DataFileMeta>>& restored_data_files,
+ const std::shared_ptr<DataFilePathFactory>& data_file_path_factory, const CoreOptions& options,
+ const std::string& temp_directory, const std::shared_ptr<MemoryPool>& memory_pool) {
+ if (!realtime_context || !file_writer || !schema_layout || !data_file_path_factory ||
+ !memory_pool) {
+ return Status::Invalid("deduplicate real-time writer is missing a dependency");
+ }
+ PAIMON_ASSIGN_OR_RAISE(int32_t key_position,
+ RealtimeUtils::GetDeduplicateKeyPosition(
+ schema_layout->StoreWriteSchema(), deduplicate_key_fields));
+ PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateOffsetField(schema_layout->StoreWriteSchema()));
+
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> context,
+ RealtimeContextImpl::Cast(realtime_context));
+ auto write_schema = std::make_unique<ArrowSchema>();
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(
+ arrow::ExportSchema(*schema_layout->StoreWriteSchema(), write_schema.get()));
+ RealtimeStoreCreateRequest request{std::move(write_schema), options.ToMap(), memory_pool,
+ RealtimeStoreMode::DEDUPLICATE,
+ options.GetRealtimeStoreStatisticsMode()};
+ request.deduplicate_key_fields = deduplicate_key_fields;
+ if (options.RealtimeSpillEnabled()) {
+ request.temp_directory = temp_directory;
+ request.file_system = options.GetFileSystem();
+ }
+ PAIMON_ASSIGN_OR_RAISE(RealtimeStoreState store_state,
+ context->GetOrCreateRealtimeStore(
+ std::move(request), RealtimePartitionBucket(partition, bucket)));
+ std::shared_ptr<ArrowDeduplicateRealtimeStore> store =
+ std::dynamic_pointer_cast<ArrowDeduplicateRealtimeStore>(store_state.store);
+ if (!store || !store_state.deduplicate_state) {
+ return Status::Invalid("DEDUPLICATE mode requires the default Arrow store and state");
+ }
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<RealtimeOffsetFileIndexLookup> created_lookup,
+ RealtimeOffsetFileIndexLookup::Create(
+ schema_layout->CommitSchema(), schema_id,
+ schema_layout->StoreWriteSchema()->field(key_position), restored_data_files,
+ data_file_path_factory, options.GetFileSystem(), memory_pool, options));
+ PAIMON_RETURN_NOT_OK(
+ store_state.deduplicate_state->AttachFileIndexLookup(created_lookup, store));
+ return std::shared_ptr<RealtimeDeduplicateWriter>(new RealtimeDeduplicateWriter(
+ store, file_writer, schema_layout, schema_layout->StoreWriteSchema()->field(key_position),
+ data_file_path_factory, options.GetFileSystem(), store_state.initial_offset, memory_pool,
+ options.ToMap()));
+}
+
+RealtimeDeduplicateWriter::RealtimeDeduplicateWriter(
+ std::shared_ptr<ArrowDeduplicateRealtimeStore> realtime_store,
+ std::shared_ptr<AppendOnlyWriter> file_writer,
+ std::shared_ptr<RealtimeSchemaLayout> schema_layout,
+ std::shared_ptr<arrow::Field> deduplicate_key_field,
+ std::shared_ptr<DataFilePathFactory> data_file_path_factory,
+ std::shared_ptr<FileSystem> file_system, int64_t next_offset,
+ std::shared_ptr<MemoryPool> memory_pool, std::map<std::string, std::string> options)
+ : arrow_pool_(GetArrowPool(memory_pool)),
+ realtime_store_(std::move(realtime_store)),
+ file_writer_(std::move(file_writer)),
+ schema_layout_(std::move(schema_layout)),
+ deduplicate_key_field_(std::move(deduplicate_key_field)),
+ data_file_path_factory_(std::move(data_file_path_factory)),
+ file_system_(std::move(file_system)),
+ memory_pool_(std::move(memory_pool)),
+ options_(std::move(options)),
+ next_offset_(next_offset) {}
+
+Status RealtimeDeduplicateWriter::Write(std::unique_ptr<RecordBatch>&& batch) {
+ if (!batch || !batch->GetData()) {
+ return Status::Invalid("deduplicate real-time write batch is null");
+ }
+ if (batch->GetData()->length == 0) {
+ return Status::OK();
+ }
+ std::vector<RecordBatch::RowKind> row_kinds = batch->GetRowKind();
+ if (row_kinds.empty()) {
+ row_kinds.assign(batch->GetData()->length, RecordBatch::RowKind::INSERT);
+ }
+ if (row_kinds.size() != static_cast<size_t>(batch->GetData()->length)) {
+ return Status::Invalid("deduplicate row-kind count does not match batch row count");
+ }
+
+ std::lock_guard<std::mutex> lock(realtime_store_mutex_);
+ PAIMON_ASSIGN_OR_RAISE(RealtimeOffsetUtils::ValidatedBatch validated,
+ RealtimeOffsetUtils::ValidateBatch(
+ batch.get(), schema_layout_->InputSchema(), next_offset_));
+ const int32_t key_position =
+ schema_layout_->StoreWriteSchema()->GetFieldIndex(deduplicate_key_field_->name());
+ if (key_position < 0) {
+ return Status::Invalid("deduplicate key field is missing from write data");
+ }
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::StructArray> keys,
+ arrow::StructArray::Make({validated.data->field(key_position)}, {deduplicate_key_field_}));
+
+ // Pin the complete lookup under the short store lock, then perform sidecar lookup without
+ // holding it. Release the building-index snapshot before mutating the index, so ordinary
+ // writes do not copy the entire map. The writer mutex keeps writes ordered while Advance may
+ // proceed.
+ RoaringBitmap64 previous_offsets;
+ {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const KeyOffsetLookup> lookup,
+ realtime_store_->AcquireKeyOffsetLookup());
+ PAIMON_ASSIGN_OR_RAISE(previous_offsets, lookup->LookupOffsets(keys));
+ }
+ PAIMON_RETURN_NOT_OK(realtime_store_->WriteAndLookup(validated.data, row_kinds,
+ validated.offset_range, previous_offsets));
+ next_offset_ = validated.offset_range.end;
+ has_building_data_ = true;
+ return Status::OK();
+}
+
+Status RealtimeDeduplicateWriter::SealCurrentSegment() {
+ PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<RealtimeSegmentHandle>> segment,
+ realtime_store_->SealForCommit());
+ std::lock_guard<std::mutex> lock(realtime_store_mutex_);
+ if (!segment) {
+ return Status::OK();
+ }
+ if (!segment.value()) {
+ return Status::Invalid("deduplicate real-time store sealed a null segment");
+ }
+ has_building_data_ = next_offset_ > segment.value()->GetOffsetRange().end;
+ sealed_segments_.push_back(std::move(segment.value()));
+ return Status::OK();
+}
+
+Status RealtimeDeduplicateWriter::Seal() {
+ std::lock_guard<std::mutex> lock(prepare_mutex_);
+ return SealCurrentSegment();
+}
+
+Result<CommitIncrement> RealtimeDeduplicateWriter::PrepareCommit(bool wait_compaction) {
+ std::lock_guard<std::mutex> prepare_lock(prepare_mutex_);
+ PAIMON_RETURN_NOT_OK(SealCurrentSegment());
+ std::vector<std::shared_ptr<RealtimeSegmentHandle>> segments;
+ {
+ std::lock_guard<std::mutex> store_lock(realtime_store_mutex_);
+ segments.swap(sealed_segments_);
+ }
+ if (segments.empty()) {
+ return CommitIncrement(DataIncrement({}, {}, {}, {}, {}),
+ CompactIncrement({}, {}, {}, {}, {}), nullptr);
+ }
+ auto deleted_offsets = std::make_shared<RoaringBitmap64>();
+ std::vector<std::shared_ptr<DataFileMeta>> new_files;
+ std::vector<std::shared_ptr<IndexFileMeta>> new_index_files;
+ std::vector<std::shared_ptr<IndexFileMeta>> deleted_index_files;
+ for (const std::shared_ptr<RealtimeSegmentHandle>& segment : segments) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const RoaringBitmap64> offset_deletions,
+ ArrowDeduplicateRealtimeStore::OffsetDeletionsOf(segment));
+ *deleted_offsets |= *offset_deletions;
+
+ if (segment->GetRowCount() == 0) {
+ continue;
+ }
+ PAIMON_ASSIGN_OR_RAISE(
+ std::unique_ptr<DataFileKeyOffsetIndexWriter> offset_index_writer,
+ DataFileKeyOffsetIndexWriter::Create(deduplicate_key_field_, data_file_path_factory_,
+ file_system_, memory_pool_, options_));
+ ScopeGuard offset_index_guard([&offset_index_writer]() {
+ if (offset_index_writer) {
+ offset_index_writer->Abort();
+ }
+ });
+ PAIMON_RETURN_NOT_OK(FlushSegment(segment, offset_index_writer.get()));
+ PAIMON_ASSIGN_OR_RAISE(CommitIncrement segment_increment,
+ file_writer_->PrepareCommit(wait_compaction));
+ const DataIncrement& increment = segment_increment.GetNewFilesIncrement();
+ // TODO(xinyu.lxy): Make offset-index writing follow data-file rolling boundaries so one
+ // deduplicate segment can produce multiple data files with one `.offset` sidecar per file.
+ if (increment.NewFiles().size() != 1 || !increment.DeletedFiles().empty() ||
+ !increment.ChangelogFiles().empty() ||
+ !segment_increment.GetCompactIncrement().IsEmpty() ||
+ segment_increment.GetCompactDeletionFile()) {
+ return Status::Invalid(
+ "one non-empty deduplicate segment must produce one append data file");
+ }
+ std::shared_ptr<DataFileMeta> data_file = increment.NewFiles().front();
+ if (data_file->row_count != segment->GetRowCount()) {
+ return Status::Invalid("deduplicate segment and data-file row counts differ");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::string offset_index_file,
+ offset_index_writer->Finish(data_file));
+ std::vector<std::optional<std::string>> extra_files = data_file->extra_files;
+ extra_files.emplace_back(std::move(offset_index_file));
+ data_file = data_file->CopyWithExtraFiles(extra_files);
+ new_files.push_back(data_file);
+ AppendAll(increment.NewIndexFiles(), &new_index_files);
+ AppendAll(increment.DeletedIndexFiles(), &deleted_index_files);
+ offset_index_guard.Release();
+ }
+
+ DataIncrement data_increment(std::move(new_files), {}, {}, std::move(new_index_files),
+ std::move(deleted_index_files));
+ CompactIncrement compact_increment({}, {}, {}, {}, {});
+ CommitIncrement result(data_increment, compact_increment, nullptr);
+ result.SetRealtimeDeletedOffsets(std::move(deleted_offsets));
+ result.SetRealtimeOffsetRange(OffsetRange(segments.front()->GetOffsetRange().begin,
+ segments.back()->GetOffsetRange().end));
+ return result;
+}
+
+Status RealtimeDeduplicateWriter::FlushSegment(
+ const std::shared_ptr<RealtimeSegmentHandle>& segment,
+ DataFileKeyOffsetIndexWriter* offset_index_writer) {
+ if (!offset_index_writer) {
+ return Status::Invalid("deduplicate offset-index writer is null");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::vector<std::unique_ptr<BatchReader>> readers,
+ realtime_store_->CreateCommitReaders(segment));
+ ConcatBatchReader reader(std::move(readers), arrow_pool_);
+ ScopeGuard reader_guard([&reader]() { reader.Close(); });
+ int64_t emitted_rows = 0;
+ while (true) {
+ PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader.NextBatch());
+ if (BatchReader::IsEofBatch(batch)) {
+ break;
+ }
+ auto& [c_array, c_schema] = batch;
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> imported,
+ arrow::ImportArray(c_array.get(), c_schema.get()));
+ if (!imported || imported->type_id() != arrow::Type::STRUCT) {
+ return Status::Invalid("deduplicate commit reader returned a non-StructArray");
+ }
+ std::shared_ptr<arrow::StructArray> data =
+ checked_pointer_cast<arrow::StructArray>(imported);
+ std::shared_ptr<arrow::Array> raw_offsets =
+ data->GetFieldByName(SpecialFields::RealtimeOffset().Name());
+ if (!raw_offsets || raw_offsets->type_id() != arrow::Type::INT64) {
+ return Status::Invalid("deduplicate commit batch has no realtime offset");
+ }
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::StructArray> keys,
+ arrow::StructArray::Make({data->GetFieldByName(deduplicate_key_field_->name())},
+ {deduplicate_key_field_}));
+ PAIMON_RETURN_NOT_OK(offset_index_writer->AddBatch(
+ keys, checked_pointer_cast<arrow::Int64Array>(raw_offsets)));
+ emitted_rows += data->length();
+
+ auto output = std::make_unique<ArrowArray>();
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*data, output.get()));
+ RecordBatchBuilder builder(output.get());
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> record_batch, builder.Finish());
+ PAIMON_RETURN_NOT_OK(file_writer_->Write(std::move(record_batch)));
+ }
+ if (emitted_rows != segment->GetRowCount()) {
+ return Status::Invalid("deduplicate commit reader row count does not match segment");
+ }
+ return Status::OK();
+}
+
+Status RealtimeDeduplicateWriter::Compact(bool) {
+ return Status::Invalid("deduplicate real-time write does not support explicit compaction");
+}
+
+uint64_t RealtimeDeduplicateWriter::GetMemoryUsage() const {
+ return realtime_store_->GetMemoryUsage();
+}
+
+Status RealtimeDeduplicateWriter::FlushMemory() {
+ return Status::OK();
+}
+
+Result<bool> RealtimeDeduplicateWriter::CompactNotCompleted() {
+ return file_writer_->CompactNotCompleted();
+}
+
+Status RealtimeDeduplicateWriter::Sync() {
+ return file_writer_->Sync();
+}
+
+Status RealtimeDeduplicateWriter::Close() {
+ return file_writer_->Close();
+}
+
+bool RealtimeDeduplicateWriter::HasUnpreparedRealtimeData() const {
+ std::lock_guard<std::mutex> lock(realtime_store_mutex_);
+ return has_building_data_ || !sealed_segments_.empty();
+}
+
+std::shared_ptr<Metrics> RealtimeDeduplicateWriter::GetMetrics() const {
+ return file_writer_->GetMetrics();
+}
+
+} // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_deduplicate_writer.h b/src/paimon/core/realtime/realtime_deduplicate_writer.h
new file mode 100644
index 00000000..2644a325
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_deduplicate_writer.h
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <vector>
+
+#include "paimon/core/utils/batch_writer.h"
+
+namespace arrow {
+class Field;
+class MemoryPool;
+} // namespace arrow
+
+namespace paimon {
+
+class AppendOnlyWriter;
+class ArrowDeduplicateRealtimeStore;
+class CoreOptions;
+class DataFileKeyOffsetIndexWriter;
+class DataFilePathFactory;
+class FileSystem;
+class MemoryPool;
+class RealtimeContext;
+class RealtimeSchemaLayout;
+class RealtimeSegmentHandle;
+struct DataFileMeta;
+
+/// Writer for an append table whose user-defined deduplicate key uses last-write-wins semantics.
+/// TODO(xinyu.lxy): Currently only the happy path is supported.
+class RealtimeDeduplicateWriter final : public BatchWriter {
+ public:
+ static Result<std::shared_ptr<RealtimeDeduplicateWriter>> Create(
+ const std::map<std::string, std::string>& partition, int32_t bucket,
+ const std::shared_ptr<RealtimeContext>& realtime_context,
+ const std::shared_ptr<AppendOnlyWriter>& file_writer,
+ const std::shared_ptr<RealtimeSchemaLayout>& schema_layout, int64_t schema_id,
+ const std::vector<std::string>& deduplicate_key_fields,
+ const std::vector<std::shared_ptr<DataFileMeta>>& restored_data_files,
+ const std::shared_ptr<DataFilePathFactory>& data_file_path_factory,
+ const CoreOptions& options, const std::string& temp_directory,
+ const std::shared_ptr<MemoryPool>& memory_pool);
+
+ Status Write(std::unique_ptr<RecordBatch>&& batch) override;
+ Status Seal() override;
+ Result<CommitIncrement> PrepareCommit(bool wait_compaction) override;
+ Status Compact(bool full_compaction) override;
+ uint64_t GetMemoryUsage() const override;
+ Status FlushMemory() override;
+ Result<bool> CompactNotCompleted() override;
+ Status Sync() override;
+ Status Close() override;
+ bool HasUnpreparedRealtimeData() const override;
+ std::shared_ptr<Metrics> GetMetrics() const override;
+
+ private:
+ RealtimeDeduplicateWriter(std::shared_ptr<ArrowDeduplicateRealtimeStore> realtime_store,
+ std::shared_ptr<AppendOnlyWriter> file_writer,
+ std::shared_ptr<RealtimeSchemaLayout> schema_layout,
+ std::shared_ptr<arrow::Field> deduplicate_key_field,
+ std::shared_ptr<DataFilePathFactory> data_file_path_factory,
+ std::shared_ptr<FileSystem> file_system, int64_t next_offset,
+ std::shared_ptr<MemoryPool> memory_pool,
+ std::map<std::string, std::string> options);
+
+ Status SealCurrentSegment();
+ Status FlushSegment(const std::shared_ptr<RealtimeSegmentHandle>& segment,
+ DataFileKeyOffsetIndexWriter* offset_index_writer);
+
+ std::shared_ptr<arrow::MemoryPool> arrow_pool_;
+ std::shared_ptr<ArrowDeduplicateRealtimeStore> realtime_store_;
+ std::shared_ptr<AppendOnlyWriter> file_writer_;
+ std::shared_ptr<RealtimeSchemaLayout> schema_layout_;
+ std::shared_ptr<arrow::Field> deduplicate_key_field_;
+ std::shared_ptr<DataFilePathFactory> data_file_path_factory_;
+ std::shared_ptr<FileSystem> file_system_;
+ std::shared_ptr<MemoryPool> memory_pool_;
+ std::map<std::string, std::string> options_;
+ std::vector<std::shared_ptr<RealtimeSegmentHandle>> sealed_segments_;
+ int64_t next_offset_;
+ bool has_building_data_ = false;
+ mutable std::mutex realtime_store_mutex_;
+ std::mutex prepare_mutex_;
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_key_offset_index.cpp b/src/paimon/core/realtime/realtime_key_offset_index.cpp
new file mode 100644
index 00000000..37ee94cc
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_key_offset_index.cpp
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/realtime/realtime_key_offset_index.h"
+
+#include <optional>
+#include <utility>
+
+#include "arrow/api.h"
+#include "paimon/common/global_index/key_serializer.h"
+#include "paimon/common/io/data_output_stream.h"
+#include "paimon/common/predicate/literal_converter.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/io/data_file_path_factory.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/io/data_input_stream.h"
+#include "paimon/macros.h"
+#include "paimon/memory/bytes.h"
+
+namespace paimon {
+namespace {
+
+constexpr int64_t kOffsetIndexMagic = 0x5041494D4F464653LL;
+constexpr int32_t kOffsetIndexVersion = 1;
+
+Result<std::vector<std::string>> EncodeKeyBatch(
+ const std::shared_ptr<arrow::StructArray>& keys, const std::shared_ptr<arrow::Field>& key_field,
+ const std::shared_ptr<KeySerializer>& key_serializer) {
+ if (!keys || keys->num_fields() != 1 || !keys->field(0)->type()->Equals(key_field->type())) {
+ return Status::Invalid(
+ "deduplicate key lookup must contain exactly the configured key field");
+ }
+ if (keys->null_count() != 0 || keys->field(0)->null_count() != 0) {
+ return Status::Invalid("deduplicate key must not contain null");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::vector<Literal> literals,
+ LiteralConverter::ConvertLiteralsFromArray(*keys->field(0),
+ /*own_data=*/true));
+ std::vector<std::string> encoded;
+ encoded.reserve(literals.size());
+ for (const Literal& literal : literals) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Bytes> bytes, key_serializer->Serialize(literal));
+ std::string key;
+ if (bytes->size() > 0) {
+ key.append(bytes->data(), bytes->size());
+ }
+ encoded.push_back(std::move(key));
+ }
+ return encoded;
+}
+
+Status WriteRaw(const std::shared_ptr<OutputStream>& output, const char* data, int64_t size) {
+ PAIMON_ASSIGN_OR_RAISE(int64_t written, output->Write(data, size));
+ if (written != size) {
+ return Status::IOError("short write for realtime .offset index");
+ }
+ return Status::OK();
+}
+
+} // namespace
+
+Result<std::shared_ptr<MutableKeyOffsetIndex>> MutableKeyOffsetIndex::Create(
+ const std::shared_ptr<arrow::Field>& key_field,
+ const std::shared_ptr<MemoryPool>& memory_pool) {
+ if (!key_field || !memory_pool) {
+ return Status::Invalid("mutable key-offset index requires a key field and memory pool");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<KeySerializer> serializer,
+ KeySerializer::Create(key_field->type(), memory_pool));
+ return std::shared_ptr<MutableKeyOffsetIndex>(
+ new MutableKeyOffsetIndex(key_field, std::move(serializer), std::make_shared<OffsetMap>()));
+}
+
+MutableKeyOffsetIndex::MutableKeyOffsetIndex(std::shared_ptr<arrow::Field> key_field,
+ std::shared_ptr<KeySerializer> key_serializer,
+ std::shared_ptr<OffsetMap> offsets)
+ : key_field_(std::move(key_field)),
+ key_serializer_(std::move(key_serializer)),
+ offsets_(std::move(offsets)) {}
+
+Result<std::vector<std::string>> MutableKeyOffsetIndex::EncodeKeys(
+ const std::shared_ptr<arrow::StructArray>& keys) const {
+ return EncodeKeyBatch(keys, key_field_, key_serializer_);
+}
+
+Status MutableKeyOffsetIndex::Add(const std::shared_ptr<arrow::StructArray>& keys,
+ const std::shared_ptr<arrow::Int64Array>& offsets) {
+ if (!offsets || offsets->null_count() != 0 || !keys || keys->length() != offsets->length()) {
+ return Status::Invalid("key-offset index input columns are not aligned");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> encoded, EncodeKeys(keys));
+ // Copy on write to keep previously pinned snapshots immutable.
+ if (!offsets_.unique()) {
+ offsets_ = std::make_shared<OffsetMap>(*offsets_);
+ }
+ for (int64_t row = 0; row < offsets->length(); ++row) {
+ const int64_t offset = offsets->Value(row);
+ if (offset < 0) {
+ return Status::Invalid("realtime offset must not be negative");
+ }
+ (*offsets_)[encoded[row]] = offset;
+ }
+ return Status::OK();
+}
+
+Status MutableKeyOffsetIndex::Erase(const std::shared_ptr<arrow::StructArray>& keys) {
+ PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> encoded, EncodeKeys(keys));
+ if (!offsets_.unique()) {
+ offsets_ = std::make_shared<OffsetMap>(*offsets_);
+ }
+ for (const std::string& key : encoded) {
+ offsets_->erase(key);
+ }
+ return Status::OK();
+}
+
+Result<RoaringBitmap64> MutableKeyOffsetIndex::LookupOffsets(
+ const std::shared_ptr<arrow::StructArray>& keys) const {
+ PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> encoded, EncodeKeys(keys));
+ RoaringBitmap64 result;
+ for (const std::string& key : encoded) {
+ auto iter = offsets_->find(key);
+ if (iter != offsets_->end()) {
+ result.Add(iter->second);
+ }
+ }
+ return result;
+}
+
+std::shared_ptr<const KeyOffsetLookup> MutableKeyOffsetIndex::Snapshot() const {
+ return std::shared_ptr<const KeyOffsetLookup>(
+ new MutableKeyOffsetIndex(key_field_, key_serializer_, offsets_));
+}
+
+Result<RoaringBitmap64> CompositeKeyOffsetLookup::LookupOffsets(
+ const std::shared_ptr<arrow::StructArray>& keys) const {
+ RoaringBitmap64 result;
+ for (const std::shared_ptr<const KeyOffsetLookup>& lookup : lookups_) {
+ if (!lookup) {
+ continue;
+ }
+ PAIMON_ASSIGN_OR_RAISE(RoaringBitmap64 offsets, lookup->LookupOffsets(keys));
+ result |= offsets;
+ }
+ return result;
+}
+
+Result<std::unique_ptr<DataFileKeyOffsetIndexWriter>> DataFileKeyOffsetIndexWriter::Create(
+ const std::shared_ptr<arrow::Field>& key_field,
+ const std::shared_ptr<DataFilePathFactory>& path_factory,
+ const std::shared_ptr<FileSystem>& file_system, const std::shared_ptr<MemoryPool>& memory_pool,
+ const std::map<std::string, std::string>& options) {
+ (void)options;
+ if (!key_field || !path_factory || !file_system || !memory_pool) {
+ return Status::Invalid("key-offset writer is missing a required dependency");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<KeySerializer> serializer,
+ KeySerializer::Create(key_field->type(), memory_pool));
+ return std::unique_ptr<DataFileKeyOffsetIndexWriter>(new DataFileKeyOffsetIndexWriter(
+ key_field, path_factory, file_system, std::move(serializer)));
+}
+
+DataFileKeyOffsetIndexWriter::DataFileKeyOffsetIndexWriter(
+ std::shared_ptr<arrow::Field> key_field, std::shared_ptr<DataFilePathFactory> path_factory,
+ std::shared_ptr<FileSystem> file_system, std::shared_ptr<KeySerializer> key_serializer)
+ : key_field_(std::move(key_field)),
+ path_factory_(std::move(path_factory)),
+ file_system_(std::move(file_system)),
+ key_serializer_(std::move(key_serializer)) {}
+
+Status DataFileKeyOffsetIndexWriter::AddBatch(const std::shared_ptr<arrow::StructArray>& keys,
+ const std::shared_ptr<arrow::Int64Array>& offsets) {
+ if (!keys || !offsets || keys->length() != offsets->length() || offsets->null_count() != 0) {
+ return Status::Invalid("key-offset writer input columns are not aligned");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> encoded,
+ EncodeKeyBatch(keys, key_field_, key_serializer_));
+ for (int64_t row = 0; row < offsets->length(); ++row) {
+ const int64_t offset = offsets->Value(row);
+ if (offset < 0) {
+ return Status::Invalid("realtime offset must not be negative");
+ }
+ if (!offsets_.emplace(encoded[row], offset).second) {
+ return Status::Invalid(
+ "deduplicate data file contains more than one offset for a deduplicate key");
+ }
+ }
+ return Status::OK();
+}
+
+Result<std::string> DataFileKeyOffsetIndexWriter::Finish(
+ const std::shared_ptr<DataFileMeta>& data_file) {
+ if (!data_file || offsets_.empty()) {
+ return Status::Invalid("cannot finish an empty key-offset index");
+ }
+ const std::string file_name = data_file->file_name + DataFileKeyOffsetIndexReader::kFileSuffix;
+ output_path_ = path_factory_->ToAlignedPath(file_name, data_file);
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<OutputStream> output,
+ file_system_->Create(output_path_, /*overwrite=*/false));
+ DataOutputStream data_output(output);
+ PAIMON_RETURN_NOT_OK(data_output.WriteValue<int64_t>(kOffsetIndexMagic));
+ PAIMON_RETURN_NOT_OK(data_output.WriteValue<int32_t>(kOffsetIndexVersion));
+ PAIMON_RETURN_NOT_OK(data_output.WriteValue<int64_t>(offsets_.size()));
+ for (const auto& [key, offset] : offsets_) {
+ PAIMON_RETURN_NOT_OK(data_output.WriteValue<int32_t>(static_cast<int32_t>(key.size())));
+ if (!key.empty()) {
+ PAIMON_RETURN_NOT_OK(WriteRaw(output, key.data(), key.size()));
+ }
+ PAIMON_RETURN_NOT_OK(data_output.WriteValue<int64_t>(offset));
+ }
+ PAIMON_RETURN_NOT_OK(output->Flush());
+ PAIMON_RETURN_NOT_OK(output->Close());
+ return file_name;
+}
+
+void DataFileKeyOffsetIndexWriter::Abort() {
+ if (!output_path_.empty()) {
+ [[maybe_unused]] Status status = file_system_->Delete(output_path_);
+ output_path_.clear();
+ }
+}
+
+Result<std::shared_ptr<DataFileKeyOffsetIndexReader>> DataFileKeyOffsetIndexReader::Create(
+ const std::shared_ptr<arrow::Field>& key_field, const std::shared_ptr<DataFileMeta>& data_file,
+ const std::shared_ptr<DataFilePathFactory>& path_factory,
+ const std::shared_ptr<FileSystem>& file_system, const std::shared_ptr<MemoryPool>& memory_pool,
+ const std::map<std::string, std::string>& options) {
+ (void)options;
+ if (!key_field || !data_file || !path_factory || !file_system || !memory_pool) {
+ return Status::Invalid("key-offset reader is missing a required dependency");
+ }
+ std::optional<std::string> file_name;
+ for (const std::optional<std::string>& extra : data_file->extra_files) {
+ if (extra && StringUtils::EndsWith(extra.value(), kFileSuffix)) {
+ if (file_name) {
+ return Status::Invalid("data file contains multiple .offset sidecars");
+ }
+ file_name = extra.value();
+ }
+ }
+ if (!file_name) {
+ return Status::Invalid("data file does not contain the required .offset sidecar: ",
+ data_file->file_name);
+ }
+
+ const std::string file_path = path_factory->ToAlignedPath(file_name.value(), data_file);
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> input, file_system->Open(file_path));
+ DataInputStream data_input(input);
+ PAIMON_ASSIGN_OR_RAISE(int64_t magic, data_input.ReadValue<int64_t>());
+ PAIMON_ASSIGN_OR_RAISE(int32_t version, data_input.ReadValue<int32_t>());
+ if (magic != kOffsetIndexMagic || version != kOffsetIndexVersion) {
+ return Status::Invalid("unsupported realtime .offset index format: ", file_name.value());
+ }
+ PAIMON_ASSIGN_OR_RAISE(int64_t entry_count, data_input.ReadValue<int64_t>());
+ if (entry_count < 0) {
+ return Status::Invalid("negative entry count in realtime .offset index");
+ }
+ auto offsets = std::make_shared<std::map<std::string, int64_t>>();
+ for (int64_t entry = 0; entry < entry_count; ++entry) {
+ PAIMON_ASSIGN_OR_RAISE(int32_t key_size, data_input.ReadValue<int32_t>());
+ if (key_size < 0) {
+ return Status::Invalid("invalid key size in realtime .offset index");
+ }
+ std::string key(key_size, '\0');
+ if (key_size > 0) {
+ PAIMON_RETURN_NOT_OK(data_input.Read(key.data(), key_size));
+ }
+ PAIMON_ASSIGN_OR_RAISE(int64_t offset, data_input.ReadValue<int64_t>());
+ if (offset < 0) {
+ return Status::Invalid("negative offset in realtime .offset index");
+ }
+ if (!offsets->emplace(std::move(key), offset).second) {
+ return Status::Invalid("duplicate key in realtime .offset index");
+ }
+ }
+ PAIMON_ASSIGN_OR_RAISE(int64_t position, data_input.GetPos());
+ PAIMON_ASSIGN_OR_RAISE(int64_t length, data_input.Length());
+ if (position != length) {
+ return Status::Invalid("trailing bytes in realtime .offset index");
+ }
+ PAIMON_RETURN_NOT_OK(input->Close());
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<KeySerializer> serializer,
+ KeySerializer::Create(key_field->type(), memory_pool));
+ return std::shared_ptr<DataFileKeyOffsetIndexReader>(
+ new DataFileKeyOffsetIndexReader(key_field, std::move(serializer), std::move(offsets)));
+}
+
+Result<RoaringBitmap64> DataFileKeyOffsetIndexReader::LookupOffsets(
+ const std::shared_ptr<arrow::StructArray>& keys) const {
+ PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> encoded,
+ EncodeKeyBatch(keys, key_field_, key_serializer_));
+ RoaringBitmap64 result;
+ for (const std::string& key : encoded) {
+ auto iter = offsets_->find(key);
+ if (iter != offsets_->end()) {
+ result.Add(iter->second);
+ }
+ }
+ return result;
+}
+
+} // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_key_offset_index.h b/src/paimon/core/realtime/realtime_key_offset_index.h
new file mode 100644
index 00000000..0c8e9f84
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_key_offset_index.h
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/type_fwd.h"
+#include "paimon/result.h"
+#include "paimon/utils/roaring_bitmap64.h"
+
+namespace paimon {
+
+class DataFilePathFactory;
+class FileSystem;
+class KeySerializer;
+class MemoryPool;
+struct DataFileMeta;
+
+/// Immutable user-defined deduplicate key to global realtime-offset lookup.
+class KeyOffsetLookup {
+ public:
+ virtual ~KeyOffsetLookup() = default;
+
+ virtual Result<RoaringBitmap64> LookupOffsets(
+ const std::shared_ptr<arrow::StructArray>& keys) const = 0;
+};
+
+/// Copy-on-write in-memory implementation used by one building segment. Each key retains only its
+/// latest live offset; overwrites replace the value and deletes erase the key.
+class MutableKeyOffsetIndex final : public KeyOffsetLookup {
+ public:
+ static Result<std::shared_ptr<MutableKeyOffsetIndex>> Create(
+ const std::shared_ptr<arrow::Field>& key_field,
+ const std::shared_ptr<MemoryPool>& memory_pool);
+
+ Status Add(const std::shared_ptr<arrow::StructArray>& keys,
+ const std::shared_ptr<arrow::Int64Array>& offsets);
+
+ Status Erase(const std::shared_ptr<arrow::StructArray>& keys);
+
+ Result<RoaringBitmap64> LookupOffsets(
+ const std::shared_ptr<arrow::StructArray>& keys) const override;
+
+ std::shared_ptr<const KeyOffsetLookup> Snapshot() const;
+
+ private:
+ using OffsetMap = std::map<std::string, int64_t>;
+
+ MutableKeyOffsetIndex(std::shared_ptr<arrow::Field> key_field,
+ std::shared_ptr<KeySerializer> key_serializer,
+ std::shared_ptr<OffsetMap> offsets);
+
+ Result<std::vector<std::string>> EncodeKeys(
+ const std::shared_ptr<arrow::StructArray>& keys) const;
+
+ std::shared_ptr<arrow::Field> key_field_;
+ std::shared_ptr<KeySerializer> key_serializer_;
+ std::shared_ptr<OffsetMap> offsets_;
+};
+
+/// Unions a fixed set of immutable lookups. The set itself is pinned by a writer lookup view.
+class CompositeKeyOffsetLookup final : public KeyOffsetLookup {
+ public:
+ explicit CompositeKeyOffsetLookup(std::vector<std::shared_ptr<const KeyOffsetLookup>>&& lookups)
+ : lookups_(std::move(lookups)) {}
+
+ Result<RoaringBitmap64> LookupOffsets(
+ const std::shared_ptr<arrow::StructArray>& keys) const override;
+
+ private:
+ std::vector<std::shared_ptr<const KeyOffsetLookup>> lookups_;
+};
+
+/// Buffers one data file's surviving key/offset pairs and writes a temporary `.offset` sidecar.
+/// Segment-level deduplication guarantees at most one offset per key in the resulting data file.
+/// TODO(xinyu.lxy): This writer and its `.offset` sidecar format are experimental. The format will
+/// be revised before production use.
+class DataFileKeyOffsetIndexWriter {
+ public:
+ static Result<std::unique_ptr<DataFileKeyOffsetIndexWriter>> Create(
+ const std::shared_ptr<arrow::Field>& key_field,
+ const std::shared_ptr<DataFilePathFactory>& path_factory,
+ const std::shared_ptr<FileSystem>& file_system,
+ const std::shared_ptr<MemoryPool>& memory_pool,
+ const std::map<std::string, std::string>& options);
+
+ Status AddBatch(const std::shared_ptr<arrow::StructArray>& keys,
+ const std::shared_ptr<arrow::Int64Array>& offsets);
+
+ Result<std::string> Finish(const std::shared_ptr<DataFileMeta>& data_file);
+
+ void Abort();
+
+ private:
+ DataFileKeyOffsetIndexWriter(std::shared_ptr<arrow::Field> key_field,
+ std::shared_ptr<DataFilePathFactory> path_factory,
+ std::shared_ptr<FileSystem> file_system,
+ std::shared_ptr<KeySerializer> key_serializer);
+
+ std::shared_ptr<arrow::Field> key_field_;
+ std::shared_ptr<DataFilePathFactory> path_factory_;
+ std::shared_ptr<FileSystem> file_system_;
+ std::shared_ptr<KeySerializer> key_serializer_;
+ std::map<std::string, int64_t> offsets_;
+ std::string output_path_;
+};
+
+/// Reads one temporary `.offset` sidecar attached to a data file.
+class DataFileKeyOffsetIndexReader final : public KeyOffsetLookup {
+ public:
+ static constexpr const char* kFileSuffix = ".offset";
+
+ static Result<std::shared_ptr<DataFileKeyOffsetIndexReader>> Create(
+ const std::shared_ptr<arrow::Field>& key_field,
+ const std::shared_ptr<DataFileMeta>& data_file,
+ const std::shared_ptr<DataFilePathFactory>& path_factory,
+ const std::shared_ptr<FileSystem>& file_system,
+ const std::shared_ptr<MemoryPool>& memory_pool,
+ const std::map<std::string, std::string>& options);
+
+ Result<RoaringBitmap64> LookupOffsets(
+ const std::shared_ptr<arrow::StructArray>& keys) const override;
+
+ private:
+ DataFileKeyOffsetIndexReader(std::shared_ptr<arrow::Field> key_field,
+ std::shared_ptr<KeySerializer> key_serializer,
+ std::shared_ptr<const std::map<std::string, int64_t>> offsets)
+ : key_field_(std::move(key_field)),
+ key_serializer_(std::move(key_serializer)),
+ offsets_(std::move(offsets)) {}
+
+ std::shared_ptr<arrow::Field> key_field_;
+ std::shared_ptr<KeySerializer> key_serializer_;
+ std::shared_ptr<const std::map<std::string, int64_t>> offsets_;
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_key_offset_index_test.cpp b/src/paimon/core/realtime/realtime_key_offset_index_test.cpp
new file mode 100644
index 00000000..c58c4f73
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_key_offset_index_test.cpp
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/realtime/realtime_key_offset_index.h"
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/ipc/json_simple.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/io/data_file_path_factory.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+namespace {
+
+std::shared_ptr<arrow::StructArray> MakeKeys(const std::shared_ptr<arrow::Field>& key_field,
+ const std::string& json) {
+ return checked_pointer_cast<arrow::StructArray>(
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({key_field}), json).ValueOrDie());
+}
+
+std::shared_ptr<arrow::Int64Array> MakeOffsets(const std::string& json) {
+ return checked_pointer_cast<arrow::Int64Array>(
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::int64(), json).ValueOrDie());
+}
+
+std::shared_ptr<DataFileMeta> MakeDataFile(
+ const std::string& file_name, int64_t row_count,
+ const std::vector<std::optional<std::string>>& extra_files = {}) {
+ return std::make_shared<DataFileMeta>(
+ file_name, /*file_size=*/0, row_count, DataFileMeta::EmptyMinKey(),
+ DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+ /*min_sequence_number=*/0, /*max_sequence_number=*/0, /*schema_id=*/0,
+ DataFileMeta::DUMMY_LEVEL, extra_files, Timestamp(0, 0),
+ /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr,
+ /*file_source=*/std::nullopt, /*value_stats_cols=*/std::nullopt,
+ /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+ /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
+}
+
+void AssertOffsets(const RoaringBitmap64& offsets, const std::vector<int64_t>& expected) {
+ ASSERT_EQ(expected.size(), offsets.Cardinality());
+ for (int64_t offset : expected) {
+ ASSERT_TRUE(offsets.Contains(offset));
+ }
+}
+
+} // namespace
+
+TEST(RealtimeKeyOffsetIndexTest, TestMutableIndexSnapshotIsCopyOnWrite) {
+ const std::shared_ptr<arrow::Field> key_field = arrow::field("id", arrow::int64());
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<MutableKeyOffsetIndex> index,
+ MutableKeyOffsetIndex::Create(key_field, GetDefaultPool()));
+ ASSERT_OK(index->Add(MakeKeys(key_field, R"([[1], [2]])"), MakeOffsets("[10, 20]")));
+ const std::shared_ptr<const KeyOffsetLookup> snapshot = index->Snapshot();
+
+ ASSERT_OK(index->Add(MakeKeys(key_field, R"([[1]])"), MakeOffsets("[30]")));
+ ASSERT_OK(index->Erase(MakeKeys(key_field, R"([[2]])")));
+
+ ASSERT_OK_AND_ASSIGN(RoaringBitmap64 current,
+ index->LookupOffsets(MakeKeys(key_field, R"([[1], [2], [3]])")));
+ AssertOffsets(current, {30});
+ ASSERT_OK_AND_ASSIGN(RoaringBitmap64 pinned,
+ snapshot->LookupOffsets(MakeKeys(key_field, R"([[1], [2], [3]])")));
+ AssertOffsets(pinned, {10, 20});
+}
+
+TEST(RealtimeKeyOffsetIndexTest, TestMutableIndexValidatesInput) {
+ const std::shared_ptr<arrow::Field> key_field = arrow::field("id", arrow::int64());
+ ASSERT_NOK_WITH_MSG(MutableKeyOffsetIndex::Create(nullptr, GetDefaultPool()),
+ "requires a key field and memory pool");
+ ASSERT_NOK_WITH_MSG(MutableKeyOffsetIndex::Create(key_field, nullptr),
+ "requires a key field and memory pool");
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<MutableKeyOffsetIndex> index,
+ MutableKeyOffsetIndex::Create(key_field, GetDefaultPool()));
+
+ ASSERT_NOK_WITH_MSG(index->Add(MakeKeys(key_field, R"([[1], [2]])"), MakeOffsets("[10]")),
+ "input columns are not aligned");
+ ASSERT_NOK_WITH_MSG(index->Add(MakeKeys(key_field, R"([[1]])"), MakeOffsets("[-1]")),
+ "offset must not be negative");
+ ASSERT_NOK_WITH_MSG(index->Add(MakeKeys(key_field, R"([[null]])"), MakeOffsets("[10]")),
+ "deduplicate key must not contain null");
+ ASSERT_NOK_WITH_MSG(
+ index->LookupOffsets(MakeKeys(arrow::field("id", arrow::utf8()), R"([["1"]])")),
+ "exactly the configured key field");
+}
+
+TEST(RealtimeKeyOffsetIndexTest, TestCompositeLookupUnionsChildren) {
+ const std::shared_ptr<arrow::Field> key_field = arrow::field("id", arrow::int64());
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<MutableKeyOffsetIndex> first,
+ MutableKeyOffsetIndex::Create(key_field, GetDefaultPool()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<MutableKeyOffsetIndex> second,
+ MutableKeyOffsetIndex::Create(key_field, GetDefaultPool()));
+ ASSERT_OK(first->Add(MakeKeys(key_field, R"([[1]])"), MakeOffsets("[10]")));
+ ASSERT_OK(second->Add(MakeKeys(key_field, R"([[1], [2]])"), MakeOffsets("[30, 20]")));
+ std::vector<std::shared_ptr<const KeyOffsetLookup>> children = {first->Snapshot(), nullptr,
+ second->Snapshot()};
+ CompositeKeyOffsetLookup composite(std::move(children));
+
+ ASSERT_OK_AND_ASSIGN(RoaringBitmap64 result,
+ composite.LookupOffsets(MakeKeys(key_field, R"([[1], [2], [3]])")));
+ AssertOffsets(result, {10, 20, 30});
+}
+
+TEST(RealtimeKeyOffsetIndexTest, TestSidecarRoundTripAndAbort) {
+ std::unique_ptr<UniqueTestDirectory> directory = UniqueTestDirectory::Create();
+ ASSERT_NE(nullptr, directory);
+ const std::shared_ptr<arrow::Field> key_field = arrow::field("id", arrow::int64());
+ auto path_factory = std::make_shared<DataFilePathFactory>();
+ ASSERT_OK(path_factory->Init(directory->Str(), /*format_identifier=*/"parquet",
+ /*data_file_prefix=*/"data-",
+ /*external_path_provider=*/nullptr));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<DataFileKeyOffsetIndexWriter> writer,
+ DataFileKeyOffsetIndexWriter::Create(key_field, path_factory, directory->GetFileSystem(),
+ GetDefaultPool(), /*options=*/{}));
+ ASSERT_OK(writer->AddBatch(MakeKeys(key_field, R"([[1], [2]])"), MakeOffsets("[7, 9]")));
+ ASSERT_NOK_WITH_MSG(writer->AddBatch(MakeKeys(key_field, R"([[2]])"), MakeOffsets("[11]")),
+ "more than one offset for a deduplicate key");
+
+ std::shared_ptr<DataFileMeta> data_file = MakeDataFile("data-0.parquet", /*row_count=*/2);
+ ASSERT_OK_AND_ASSIGN(std::string sidecar, writer->Finish(data_file));
+ ASSERT_EQ("data-0.parquet.offset", sidecar);
+ data_file->extra_files.emplace_back(sidecar);
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<DataFileKeyOffsetIndexReader> reader,
+ DataFileKeyOffsetIndexReader::Create(key_field, data_file, path_factory,
+ directory->GetFileSystem(), GetDefaultPool(), {}));
+ ASSERT_OK_AND_ASSIGN(RoaringBitmap64 result,
+ reader->LookupOffsets(MakeKeys(key_field, R"([[2], [3], [1]])")));
+ AssertOffsets(result, {7, 9});
+
+ const std::string sidecar_path = path_factory->ToAlignedPath(sidecar, data_file);
+ ASSERT_OK_AND_ASSIGN(bool exists, directory->GetFileSystem()->Exists(sidecar_path));
+ ASSERT_TRUE(exists);
+ writer->Abort();
+ ASSERT_OK_AND_ASSIGN(exists, directory->GetFileSystem()->Exists(sidecar_path));
+ ASSERT_FALSE(exists);
+ ASSERT_OK_AND_ASSIGN(result, reader->LookupOffsets(MakeKeys(key_field, R"([[1], [2]])")));
+ AssertOffsets(result, {7, 9});
+}
+
+TEST(RealtimeKeyOffsetIndexTest, TestSidecarRoundTripWithEmptyStringKey) {
+ std::unique_ptr<UniqueTestDirectory> directory = UniqueTestDirectory::Create();
+ ASSERT_NE(nullptr, directory);
+ const std::shared_ptr<arrow::Field> key_field = arrow::field("id", arrow::utf8());
+ auto path_factory = std::make_shared<DataFilePathFactory>();
+ ASSERT_OK(path_factory->Init(directory->Str(), /*format_identifier=*/"parquet",
+ /*data_file_prefix=*/"data-",
+ /*external_path_provider=*/nullptr));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<DataFileKeyOffsetIndexWriter> writer,
+ DataFileKeyOffsetIndexWriter::Create(key_field, path_factory, directory->GetFileSystem(),
+ GetDefaultPool(), /*options=*/{}));
+ ASSERT_OK(writer->AddBatch(MakeKeys(key_field, R"([[""], ["alpha"]])"), MakeOffsets("[7, 9]")));
+
+ std::shared_ptr<DataFileMeta> data_file = MakeDataFile("data-0.parquet", /*row_count=*/2);
+ ASSERT_OK_AND_ASSIGN(std::string sidecar, writer->Finish(data_file));
+ data_file->extra_files.emplace_back(sidecar);
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<DataFileKeyOffsetIndexReader> reader,
+ DataFileKeyOffsetIndexReader::Create(key_field, data_file, path_factory,
+ directory->GetFileSystem(), GetDefaultPool(), {}));
+ ASSERT_OK_AND_ASSIGN(RoaringBitmap64 empty_key_offset,
+ reader->LookupOffsets(MakeKeys(key_field, R"([[""]])")));
+ AssertOffsets(empty_key_offset, {7});
+ ASSERT_OK_AND_ASSIGN(RoaringBitmap64 nonempty_key_offset,
+ reader->LookupOffsets(MakeKeys(key_field, R"([["alpha"]])")));
+ AssertOffsets(nonempty_key_offset, {9});
+ ASSERT_OK_AND_ASSIGN(RoaringBitmap64 missing_key_offset,
+ reader->LookupOffsets(MakeKeys(key_field, R"([["missing"]])")));
+ ASSERT_TRUE(missing_key_offset.IsEmpty());
+}
+
+TEST(RealtimeKeyOffsetIndexTest, TestSidecarValidation) {
+ std::unique_ptr<UniqueTestDirectory> directory = UniqueTestDirectory::Create();
+ ASSERT_NE(nullptr, directory);
+ const std::shared_ptr<arrow::Field> key_field = arrow::field("id", arrow::int64());
+ auto path_factory = std::make_shared<DataFilePathFactory>();
+ ASSERT_OK(path_factory->Init(directory->Str(), /*format_identifier=*/"parquet",
+ /*data_file_prefix=*/"data-",
+ /*external_path_provider=*/nullptr));
+
+ ASSERT_NOK_WITH_MSG(DataFileKeyOffsetIndexWriter::Create(
+ key_field, nullptr, directory->GetFileSystem(), GetDefaultPool(), {}),
+ "missing a required dependency");
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<DataFileKeyOffsetIndexWriter> writer,
+ DataFileKeyOffsetIndexWriter::Create(key_field, path_factory, directory->GetFileSystem(),
+ GetDefaultPool(), /*options=*/{}));
+ ASSERT_NOK_WITH_MSG(writer->Finish(MakeDataFile("empty.parquet", /*row_count=*/0)),
+ "cannot finish an empty key-offset index");
+
+ ASSERT_NOK_WITH_MSG(DataFileKeyOffsetIndexReader::Create(
+ key_field, MakeDataFile("missing.parquet", /*row_count=*/1),
+ path_factory, directory->GetFileSystem(), GetDefaultPool(), {}),
+ "does not contain the required .offset sidecar");
+ ASSERT_NOK_WITH_MSG(
+ DataFileKeyOffsetIndexReader::Create(
+ key_field,
+ MakeDataFile("duplicate.parquet", /*row_count=*/1, {"first.offset", "second.offset"}),
+ path_factory, directory->GetFileSystem(), GetDefaultPool(), {}),
+ "contains multiple .offset sidecars");
+}
+
+} // namespace paimon::test
diff --git a/src/paimon/core/realtime/realtime_offset_batch_reader.cpp b/src/paimon/core/realtime/realtime_offset_batch_reader.cpp
index ba13d8d4..0237d3ec 100644
--- a/src/paimon/core/realtime/realtime_offset_batch_reader.cpp
+++ b/src/paimon/core/realtime/realtime_offset_batch_reader.cpp
@@ -36,8 +36,9 @@
namespace paimon {
RealtimeOffsetBatchReader::RealtimeOffsetBatchReader(std::unique_ptr<BatchReader>&& reader,
- const OffsetRange& visible_offsets)
- : reader_(std::move(reader)), visible_offsets_(visible_offsets) {}
+ const OffsetRange& visible_offsets,
+ bool keep_offset)
+ : reader_(std::move(reader)), visible_offsets_(visible_offsets), keep_offset_(keep_offset) {}
Result<BatchReader::ReadBatch> RealtimeOffsetBatchReader::NextBatch() {
return Status::Invalid(
@@ -91,9 +92,12 @@ Result<BatchReader::ReadBatchWithBitmap> RealtimeOffsetBatchReader::NextBatchWit
if (output_bitmap.IsEmpty()) {
continue;
}
- PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::StructArray> output,
- ArrowUtils::RemoveFieldFromStructArray(
- struct_array, SpecialFields::RealtimeOffset().Name()));
+ std::shared_ptr<arrow::StructArray> output = struct_array;
+ if (!keep_offset_) {
+ PAIMON_ASSIGN_OR_RAISE(output,
+ ArrowUtils::RemoveFieldFromStructArray(
+ struct_array, SpecialFields::RealtimeOffset().Name()));
+ }
PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*output, c_array.get(), c_schema.get()));
return ReadBatchWithBitmap(std::move(batch), std::move(output_bitmap));
}
diff --git a/src/paimon/core/realtime/realtime_offset_batch_reader.h b/src/paimon/core/realtime/realtime_offset_batch_reader.h
index f72177da..227f9e61 100644
--- a/src/paimon/core/realtime/realtime_offset_batch_reader.h
+++ b/src/paimon/core/realtime/realtime_offset_batch_reader.h
@@ -28,7 +28,7 @@ namespace paimon {
class RealtimeOffsetBatchReader final : public BatchReader {
public:
RealtimeOffsetBatchReader(std::unique_ptr<BatchReader>&& reader,
- const OffsetRange& visible_offsets);
+ const OffsetRange& visible_offsets, bool keep_offset);
Result<ReadBatch> NextBatch() override;
@@ -41,6 +41,7 @@ class RealtimeOffsetBatchReader final : public BatchReader {
private:
std::unique_ptr<BatchReader> reader_;
OffsetRange visible_offsets_;
+ bool keep_offset_;
};
} // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_offset_batch_reader_test.cpp b/src/paimon/core/realtime/realtime_offset_batch_reader_test.cpp
index 60964955..d8d687d2 100644
--- a/src/paimon/core/realtime/realtime_offset_batch_reader_test.cpp
+++ b/src/paimon/core/realtime/realtime_offset_batch_reader_test.cpp
@@ -55,7 +55,7 @@ TEST(RealtimeOffsetBatchReaderTest, TestFilterBitmapAndRemoveOffset) {
input_bitmap.AddRange(0, static_cast<int32_t>(data->length()));
auto input =
std::make_unique<MockFileBatchReader>(data, type, input_bitmap, /*read_batch_size=*/2);
- RealtimeOffsetBatchReader reader(std::move(input), OffsetRange(2, 5));
+ RealtimeOffsetBatchReader reader(std::move(input), OffsetRange(2, 5), /*keep_offset=*/false);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> result,
ReadResultCollector::CollectResult(&reader));
@@ -76,7 +76,7 @@ TEST(RealtimeOffsetBatchReaderTest, TestRejectsPartialBitmap) {
auto input =
std::make_unique<MockFileBatchReader>(data, type, input_bitmap, /*read_batch_size=*/2);
input->EnableRandomizeBatchSize(false);
- RealtimeOffsetBatchReader reader(std::move(input), OffsetRange(0, 2));
+ RealtimeOffsetBatchReader reader(std::move(input), OffsetRange(0, 2), /*keep_offset=*/false);
ASSERT_NOK_WITH_MSG(ReadResultCollector::CollectResult(&reader),
"must cover every raw transport row");
@@ -87,7 +87,7 @@ TEST(RealtimeOffsetBatchReaderTest, TestFilterWithoutInputBitmap) {
std::shared_ptr<arrow::Array> data =
MakeArray(type, R"([[10, 0], [11, 1], [12, 2], [13, 3], [14, 4]])");
auto input = std::make_unique<MockFileBatchReader>(data, type, /*read_batch_size=*/3);
- RealtimeOffsetBatchReader reader(std::move(input), OffsetRange(1, 4));
+ RealtimeOffsetBatchReader reader(std::move(input), OffsetRange(1, 4), /*keep_offset=*/false);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> result,
ReadResultCollector::CollectResult(&reader));
@@ -104,7 +104,7 @@ TEST(RealtimeOffsetBatchReaderTest, TestRejectsNullOffsetInBatch) {
std::shared_ptr<arrow::DataType> type = MakeDataType(/*offset_nullable=*/true);
std::shared_ptr<arrow::Array> data = MakeArray(type, R"([[10, 0], [11, null]])");
auto input = std::make_unique<MockFileBatchReader>(data, type, /*read_batch_size=*/2);
- RealtimeOffsetBatchReader reader(std::move(input), OffsetRange(0, 1));
+ RealtimeOffsetBatchReader reader(std::move(input), OffsetRange(0, 1), /*keep_offset=*/false);
ASSERT_NOK_WITH_MSG(ReadResultCollector::CollectResult(&reader), "offset column contains null");
}
@@ -113,7 +113,7 @@ TEST(RealtimeOffsetBatchReaderTest, TestNextBatchIsUnsupported) {
std::shared_ptr<arrow::DataType> type = MakeDataType();
std::shared_ptr<arrow::Array> data = MakeArray(type, R"([[10, 0]])");
auto input = std::make_unique<MockFileBatchReader>(data, type, /*read_batch_size=*/1);
- RealtimeOffsetBatchReader reader(std::move(input), OffsetRange(0, 1));
+ RealtimeOffsetBatchReader reader(std::move(input), OffsetRange(0, 1), /*keep_offset=*/false);
ASSERT_NOK_WITH_MSG(reader.NextBatch(), "should use NextBatchWithBitmap");
}
diff --git a/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp b/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp
new file mode 100644
index 00000000..83f39ac7
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/realtime/realtime_offset_file_index_lookup.h"
+
+#include <algorithm>
+#include <optional>
+#include <utility>
+
+#include "arrow/api.h"
+#include "fmt/format.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/io/data_file_path_factory.h"
+#include "paimon/core/io/file_index_evaluator.h"
+#include "paimon/core/realtime/realtime_key_offset_index.h"
+#include "paimon/file_index/bitmap_index_result.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate_builder.h"
+
+namespace paimon {
+
+Result<std::shared_ptr<RealtimeOffsetFileIndexLookup>> RealtimeOffsetFileIndexLookup::Create(
+ const std::shared_ptr<arrow::Schema>& data_schema, int64_t data_schema_id,
+ const std::shared_ptr<arrow::Field>& deduplicate_key_field,
+ const std::vector<std::shared_ptr<DataFileMeta>>& data_files,
+ const std::shared_ptr<DataFilePathFactory>& path_factory,
+ const std::shared_ptr<FileSystem>& file_system, const std::shared_ptr<MemoryPool>& memory_pool,
+ const CoreOptions& options) {
+ if (!data_schema || data_schema_id < 0 || !deduplicate_key_field || !path_factory ||
+ !file_system || !memory_pool) {
+ return Status::Invalid("realtime offset file-index lookup is missing a dependency");
+ }
+ // TODO(xinyu.lxy): Bitmap is the mandatory first implementation. Select a more suitable
+ // high-cardinality offset index (for example BSI) after its exact row-position API is ready.
+ for (const std::shared_ptr<DataFileMeta>& file : data_files) {
+ if (!file || file->row_count < 0 || file->row_count > RoaringBitmap32::MAX_VALUE) {
+ return Status::Invalid("invalid data file for realtime offset lookup");
+ }
+ }
+ return std::shared_ptr<RealtimeOffsetFileIndexLookup>(
+ new RealtimeOffsetFileIndexLookup(data_schema, data_schema_id, deduplicate_key_field,
+ std::vector<std::shared_ptr<DataFileMeta>>(data_files),
+ path_factory, file_system, memory_pool, options.ToMap()));
+}
+
+RealtimeOffsetFileIndexLookup::RealtimeOffsetFileIndexLookup(
+ std::shared_ptr<arrow::Schema> data_schema, int64_t data_schema_id,
+ std::shared_ptr<arrow::Field> deduplicate_key_field,
+ std::vector<std::shared_ptr<DataFileMeta>> data_files,
+ std::shared_ptr<DataFilePathFactory> path_factory, std::shared_ptr<FileSystem> file_system,
+ std::shared_ptr<MemoryPool> memory_pool, std::map<std::string, std::string> options)
+ : data_schema_(std::move(data_schema)),
+ data_schema_id_(data_schema_id),
+ deduplicate_key_field_(std::move(deduplicate_key_field)),
+ data_files_(std::move(data_files)),
+ path_factory_(std::move(path_factory)),
+ file_system_(std::move(file_system)),
+ memory_pool_(std::move(memory_pool)),
+ options_(std::move(options)) {}
+
+Result<std::map<std::string, RoaringBitmap32>> RealtimeOffsetFileIndexLookup::LookupFilePositions(
+ const RoaringBitmap64& offsets) const {
+ std::map<std::string, RoaringBitmap32> result;
+ if (offsets.IsEmpty()) {
+ return result;
+ }
+ const std::string& offset_name = SpecialFields::RealtimeOffset().Name();
+ const int32_t offset_position = data_schema_->GetFieldIndex(offset_name);
+ for (const std::shared_ptr<DataFileMeta>& file : data_files_) {
+ std::optional<int32_t> stats_offset_position;
+ if (file->value_stats_cols) {
+ const auto iter = std::find(file->value_stats_cols->begin(),
+ file->value_stats_cols->end(), offset_name);
+ if (iter != file->value_stats_cols->end()) {
+ stats_offset_position =
+ static_cast<int32_t>(iter - file->value_stats_cols->begin());
+ }
+ } else if (file->schema_id == data_schema_id_) {
+ stats_offset_position = offset_position;
+ }
+ // TODO(xinyu.lxy): Resolve `file->schema_id` to its write-time schema before using stats
+ // from an evolved schema. `write_cols` belongs to data evolution, which real-time
+ // deduplicate does not support yet.
+
+ const BinaryRow& min_values = file->value_stats.MinValues();
+ const BinaryRow& max_values = file->value_stats.MaxValues();
+ const bool has_offset_stats = stats_offset_position && *stats_offset_position >= 0 &&
+ *stats_offset_position < min_values.GetFieldCount() &&
+ *stats_offset_position < max_values.GetFieldCount() &&
+ !min_values.IsNullAt(*stats_offset_position) &&
+ !max_values.IsNullAt(*stats_offset_position) &&
+ min_values.GetLong(*stats_offset_position) <=
+ max_values.GetLong(*stats_offset_position);
+
+ std::vector<Literal> literals;
+ if (has_offset_stats) {
+ const int64_t min_offset = min_values.GetLong(*stats_offset_position);
+ const int64_t max_offset = max_values.GetLong(*stats_offset_position);
+ for (RoaringBitmap64::Iterator iter = offsets.EqualOrLarger(min_offset);
+ iter != offsets.End() && *iter <= max_offset; ++iter) {
+ literals.emplace_back(*iter);
+ }
+ } else {
+ literals.reserve(offsets.Cardinality());
+ for (RoaringBitmap64::Iterator iter = offsets.Begin(); iter != offsets.End(); ++iter) {
+ literals.emplace_back(*iter);
+ }
+ }
+ if (literals.empty()) {
+ continue;
+ }
+
+ std::shared_ptr<Predicate> predicate =
+ PredicateBuilder::In(offset_position, offset_name, FieldType::BIGINT, literals);
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<FileIndexResult> index_result,
+ FileIndexEvaluator::Evaluate(data_schema_, options_, predicate, path_factory_, file,
+ file_system_, memory_pool_));
+ PAIMON_ASSIGN_OR_RAISE(bool remains, index_result->IsRemain());
+ if (!remains) {
+ continue;
+ }
+ std::shared_ptr<BitmapIndexResult> bitmap =
+ std::dynamic_pointer_cast<BitmapIndexResult>(index_result);
+ if (!bitmap) {
+ return Status::Invalid(
+ fmt::format("_REALTIME_OFFSET index for data file {} is not an exact bitmap index",
+ file->file_name));
+ }
+ PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap32* positions, bitmap->GetBitmap());
+ if (!positions->IsEmpty()) {
+ result.emplace(file->file_name, *positions);
+ }
+ }
+ return result;
+}
+
+Result<std::shared_ptr<const KeyOffsetLookup>>
+RealtimeOffsetFileIndexLookup::CreateKeyOffsetLookup() const {
+ std::vector<std::shared_ptr<const KeyOffsetLookup>> readers;
+ readers.reserve(data_files_.size());
+ for (const std::shared_ptr<DataFileMeta>& file : data_files_) {
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<DataFileKeyOffsetIndexReader> reader,
+ DataFileKeyOffsetIndexReader::Create(deduplicate_key_field_, file, path_factory_,
+ file_system_, memory_pool_, options_));
+ readers.push_back(std::move(reader));
+ }
+ return std::shared_ptr<const KeyOffsetLookup>(new CompositeKeyOffsetLookup(std::move(readers)));
+}
+
+Result<std::shared_ptr<RealtimeOffsetFileIndexLookup>> RealtimeOffsetFileIndexLookup::WithDataFiles(
+ const std::vector<std::shared_ptr<DataFileMeta>>& data_files) const {
+ for (const std::shared_ptr<DataFileMeta>& file : data_files) {
+ if (!file || file->row_count < 0 || file->row_count > RoaringBitmap32::MAX_VALUE) {
+ return Status::Invalid("invalid data file for realtime offset lookup");
+ }
+ }
+ return std::shared_ptr<RealtimeOffsetFileIndexLookup>(
+ new RealtimeOffsetFileIndexLookup(data_schema_, data_schema_id_, deduplicate_key_field_,
+ std::vector<std::shared_ptr<DataFileMeta>>(data_files),
+ path_factory_, file_system_, memory_pool_, options_));
+}
+
+} // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_offset_file_index_lookup.h b/src/paimon/core/realtime/realtime_offset_file_index_lookup.h
new file mode 100644
index 00000000..fa214377
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_offset_file_index_lookup.h
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "paimon/result.h"
+#include "paimon/utils/roaring_bitmap32.h"
+#include "paimon/utils/roaring_bitmap64.h"
+
+namespace arrow {
+class Field;
+class Schema;
+} // namespace arrow
+
+namespace paimon {
+
+class CoreOptions;
+class DataFilePathFactory;
+class FileSystem;
+class KeyOffsetLookup;
+class MemoryPool;
+struct DataFileMeta;
+
+/// Uses the configured bitmap file index on `_REALTIME_OFFSET` to translate a pinned offset DV to
+/// physical row positions. Instances are immutable and therefore safe to pin for lock-free I/O.
+class RealtimeOffsetFileIndexLookup {
+ public:
+ static Result<std::shared_ptr<RealtimeOffsetFileIndexLookup>> Create(
+ const std::shared_ptr<arrow::Schema>& data_schema, int64_t data_schema_id,
+ const std::shared_ptr<arrow::Field>& deduplicate_key_field,
+ const std::vector<std::shared_ptr<DataFileMeta>>& data_files,
+ const std::shared_ptr<DataFilePathFactory>& path_factory,
+ const std::shared_ptr<FileSystem>& file_system,
+ const std::shared_ptr<MemoryPool>& memory_pool, const CoreOptions& options);
+
+ Result<std::map<std::string, RoaringBitmap32>> LookupFilePositions(
+ const RoaringBitmap64& offsets) const;
+
+ Result<std::shared_ptr<const KeyOffsetLookup>> CreateKeyOffsetLookup() const;
+
+ Result<std::shared_ptr<RealtimeOffsetFileIndexLookup>> WithDataFiles(
+ const std::vector<std::shared_ptr<DataFileMeta>>& data_files) const;
+
+ const std::vector<std::shared_ptr<DataFileMeta>>& DataFiles() const {
+ return data_files_;
+ }
+
+ private:
+ RealtimeOffsetFileIndexLookup(std::shared_ptr<arrow::Schema> data_schema,
+ int64_t data_schema_id,
+ std::shared_ptr<arrow::Field> deduplicate_key_field,
+ std::vector<std::shared_ptr<DataFileMeta>> data_files,
+ std::shared_ptr<DataFilePathFactory> path_factory,
+ std::shared_ptr<FileSystem> file_system,
+ std::shared_ptr<MemoryPool> memory_pool,
+ std::map<std::string, std::string> options);
+
+ std::shared_ptr<arrow::Schema> data_schema_;
+ int64_t data_schema_id_;
+ std::shared_ptr<arrow::Field> deduplicate_key_field_;
+ std::vector<std::shared_ptr<DataFileMeta>> data_files_;
+ std::shared_ptr<DataFilePathFactory> path_factory_;
+ std::shared_ptr<FileSystem> file_system_;
+ std::shared_ptr<MemoryPool> memory_pool_;
+ std::map<std::string, std::string> options_;
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_offset_file_index_lookup_test.cpp b/src/paimon/core/realtime/realtime_offset_file_index_lookup_test.cpp
new file mode 100644
index 00000000..5d35450c
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_offset_file_index_lookup_test.cpp
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/realtime/realtime_offset_file_index_lookup.h"
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/ipc/json_simple.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/io/data_file_index_writer.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/io/data_file_path_factory.h"
+#include "paimon/core/io/file_index_options.h"
+#include "paimon/core/realtime/realtime_key_offset_index.h"
+#include "paimon/defs.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/testing/utils/binary_row_generator.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+namespace {
+
+std::shared_ptr<DataFileMeta> MakeDataFile(const std::string& file_name, int64_t row_count) {
+ return std::make_shared<DataFileMeta>(
+ file_name, /*file_size=*/0, row_count, DataFileMeta::EmptyMinKey(),
+ DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+ /*min_sequence_number=*/0, /*max_sequence_number=*/0, /*schema_id=*/0,
+ DataFileMeta::DUMMY_LEVEL, /*extra_files=*/std::vector<std::optional<std::string>>(),
+ Timestamp(0, 0), /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr,
+ /*file_source=*/std::nullopt, /*value_stats_cols=*/std::nullopt,
+ /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+ /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
+}
+
+std::shared_ptr<arrow::StructArray> MakeKeys(const std::shared_ptr<arrow::Field>& key_field) {
+ return checked_pointer_cast<arrow::StructArray>(
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({key_field}), R"([[1], [2]])")
+ .ValueOrDie());
+}
+
+} // namespace
+
+class RealtimeOffsetFileIndexLookupTest : public testing::Test {
+ public:
+ void SetUp() override {
+ directory_ = UniqueTestDirectory::Create();
+ ASSERT_NE(nullptr, directory_);
+ path_factory_ = std::make_shared<DataFilePathFactory>();
+ ASSERT_OK(path_factory_->Init(directory_->Str(), /*format_identifier=*/"parquet",
+ /*data_file_prefix=*/"data-",
+ /*external_path_provider=*/nullptr));
+ key_field_ = arrow::field("id", arrow::int64());
+ schema_ = arrow::schema(
+ {DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()), key_field_});
+ options_ = {{"file-index.bitmap.columns", SpecialFields::RealtimeOffset().Name()},
+ {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1MB"}};
+ }
+
+ Result<std::shared_ptr<RealtimeOffsetFileIndexLookup>> CreateLookup(
+ const std::vector<std::shared_ptr<DataFileMeta>>& data_files,
+ int64_t data_schema_id = 0) const {
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions options,
+ CoreOptions::FromMap(options_, directory_->GetFileSystem()));
+ return RealtimeOffsetFileIndexLookup::Create(
+ schema_, data_schema_id, key_field_, data_files, path_factory_,
+ directory_->GetFileSystem(), GetDefaultPool(), options);
+ }
+
+ Result<std::shared_ptr<DataFileMeta>> CreateIndexedDataFile(
+ const std::string& file_name, const std::string& json, int64_t min_offset,
+ int64_t max_offset, int64_t schema_id = 0, bool name_value_stats = true) const {
+ std::shared_ptr<arrow::StructArray> batch = checked_pointer_cast<arrow::StructArray>(
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema_->fields()), json)
+ .ValueOrDie());
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions options,
+ CoreOptions::FromMap(options_, directory_->GetFileSystem()));
+ PAIMON_ASSIGN_OR_RAISE(FileIndexOptions file_index_options,
+ FileIndexOptions::FromCoreOptions(options));
+ PAIMON_ASSIGN_OR_RAISE(
+ std::unique_ptr<DataFileIndexWriter> writer,
+ DataFileIndexWriter::Create(schema_, file_index_options, directory_->GetFileSystem(),
+ path_factory_, GetDefaultPool()));
+ PAIMON_RETURN_NOT_OK(writer->AddBatch(batch));
+ PAIMON_ASSIGN_OR_RAISE(FileIndexWriteResult index,
+ writer->Finish(path_factory_->ToPath(file_name)));
+ if (!index.embedded_index || !index.extra_files.empty()) {
+ return Status::Invalid("test bitmap index was not embedded");
+ }
+ const SimpleStats value_stats = BinaryRowGenerator::GenerateStats(
+ {min_offset}, {max_offset}, {0}, GetDefaultPool().get());
+ const std::optional<std::vector<std::string>> value_stats_cols =
+ name_value_stats
+ ? std::optional<std::vector<std::string>>({SpecialFields::RealtimeOffset().Name()})
+ : std::nullopt;
+ return std::make_shared<DataFileMeta>(
+ file_name, /*file_size=*/0, batch->length(), DataFileMeta::EmptyMinKey(),
+ DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), value_stats,
+ /*min_sequence_number=*/0, /*max_sequence_number=*/0, schema_id,
+ DataFileMeta::DUMMY_LEVEL, index.extra_files, Timestamp(0, 0),
+ /*delete_row_count=*/std::nullopt, index.embedded_index,
+ /*file_source=*/std::nullopt, value_stats_cols,
+ /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+ /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt);
+ }
+
+ protected:
+ std::unique_ptr<UniqueTestDirectory> directory_;
+ std::shared_ptr<DataFilePathFactory> path_factory_;
+ std::shared_ptr<arrow::Field> key_field_;
+ std::shared_ptr<arrow::Schema> schema_;
+ std::map<std::string, std::string> options_;
+};
+
+TEST_F(RealtimeOffsetFileIndexLookupTest, TestLookupFilePositionsAcrossDataFiles) {
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<DataFileMeta> first,
+ CreateIndexedDataFile("data-0.parquet", R"([[10, 1], [12, 2], [14, 3], [15, 4]])",
+ /*min_offset=*/10, /*max_offset=*/15));
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<DataFileMeta> second,
+ CreateIndexedDataFile("data-1.parquet", R"([[20, 5], [21, 6], [22, 7], [23, 8]])",
+ /*min_offset=*/20, /*max_offset=*/23));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeOffsetFileIndexLookup> lookup,
+ CreateLookup({first, second}));
+
+ RoaringBitmap64 deleted_offsets;
+ deleted_offsets.Add(10);
+ deleted_offsets.Add(14);
+ deleted_offsets.Add(21);
+ deleted_offsets.Add(23);
+ deleted_offsets.Add(1000);
+ std::map<std::string, RoaringBitmap32> positions;
+ ASSERT_OK_AND_ASSIGN(positions, lookup->LookupFilePositions(deleted_offsets));
+ ASSERT_EQ(2, positions.size());
+ ASSERT_EQ("{0,2}", positions.at("data-0.parquet").ToString());
+ ASSERT_EQ("{1,3}", positions.at("data-1.parquet").ToString());
+
+ RoaringBitmap64 outside_file_stats;
+ outside_file_stats.Add(16);
+ outside_file_stats.Add(100);
+ ASSERT_OK_AND_ASSIGN(positions, lookup->LookupFilePositions(outside_file_stats));
+ ASSERT_TRUE(positions.empty());
+}
+
+TEST_F(RealtimeOffsetFileIndexLookupTest, TestCreateAndCloneDataFiles) {
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeOffsetFileIndexLookup> lookup,
+ CreateLookup(/*data_files=*/{}));
+ ASSERT_TRUE(lookup->DataFiles().empty());
+
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<const KeyOffsetLookup> key_lookup,
+ lookup->CreateKeyOffsetLookup());
+ ASSERT_OK_AND_ASSIGN(RoaringBitmap64 offsets, key_lookup->LookupOffsets(MakeKeys(key_field_)));
+ ASSERT_TRUE(offsets.IsEmpty());
+
+ std::shared_ptr<DataFileMeta> data_file = MakeDataFile("data-0.parquet", /*row_count=*/2);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeOffsetFileIndexLookup> cloned,
+ lookup->WithDataFiles({data_file}));
+ ASSERT_TRUE(lookup->DataFiles().empty());
+ ASSERT_EQ(1, cloned->DataFiles().size());
+ ASSERT_EQ(data_file, cloned->DataFiles()[0]);
+
+ ASSERT_NOK_WITH_MSG(lookup->WithDataFiles({nullptr}), "invalid data file");
+ ASSERT_NOK_WITH_MSG(lookup->WithDataFiles({MakeDataFile("negative.parquet", -1)}),
+ "invalid data file");
+ ASSERT_NOK_WITH_MSG(
+ lookup->WithDataFiles({MakeDataFile("too-large.parquet",
+ static_cast<int64_t>(RoaringBitmap32::MAX_VALUE) + 1)}),
+ "invalid data file");
+}
+
+TEST_F(RealtimeOffsetFileIndexLookupTest, TestDifferentSchemaDoesNotUseCurrentStatsPosition) {
+ // These unnamed stats represent another field at position 0 in the old schema. The current
+ // schema has _REALTIME_OFFSET at position 0, so interpreting them with the current layout would
+ // incorrectly prune offset 10 before consulting its bitmap index.
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<DataFileMeta> old_schema_file,
+ CreateIndexedDataFile("data-0.parquet", R"([[10, 1], [12, 2]])",
+ /*min_offset=*/100, /*max_offset=*/200,
+ /*schema_id=*/0, /*name_value_stats=*/false));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeOffsetFileIndexLookup> lookup,
+ CreateLookup({old_schema_file}, /*data_schema_id=*/1));
+
+ RoaringBitmap64 deleted_offsets;
+ deleted_offsets.Add(10);
+ std::map<std::string, RoaringBitmap32> positions;
+ ASSERT_OK_AND_ASSIGN(positions, lookup->LookupFilePositions(deleted_offsets));
+ ASSERT_EQ(1, positions.size());
+ ASSERT_EQ("{0}", positions.at("data-0.parquet").ToString());
+}
+
+TEST_F(RealtimeOffsetFileIndexLookupTest, TestCreateValidation) {
+ ASSERT_OK_AND_ASSIGN(CoreOptions valid_options,
+ CoreOptions::FromMap({{"file-index.bitmap.columns",
+ SpecialFields::RealtimeOffset().Name()}}));
+ ASSERT_NOK_WITH_MSG(RealtimeOffsetFileIndexLookup::Create(
+ schema_, /*data_schema_id=*/0, key_field_, {nullptr}, path_factory_,
+ directory_->GetFileSystem(), GetDefaultPool(), valid_options),
+ "invalid data file");
+ ASSERT_NOK_WITH_MSG(RealtimeOffsetFileIndexLookup::Create(
+ schema_, /*data_schema_id=*/0, key_field_, {}, nullptr,
+ directory_->GetFileSystem(), GetDefaultPool(), valid_options),
+ "missing a dependency");
+}
+
+} // namespace paimon::test
diff --git a/src/paimon/core/realtime/realtime_schema_layout.cpp b/src/paimon/core/realtime/realtime_schema_layout.cpp
index b67747f8..d95ef2be 100644
--- a/src/paimon/core/realtime/realtime_schema_layout.cpp
+++ b/src/paimon/core/realtime/realtime_schema_layout.cpp
@@ -24,6 +24,8 @@
#include "arrow/api.h"
#include "paimon/common/table/special_fields.h"
#include "paimon/common/types/data_field.h"
+#include "paimon/core/realtime/realtime_utils.h"
+#include "paimon/macros.h"
#include "paimon/status.h"
namespace paimon {
@@ -51,15 +53,21 @@ Result<std::unique_ptr<RealtimeSchemaLayout>> RealtimeSchemaLayout::Create(
if (!user_schema) {
return Status::Invalid("real-time schema layout requires a user schema");
}
- if (mode != RealtimeStoreMode::APPEND_ONLY && mode != RealtimeStoreMode::PRIMARY_KEY) {
+ if (mode != RealtimeStoreMode::APPEND_ONLY && mode != RealtimeStoreMode::PRIMARY_KEY &&
+ mode != RealtimeStoreMode::DEDUPLICATE) {
return Status::Invalid("unknown real-time store mode");
}
- return std::unique_ptr<RealtimeSchemaLayout>(new RealtimeSchemaLayout(mode, user_schema));
+ std::unique_ptr<RealtimeSchemaLayout> layout(new RealtimeSchemaLayout(mode, user_schema));
+ PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateOffsetField(layout->StoreWriteSchema()));
+ return layout;
}
RealtimeSchemaLayout::RealtimeSchemaLayout(RealtimeStoreMode mode,
const std::shared_ptr<arrow::Schema>& user_schema)
- : user_schema_(user_schema), input_schema_(Prepend({RealtimeOffsetField()}, user_schema_)) {
+ : user_schema_(user_schema),
+ input_schema_(mode == RealtimeStoreMode::DEDUPLICATE
+ ? user_schema_
+ : Prepend({RealtimeOffsetField()}, user_schema_)) {
if (mode == RealtimeStoreMode::APPEND_ONLY) {
store_write_schema_ = input_schema_;
store_commit_schema_ = store_write_schema_;
@@ -68,6 +76,16 @@ RealtimeSchemaLayout::RealtimeSchemaLayout(RealtimeStoreMode mode,
return;
}
+ if (mode == RealtimeStoreMode::DEDUPLICATE) {
+ // The offset is a real table column in this mode. Reuse the TableSchema field (including
+ // its field id and position) instead of synthesizing an internal prefix.
+ store_write_schema_ = user_schema_;
+ store_commit_schema_ = store_write_schema_;
+ commit_schema_ = user_schema_;
+ query_schema_ = Prepend({ValueKindField()}, user_schema_);
+ return;
+ }
+
store_write_schema_ = SpecialFields::CompleteSequenceAndValueKindField(input_schema_);
store_commit_schema_ = store_write_schema_;
commit_schema_ = SpecialFields::CompleteSequenceAndValueKindField(user_schema_);
diff --git a/src/paimon/core/realtime/realtime_schema_layout.h b/src/paimon/core/realtime/realtime_schema_layout.h
index 11668a30..3fd8ee88 100644
--- a/src/paimon/core/realtime/realtime_schema_layout.h
+++ b/src/paimon/core/realtime/realtime_schema_layout.h
@@ -31,7 +31,8 @@ namespace paimon {
class RealtimeSchemaLayout {
public:
/// Creates all real-time boundary schemas for the requested store mode.
- /// User fields retain their original order after any internal field prefix.
+ /// User fields retain their original order after any internal field prefix. DEDUPLICATE mode
+ /// requires `_REALTIME_OFFSET` to already exist in this schema as non-null int64.
static Result<std::unique_ptr<RealtimeSchemaLayout>> Create(
RealtimeStoreMode mode, const std::shared_ptr<arrow::Schema>& user_schema);
@@ -40,36 +41,42 @@ class RealtimeSchemaLayout {
/// Append-only fields: [user fields].
/// Primary-key fields: [user fields].
+ /// Deduplicate fields: [user fields, including _REALTIME_OFFSET].
const std::shared_ptr<arrow::Schema>& UserSchema() const {
return user_schema_;
}
/// Append-only fields: [_REALTIME_OFFSET, user fields].
/// Primary-key fields: [_REALTIME_OFFSET, user fields].
+ /// Deduplicate fields: [user fields, including _REALTIME_OFFSET].
const std::shared_ptr<arrow::Schema>& InputSchema() const {
return input_schema_;
}
/// Append-only fields: [_REALTIME_OFFSET, user fields].
/// Primary-key fields: [_SEQUENCE_NUMBER, _VALUE_KIND, _REALTIME_OFFSET, user fields].
+ /// Deduplicate fields: [user fields, including _REALTIME_OFFSET].
const std::shared_ptr<arrow::Schema>& StoreWriteSchema() const {
return store_write_schema_;
}
/// Append-only fields: [_REALTIME_OFFSET, user fields].
/// Primary-key fields: [_SEQUENCE_NUMBER, _VALUE_KIND, _REALTIME_OFFSET, user fields].
+ /// Deduplicate fields: [user fields, including _REALTIME_OFFSET].
const std::shared_ptr<arrow::Schema>& StoreCommitSchema() const {
return store_commit_schema_;
}
/// Append-only fields: [user fields].
/// Primary-key fields: [_SEQUENCE_NUMBER, _VALUE_KIND, user fields].
+ /// Deduplicate fields: [user fields, including _REALTIME_OFFSET].
const std::shared_ptr<arrow::Schema>& CommitSchema() const {
return commit_schema_;
}
/// Append-only fields: [_VALUE_KIND, user fields].
/// Primary-key fields: [_SEQUENCE_NUMBER, _VALUE_KIND, user fields].
+ /// Deduplicate fields: [_VALUE_KIND, user fields, including _REALTIME_OFFSET].
const std::shared_ptr<arrow::Schema>& QuerySchema() const {
return query_schema_;
}
diff --git a/src/paimon/core/realtime/realtime_schema_layout_test.cpp b/src/paimon/core/realtime/realtime_schema_layout_test.cpp
index 397102b7..8c0670e7 100644
--- a/src/paimon/core/realtime/realtime_schema_layout_test.cpp
+++ b/src/paimon/core/realtime/realtime_schema_layout_test.cpp
@@ -57,4 +57,36 @@ TEST(RealtimeSchemaLayoutTest, TestSchemaLayouts) {
ASSERT_TRUE(primary_key_layout->CommitSchema()->Equals(*primary_key_layout->QuerySchema()));
}
+TEST(RealtimeSchemaLayoutTest, TestDeduplicateSchemaKeepsOffsetFieldInUserOrder) {
+ std::shared_ptr<arrow::Schema> user_schema = arrow::schema({
+ arrow::field("key", arrow::int64(), false),
+ DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
+ arrow::field("value", arrow::utf8()),
+ });
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RealtimeSchemaLayout> layout,
+ RealtimeSchemaLayout::Create(RealtimeStoreMode::DEDUPLICATE, user_schema));
+
+ ASSERT_EQ(user_schema, layout->UserSchema());
+ ASSERT_EQ(user_schema, layout->InputSchema());
+ ASSERT_EQ(user_schema, layout->StoreWriteSchema());
+ ASSERT_EQ(user_schema, layout->StoreCommitSchema());
+ ASSERT_EQ(user_schema, layout->CommitSchema());
+ ASSERT_EQ((std::vector<std::string>{"_VALUE_KIND", "key", "_REALTIME_OFFSET", "value"}),
+ layout->QuerySchema()->field_names());
+}
+
+TEST(RealtimeSchemaLayoutTest, TestDeduplicateSchemaRequiresPersistedOffset) {
+ ASSERT_NOK_WITH_MSG(
+ RealtimeSchemaLayout::Create(RealtimeStoreMode::DEDUPLICATE,
+ arrow::schema({arrow::field("key", arrow::int64())})),
+ "requires non-null int64 _REALTIME_OFFSET");
+ ASSERT_NOK_WITH_MSG(
+ RealtimeSchemaLayout::Create(
+ RealtimeStoreMode::DEDUPLICATE,
+ arrow::schema({arrow::field(SpecialFields::RealtimeOffset().Name(), arrow::int64(),
+ /*nullable=*/true),
+ arrow::field("key", arrow::int64())})),
+ "requires non-null int64 _REALTIME_OFFSET");
+}
+
} // namespace paimon::test
diff --git a/src/paimon/core/realtime/realtime_store_read_pipeline.cpp b/src/paimon/core/realtime/realtime_store_read_pipeline.cpp
index 0aaccb01..6511a986 100644
--- a/src/paimon/core/realtime/realtime_store_read_pipeline.cpp
+++ b/src/paimon/core/realtime/realtime_store_read_pipeline.cpp
@@ -176,9 +176,13 @@ Result<std::unique_ptr<RealtimeStoreReadPipeline>> RealtimeStoreReadPipeline::Cr
}
}
+ const bool keep_realtime_offset =
+ logical_schema->GetFieldIndex(SpecialFields::RealtimeOffset().Name()) >= 0;
bool needs_conversion = !plans.empty();
// PK: _SEQUENCE_NUMBER, _VALUE_KIND, _REALTIME_OFFSET, then requested physical fields.
- // Append: _REALTIME_OFFSET, then requested physical fields.
+ // Append: _REALTIME_OFFSET, then requested physical fields. In DEDUPLICATE mode the offset is
+ // a normal user field: preserve its requested position, or inject it only as a hidden field
+ // when the query does not project it.
arrow::FieldVector store_read_fields;
store_read_fields.reserve(store_write_schema->num_fields());
for (const std::shared_ptr<arrow::Field>& write_field : store_write_schema->fields()) {
@@ -186,6 +190,15 @@ Result<std::unique_ptr<RealtimeStoreReadPipeline>> RealtimeStoreReadPipeline::Cr
store_read_fields.push_back(write_field);
}
}
+ if (!keep_realtime_offset) {
+ const std::shared_ptr<arrow::Field> offset_field =
+ store_write_schema->GetFieldByName(SpecialFields::RealtimeOffset().Name());
+ if (!offset_field) {
+ return Status::Invalid(
+ "real-time store write schema does not contain _REALTIME_OFFSET");
+ }
+ store_read_fields.push_back(offset_field);
+ }
for (const std::shared_ptr<arrow::Field>& read_field : logical_schema->fields()) {
if (SpecialFields::IsSystemField(read_field->name())) {
continue;
@@ -219,8 +232,9 @@ Result<std::unique_ptr<BatchReader>> RealtimeStoreReadPipeline::Wrap(
if (!store_reader) {
return Status::Invalid("real-time store read pipeline received a null reader");
}
- std::unique_ptr<BatchReader> reader =
- std::make_unique<RealtimeOffsetBatchReader>(std::move(store_reader), visible_offsets);
+ std::unique_ptr<BatchReader> reader = std::make_unique<RealtimeOffsetBatchReader>(
+ std::move(store_reader), visible_offsets,
+ logical_schema_->GetFieldIndex(SpecialFields::RealtimeOffset().Name()) >= 0);
if (!needs_conversion_) {
return std::move(reader);
}
diff --git a/src/paimon/core/realtime/realtime_utils.h b/src/paimon/core/realtime/realtime_utils.h
new file mode 100644
index 00000000..4a4ce60e
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_utils.h
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/api.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/io/file_index_options.h"
+#include "paimon/core/schema/arrow_schema_validator.h"
+#include "paimon/result.h"
+
+namespace paimon {
+
+class RealtimeUtils {
+ public:
+ RealtimeUtils() = delete;
+ ~RealtimeUtils() = delete;
+
+ static Result<int32_t> GetDeduplicateKeyPosition(
+ const std::shared_ptr<arrow::Schema>& schema,
+ const std::vector<std::string>& deduplicate_key_fields) {
+ if (!schema) {
+ return Status::Invalid("real-time deduplicate schema must not be null");
+ }
+ // TODO(xinyu.lxy): Support multiple user-defined deduplicate key fields. This requires
+ // composite-key projection and serialization in the configured field order.
+ if (deduplicate_key_fields.size() != 1) {
+ return Status::Invalid(
+ "real-time deduplicate currently requires exactly one deduplicate key field");
+ }
+ const int32_t key_position = schema->GetFieldIndex(deduplicate_key_fields[0]);
+ if (key_position < 0) {
+ return Status::Invalid("real-time deduplicate key field does not exist: ",
+ deduplicate_key_fields[0]);
+ }
+ return key_position;
+ }
+
+ static Status ValidateOffsetField(const std::shared_ptr<arrow::Schema>& schema) {
+ if (!schema) {
+ return Status::Invalid("real-time schema must not be null");
+ }
+ const std::shared_ptr<arrow::Field> offset_field =
+ schema->GetFieldByName(SpecialFields::RealtimeOffset().Name());
+ if (!offset_field || offset_field->type()->id() != arrow::Type::INT64 ||
+ offset_field->nullable()) {
+ return Status::Invalid("real-time schema requires non-null int64 _REALTIME_OFFSET");
+ }
+ return Status::OK();
+ }
+
+ static Status ValidateDeduplicateSchema(
+ const std::shared_ptr<arrow::Schema>& schema,
+ const std::vector<std::string>& deduplicate_key_fields) {
+ PAIMON_RETURN_NOT_OK(ValidateOffsetField(schema));
+ PAIMON_ASSIGN_OR_RAISE(int32_t key_position,
+ GetDeduplicateKeyPosition(schema, deduplicate_key_fields));
+ const std::shared_ptr<arrow::Field>& key_field = schema->field(key_position);
+ if (key_field->name() == SpecialFields::RealtimeOffset().Name()) {
+ return Status::Invalid("real-time offset field cannot be used as the deduplicate key");
+ }
+ if (key_field->nullable()) {
+ return Status::Invalid("real-time deduplicate key field must be non-null: ",
+ key_field->name());
+ }
+ if (ArrowSchemaValidator::IsNestedType(key_field->type())) {
+ return Status::Invalid("real-time deduplicate key field must not be nested: ",
+ key_field->name());
+ }
+ return Status::OK();
+ }
+
+ // TODO(xinyu.lxy): Relax the bitmap-only restriction after other exact file indexes can
+ // return row positions for realtime offsets.
+ static Status ValidateDeduplicateFileIndex(const CoreOptions& options) {
+ if (!options.FileIndexReadEnabled()) {
+ return Status::Invalid(
+ "real-time deduplicate mode requires 'file-index.read.enabled=true'");
+ }
+ PAIMON_ASSIGN_OR_RAISE(FileIndexOptions file_indexes,
+ FileIndexOptions::FromCoreOptions(options));
+ const std::string& offset_name = SpecialFields::RealtimeOffset().Name();
+ for (const FileIndexDefinition& definition : file_indexes.Definitions()) {
+ if (definition.column_name == offset_name && definition.index_type == "bitmap") {
+ return Status::OK();
+ }
+ }
+ return Status::Invalid(
+ "real-time deduplicate mode requires file-index.bitmap.columns to include ",
+ offset_name);
+ }
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/realtime/realtime_utils_test.cpp b/src/paimon/core/realtime/realtime_utils_test.cpp
new file mode 100644
index 00000000..685c5183
--- /dev/null
+++ b/src/paimon/core/realtime/realtime_utils_test.cpp
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/realtime/realtime_utils.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/api.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/defs.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+TEST(RealtimeUtilsTest, TestValidateOffsetField) {
+ const std::shared_ptr<arrow::Field> offset_field =
+ DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset());
+ ASSERT_OK(RealtimeUtils::ValidateOffsetField(
+ arrow::schema({arrow::field("id", arrow::int64()), offset_field})));
+
+ ASSERT_NOK_WITH_MSG(RealtimeUtils::ValidateOffsetField(nullptr), "schema must not be null");
+ ASSERT_NOK_WITH_MSG(
+ RealtimeUtils::ValidateOffsetField(arrow::schema({arrow::field("id", arrow::int64())})),
+ "requires non-null int64 _REALTIME_OFFSET");
+ ASSERT_NOK_WITH_MSG(
+ RealtimeUtils::ValidateOffsetField(arrow::schema({arrow::field(
+ SpecialFields::RealtimeOffset().Name(), arrow::int32(), /*nullable=*/false)})),
+ "requires non-null int64 _REALTIME_OFFSET");
+ ASSERT_NOK_WITH_MSG(
+ RealtimeUtils::ValidateOffsetField(arrow::schema({arrow::field(
+ SpecialFields::RealtimeOffset().Name(), arrow::int64(), /*nullable=*/true)})),
+ "requires non-null int64 _REALTIME_OFFSET");
+}
+
+TEST(RealtimeUtilsTest, TestGetDeduplicateKeyPosition) {
+ const std::shared_ptr<arrow::Schema> schema = arrow::schema({
+ DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset()),
+ arrow::field("id", arrow::int64()),
+ arrow::field("value", arrow::utf8()),
+ });
+ ASSERT_OK_AND_ASSIGN(int32_t position,
+ RealtimeUtils::GetDeduplicateKeyPosition(schema, {"id"}));
+ ASSERT_EQ(1, position);
+
+ ASSERT_NOK_WITH_MSG(RealtimeUtils::GetDeduplicateKeyPosition(nullptr, {"id"}),
+ "schema must not be null");
+ ASSERT_NOK_WITH_MSG(RealtimeUtils::GetDeduplicateKeyPosition(schema, {}),
+ "requires exactly one deduplicate key field");
+ ASSERT_NOK_WITH_MSG(RealtimeUtils::GetDeduplicateKeyPosition(schema, {"id", "value"}),
+ "requires exactly one deduplicate key field");
+ ASSERT_NOK_WITH_MSG(RealtimeUtils::GetDeduplicateKeyPosition(schema, {"missing"}),
+ "deduplicate key field does not exist");
+}
+
+TEST(RealtimeUtilsTest, TestValidateDeduplicateSchema) {
+ const std::shared_ptr<arrow::Field> offset_field =
+ DataField::ConvertDataFieldToArrowField(SpecialFields::RealtimeOffset());
+ ASSERT_OK(RealtimeUtils::ValidateDeduplicateSchema(
+ arrow::schema({arrow::field("id", arrow::int64(), /*nullable=*/false), offset_field}),
+ {"id"}));
+
+ ASSERT_NOK_WITH_MSG(
+ RealtimeUtils::ValidateDeduplicateSchema(
+ arrow::schema({arrow::field("id", arrow::int64(), /*nullable=*/true), offset_field}),
+ {"id"}),
+ "deduplicate key field must be non-null");
+ ASSERT_NOK_WITH_MSG(RealtimeUtils::ValidateDeduplicateSchema(arrow::schema({offset_field}),
+ {offset_field->name()}),
+ "offset field cannot be used as the deduplicate key");
+ ASSERT_NOK_WITH_MSG(
+ RealtimeUtils::ValidateDeduplicateSchema(
+ arrow::schema(
+ {arrow::field("id", arrow::struct_({arrow::field("nested", arrow::int64())}),
+ /*nullable=*/false),
+ offset_field}),
+ {"id"}),
+ "deduplicate key field must not be nested");
+}
+
+TEST(RealtimeUtilsTest, TestValidateDeduplicateFileIndex) {
+ ASSERT_OK_AND_ASSIGN(CoreOptions valid_options,
+ CoreOptions::FromMap({{"file-index.bitmap.columns",
+ SpecialFields::RealtimeOffset().Name()}}));
+ ASSERT_OK(RealtimeUtils::ValidateDeduplicateFileIndex(valid_options));
+
+ ASSERT_OK_AND_ASSIGN(CoreOptions missing_index_options, CoreOptions::FromMap({}));
+ ASSERT_NOK_WITH_MSG(RealtimeUtils::ValidateDeduplicateFileIndex(missing_index_options),
+ "requires file-index.bitmap.columns to include _REALTIME_OFFSET");
+
+ ASSERT_OK_AND_ASSIGN(CoreOptions wrong_column_options,
+ CoreOptions::FromMap({{"file-index.bitmap.columns", "id"}}));
+ ASSERT_NOK_WITH_MSG(RealtimeUtils::ValidateDeduplicateFileIndex(wrong_column_options),
+ "requires file-index.bitmap.columns to include _REALTIME_OFFSET");
+
+ ASSERT_OK_AND_ASSIGN(
+ CoreOptions disabled_read_options,
+ CoreOptions::FromMap({{"file-index.bitmap.columns", SpecialFields::RealtimeOffset().Name()},
+ {Options::FILE_INDEX_READ_ENABLED, "false"}}));
+ ASSERT_NOK_WITH_MSG(RealtimeUtils::ValidateDeduplicateFileIndex(disabled_read_options),
+ "requires 'file-index.read.enabled=true'");
+}
+
+} // namespace paimon::test
diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp
index 24c2649f..694a4e5c 100644
--- a/src/paimon/core/schema/schema_validation.cpp
+++ b/src/paimon/core/schema/schema_validation.cpp
@@ -61,6 +61,7 @@
#include "paimon/core/options/map_storage_layout.h"
#include "paimon/core/options/merge_engine.h"
#include "paimon/core/options/table_type.h"
+#include "paimon/core/realtime/realtime_utils.h"
#include "paimon/core/schema/arrow_schema_validator.h"
#include "paimon/core/schema/table_schema.h"
#include "paimon/core/table/bucket_mode.h"
@@ -299,6 +300,10 @@ Status SchemaValidation::ValidateFormatTableSchema(
// place. `target-file-row-num` is validated there, so it needs no check of its own here.
PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options,
CoreOptions::FromMap(effective_options, file_system));
+ if (core_options.RealtimeEnabled() || core_options.RealtimeSpillEnabled() ||
+ !core_options.GetRealtimeDeduplicateKeyFields().empty()) {
+ return Status::Invalid("a format table does not support real-time options");
+ }
// A format table has no branches: its data is the files under its one location, which is
// where a read or a write would go whatever this named. Refused wherever the option comes
@@ -358,6 +363,7 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) {
PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(schema.Options()));
PAIMON_RETURN_NOT_OK(ValidateBucket(schema, options));
+ PAIMON_RETURN_NOT_OK(ValidateRealtime(schema, options));
// PAIMON_RETURN_NOT_OK(ValidateDefaultValues(schema));
// PAIMON_RETURN_NOT_OK(ValidateStartupMode(options));
PAIMON_RETURN_NOT_OK(ValidateFieldsPrefix(schema, options));
@@ -620,6 +626,49 @@ Status SchemaValidation::ValidatePrimaryKeyBTreeIndexes(const TableSchema& schem
return Status::OK();
}
+Status SchemaValidation::ValidateRealtime(const TableSchema& schema, const CoreOptions& options) {
+ const std::vector<std::string>& deduplicate_key_fields =
+ options.GetRealtimeDeduplicateKeyFields();
+ if (!options.RealtimeEnabled()) {
+ PAIMON_RETURN_NOT_OK(Preconditions::CheckState(
+ deduplicate_key_fields.empty(), "'{}' requires '{}=true'.",
+ Options::REALTIME_DEDUPLICATE_KEY_FIELDS, Options::REALTIME_ENABLED));
+ return Preconditions::CheckState(
+ !options.RealtimeSpillEnabled(), "'{}' requires '{}=true'.",
+ Options::REALTIME_SPILL_ENABLED, Options::REALTIME_ENABLED);
+ }
+
+ PAIMON_RETURN_NOT_OK(Preconditions::CheckState(
+ options.GetBucket() > 0, "Real-time mode requires a fixed positive bucket count."));
+ PAIMON_RETURN_NOT_OK(Preconditions::CheckState(
+ !options.DataEvolutionEnabled(), "Real-time mode does not support data evolution."));
+
+ if (deduplicate_key_fields.empty()) {
+ return Status::OK();
+ }
+
+ PAIMON_RETURN_NOT_OK(
+ Preconditions::CheckState(schema.PrimaryKeys().empty(),
+ "'{}' is only supported on append tables without primary keys.",
+ Options::REALTIME_DEDUPLICATE_KEY_FIELDS));
+ PAIMON_RETURN_NOT_OK(Preconditions::CheckState(
+ options.DeletionVectorsEnabled(),
+ "Real-time deduplicate mode requires 'deletion-vectors.enabled=true'."));
+ // Bitmap64 deletion vectors bypass the exact file-index visibility filter.
+ PAIMON_RETURN_NOT_OK(Preconditions::CheckState(
+ !options.DeletionVectorsBitmap64(),
+ "Real-time deduplicate mode does not support 'deletion-vectors.bitmap64=true'."));
+ PAIMON_RETURN_NOT_OK(Preconditions::CheckState(
+ !options.RealtimeSpillEnabled(),
+ "Real-time deduplicate mode does not support 'realtime.spill-enabled=true'."));
+
+ const std::shared_ptr<arrow::Schema> arrow_schema =
+ DataField::ConvertDataFieldsToArrowSchema(schema.Fields());
+ PAIMON_RETURN_NOT_OK(
+ RealtimeUtils::ValidateDeduplicateSchema(arrow_schema, deduplicate_key_fields));
+ return RealtimeUtils::ValidateDeduplicateFileIndex(options);
+}
+
Status SchemaValidation::ValidateSequenceGroup(const TableSchema& schema,
const CoreOptions& options) {
std::unordered_map<std::string, std::set<std::string>> fields2_group;
diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h
index c89b8019..8c1e41f3 100644
--- a/src/paimon/core/schema/schema_validation.h
+++ b/src/paimon/core/schema/schema_validation.h
@@ -51,6 +51,10 @@ class SchemaValidation {
static Status ValidateTableSchema(const TableSchema& schema);
+ /// Validates real-time options after operation-specific options have been merged with the
+ /// table schema options.
+ static Status ValidateRealtime(const TableSchema& schema, const CoreOptions& options);
+
/// Validates a schema a table is about to be created from, whichever catalog is creating it.
///
/// It picks the rules by the schema's `type` option: a format table owns no bucket, manifest
@@ -124,7 +128,6 @@ class SchemaValidation {
static Status ValidateForDeletionVectors(const CoreOptions& options);
static Status ValidatePrimaryKeyBTreeIndexes(const TableSchema& schema,
const CoreOptions& options);
-
static Status ValidateRowTracking(const TableSchema& table_schema, const CoreOptions& options);
static Status ValidateBlobFields(const TableSchema& schema, const CoreOptions& options);
diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp
index 9356b0be..d25aa0fa 100644
--- a/src/paimon/core/schema/schema_validation_test.cpp
+++ b/src/paimon/core/schema/schema_validation_test.cpp
@@ -27,6 +27,7 @@
#include "gtest/gtest.h"
#include "paimon/common/data/blob_utils.h"
#include "paimon/common/data/variant/variant_type_utils.h"
+#include "paimon/common/table/special_fields.h"
#include "paimon/common/utils/checked_cast.h"
#include "paimon/core/schema/table_schema.h"
#include "paimon/defs.h"
@@ -82,6 +83,15 @@ Result<std::unique_ptr<TableSchema>> MakeContainerBlobSchema(const std::string&
} // namespace
+Status CreateAndValidateTableSchema(const std::shared_ptr<arrow::Schema>& arrow_schema,
+ const std::vector<std::string>& primary_keys,
+ const std::map<std::string, std::string>& options) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<TableSchema> table_schema,
+ TableSchema::Create(/*schema_id=*/0, arrow_schema, /*partition_keys=*/{},
+ primary_keys, options));
+ return SchemaValidation::ValidateTableSchema(*table_schema);
+}
+
TEST(SchemaValidationTest, TestSimple) {
auto f0 = arrow::field("f0", arrow::utf8());
auto f1 = arrow::field("f1", arrow::int32());
@@ -98,12 +108,111 @@ TEST(SchemaValidationTest, TestSimple) {
ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
}
-TEST(SchemaValidationTest, TestRealtimeOffsetIsGloballyReserved) {
- auto schema = arrow::schema({arrow::field("_REALTIME_OFFSET", arrow::int64())});
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> table_schema,
- TableSchema::Create(0, schema, {}, {}, {}));
- ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
- "field name '_REALTIME_OFFSET' in schema cannot be special field");
+TEST(SchemaValidationTest, TestRealtime) {
+ const std::shared_ptr<arrow::Schema> schema =
+ arrow::schema({arrow::field("id", arrow::int64(), /*nullable=*/false)});
+
+ const std::map<std::string, std::string> valid_options = {
+ {Options::BUCKET, "1"},
+ {Options::BUCKET_KEY, "id"},
+ {Options::REALTIME_ENABLED, "true"},
+ };
+ ASSERT_OK(CreateAndValidateTableSchema(schema, /*primary_keys=*/{}, valid_options));
+ ASSERT_OK(CreateAndValidateTableSchema(schema, /*primary_keys=*/{"id"}, valid_options));
+
+ std::map<std::string, std::string> deletion_vector_options = valid_options;
+ deletion_vector_options[Options::DELETION_VECTORS_ENABLED] = "true";
+ ASSERT_OK(CreateAndValidateTableSchema(schema, /*primary_keys=*/{}, deletion_vector_options));
+ deletion_vector_options[Options::DELETION_VECTOR_BITMAP64] = "true";
+ ASSERT_OK(CreateAndValidateTableSchema(schema, /*primary_keys=*/{}, deletion_vector_options));
+
+ std::map<std::string, std::string> spill_options = valid_options;
+ spill_options[Options::REALTIME_SPILL_ENABLED] = "true";
+ ASSERT_OK(CreateAndValidateTableSchema(schema, /*primary_keys=*/{}, spill_options));
+
+ std::map<std::string, std::string> deduplicate_without_realtime = valid_options;
+ deduplicate_without_realtime.erase(Options::REALTIME_ENABLED);
+ deduplicate_without_realtime[Options::REALTIME_DEDUPLICATE_KEY_FIELDS] = "id";
+ ASSERT_NOK_WITH_MSG(
+ CreateAndValidateTableSchema(schema, /*primary_keys=*/{}, deduplicate_without_realtime),
+ "realtime.deduplicate-key-fields' requires 'realtime.enabled=true");
+
+ std::map<std::string, std::string> spill_without_realtime = valid_options;
+ spill_without_realtime.erase(Options::REALTIME_ENABLED);
+ spill_without_realtime[Options::REALTIME_SPILL_ENABLED] = "true";
+ ASSERT_NOK_WITH_MSG(
+ CreateAndValidateTableSchema(schema, /*primary_keys=*/{}, spill_without_realtime),
+ "realtime.spill-enabled' requires 'realtime.enabled=true");
+
+ std::map<std::string, std::string> dynamic_bucket_options = valid_options;
+ dynamic_bucket_options[Options::BUCKET] = "-1";
+ dynamic_bucket_options.erase(Options::BUCKET_KEY);
+ ASSERT_NOK_WITH_MSG(
+ CreateAndValidateTableSchema(schema, /*primary_keys=*/{}, dynamic_bucket_options),
+ "Real-time mode requires a fixed positive bucket count");
+
+ std::map<std::string, std::string> data_evolution_options = valid_options;
+ data_evolution_options[Options::DATA_EVOLUTION_ENABLED] = "true";
+ ASSERT_NOK_WITH_MSG(
+ CreateAndValidateTableSchema(schema, /*primary_keys=*/{}, data_evolution_options),
+ "Real-time mode does not support data evolution");
+}
+
+TEST(SchemaValidationTest, TestRealtimeDeduplicate) {
+ const std::shared_ptr<arrow::Field> offset_field =
+ arrow::field(SpecialFields::RealtimeOffset().Name(), arrow::int64(), /*nullable=*/false);
+ const std::shared_ptr<arrow::Schema> schema =
+ arrow::schema({arrow::field("id", arrow::int64(), /*nullable=*/false), offset_field});
+
+ const std::map<std::string, std::string> valid_options = {
+ {Options::BUCKET, "1"},
+ {Options::BUCKET_KEY, "id"},
+ {Options::REALTIME_ENABLED, "true"},
+ {Options::REALTIME_DEDUPLICATE_KEY_FIELDS, "id"},
+ {Options::DELETION_VECTORS_ENABLED, "true"},
+ {"file-index.bitmap.columns", SpecialFields::RealtimeOffset().Name()},
+ };
+ ASSERT_OK(CreateAndValidateTableSchema(schema, /*primary_keys=*/{}, valid_options));
+
+ ASSERT_NOK_WITH_MSG(
+ CreateAndValidateTableSchema(schema, /*primary_keys=*/{"id"}, valid_options),
+ "is only supported on append tables without primary keys");
+
+ std::map<std::string, std::string> no_deletion_vector_options = valid_options;
+ no_deletion_vector_options.erase(Options::DELETION_VECTORS_ENABLED);
+ ASSERT_NOK_WITH_MSG(
+ CreateAndValidateTableSchema(schema, /*primary_keys=*/{}, no_deletion_vector_options),
+ "requires 'deletion-vectors.enabled=true'");
+
+ std::map<std::string, std::string> spill_options = valid_options;
+ spill_options[Options::REALTIME_SPILL_ENABLED] = "true";
+ ASSERT_NOK_WITH_MSG(CreateAndValidateTableSchema(schema, /*primary_keys=*/{}, spill_options),
+ "does not support 'realtime.spill-enabled=true'");
+
+ std::map<std::string, std::string> bitmap64_options = valid_options;
+ bitmap64_options[Options::DELETION_VECTOR_BITMAP64] = "true";
+ ASSERT_NOK_WITH_MSG(CreateAndValidateTableSchema(schema, /*primary_keys=*/{}, bitmap64_options),
+ "does not support 'deletion-vectors.bitmap64=true'");
+ bitmap64_options[Options::DELETION_VECTOR_BITMAP64] = "false";
+ ASSERT_OK(CreateAndValidateTableSchema(schema, /*primary_keys=*/{}, bitmap64_options));
+
+ const std::shared_ptr<arrow::Schema> no_offset_schema =
+ arrow::schema({arrow::field("id", arrow::int64(), /*nullable=*/false)});
+ ASSERT_NOK_WITH_MSG(
+ CreateAndValidateTableSchema(no_offset_schema, /*primary_keys=*/{}, valid_options),
+ "requires non-null int64 _REALTIME_OFFSET");
+
+ std::map<std::string, std::string> no_offset_index_options = valid_options;
+ no_offset_index_options.erase("file-index.bitmap.columns");
+ ASSERT_NOK_WITH_MSG(
+ CreateAndValidateTableSchema(schema, /*primary_keys=*/{}, no_offset_index_options),
+ "requires file-index.bitmap.columns to include _REALTIME_OFFSET");
+
+ std::map<std::string, std::string> disabled_file_index_read_options = valid_options;
+ disabled_file_index_read_options[Options::FILE_INDEX_READ_ENABLED] = "false";
+ ASSERT_NOK_WITH_MSG(
+ CreateAndValidateTableSchema(schema, /*primary_keys=*/{}, disabled_file_index_read_options),
+ "requires 'file-index.read.enabled=true'");
}
TEST(SchemaValidationTest, TestVectorType) {
diff --git a/src/paimon/core/table/source/append_only_table_read.cpp b/src/paimon/core/table/source/append_only_table_read.cpp
index c7a3350e..2e9cdb99 100644
--- a/src/paimon/core/table/source/append_only_table_read.cpp
+++ b/src/paimon/core/table/source/append_only_table_read.cpp
@@ -19,7 +19,6 @@
#include "paimon/core/table/source/append_only_table_read.h"
-#include <map>
#include <optional>
#include <string>
#include <utility>
@@ -31,10 +30,15 @@
#include "paimon/common/reader/complete_row_kind_batch_reader.h"
#include "paimon/common/reader/concat_batch_reader.h"
#include "paimon/common/reader/predicate_batch_reader.h"
+#include "paimon/common/table/special_fields.h"
#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/arrow/arrow_utils.h"
#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
#include "paimon/common/utils/scope_guard.h"
#include "paimon/core/core_options.h"
+#include "paimon/core/deletionvectors/deletion_vector.h"
+#include "paimon/core/global_index/indexed_split_impl.h"
#include "paimon/core/operation/data_evolution_split_read.h"
#include "paimon/core/operation/internal_read_context.h"
#include "paimon/core/operation/raw_file_split_read.h"
@@ -43,11 +47,15 @@
#include "paimon/core/realtime/realtime_schema_layout.h"
#include "paimon/core/realtime/realtime_store_read_pipeline.h"
#include "paimon/core/table/source/append_count_reader.h"
+#include "paimon/core/table/source/data_split_impl.h"
#include "paimon/core/table/source/realtime_split.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate_builder.h"
#include "paimon/predicate/predicate_utils.h"
#include "paimon/realtime/realtime_context.h"
#include "paimon/realtime/realtime_store.h"
#include "paimon/status.h"
+#include "paimon/utils/roaring_bitmap64.h"
namespace paimon {
class DataSplit;
@@ -55,6 +63,57 @@ class Executor;
class FileStorePathFactory;
class MemoryPool;
+namespace {
+
+class RemoveRealtimeOffsetBatchReader final : public BatchReader {
+ public:
+ explicit RemoveRealtimeOffsetBatchReader(std::unique_ptr<BatchReader>&& reader)
+ : reader_(std::move(reader)) {}
+
+ Result<ReadBatch> NextBatch() override {
+ PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, reader_->NextBatch());
+ PAIMON_RETURN_NOT_OK(RemoveOffset(&batch));
+ return batch;
+ }
+
+ Result<ReadBatchWithBitmap> NextBatchWithBitmap() override {
+ PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch, reader_->NextBatchWithBitmap());
+ PAIMON_RETURN_NOT_OK(RemoveOffset(&batch.first));
+ return batch;
+ }
+
+ std::shared_ptr<Metrics> GetReaderMetrics() const override {
+ return reader_->GetReaderMetrics();
+ }
+
+ void Close() override {
+ reader_->Close();
+ }
+
+ private:
+ Status RemoveOffset(ReadBatch* batch) const {
+ if (IsEofBatch(*batch)) {
+ return Status::OK();
+ }
+ auto& [c_array, c_schema] = *batch;
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
+ arrow::ImportArray(c_array.get(), c_schema.get()));
+ if (!array || array->type_id() != arrow::Type::STRUCT) {
+ return Status::Invalid("deduplicate disk reader must return a StructArray");
+ }
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<arrow::StructArray> output,
+ ArrowUtils::RemoveFieldFromStructArray(checked_pointer_cast<arrow::StructArray>(array),
+ SpecialFields::RealtimeOffset().Name()));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*output, c_array.get(), c_schema.get()));
+ return Status::OK();
+ }
+
+ std::unique_ptr<BatchReader> reader_;
+};
+
+} // namespace
+
AppendOnlyTableRead::AppendOnlyTableRead(const std::shared_ptr<FileStorePathFactory>& path_factory,
const std::shared_ptr<InternalReadContext>& context,
const std::shared_ptr<MemoryPool>& memory_pool,
@@ -155,16 +214,20 @@ Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateRealtimeReader(
}
for (const std::shared_ptr<Split>& disk_split : realtime_split->DiskSplits()) {
- PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> disk_reader,
- CreateDiskReader(disk_split));
+ std::unique_ptr<BatchReader> disk_reader;
+ if (memory.mode == RealtimeStoreMode::DEDUPLICATE) {
+ PAIMON_ASSIGN_OR_RAISE(
+ disk_reader, CreateDeduplicateDiskReader(disk_split, memory.offset_deletions));
+ } else {
+ PAIMON_ASSIGN_OR_RAISE(disk_reader, CreateDiskReader(disk_split));
+ }
readers.push_back(std::move(disk_reader));
}
std::shared_ptr<arrow::Schema> table_schema =
DataField::ConvertDataFieldsToArrowSchema(context_->GetTableSchema()->Fields());
- PAIMON_ASSIGN_OR_RAISE(
- std::unique_ptr<RealtimeSchemaLayout> schema_layout,
- RealtimeSchemaLayout::Create(RealtimeStoreMode::APPEND_ONLY, table_schema));
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RealtimeSchemaLayout> schema_layout,
+ RealtimeSchemaLayout::Create(memory.mode, table_schema));
PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RealtimeStoreReadPipeline> pipeline,
RealtimeStoreReadPipeline::Create(
context_->GetReadSchema(), *schema_layout, context_->GetMemoryPool(),
@@ -232,6 +295,85 @@ Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateDiskReader(
return Status::Invalid("create reader failed, not read match with split.");
}
+Result<std::unique_ptr<BatchReader>> AppendOnlyTableRead::CreateDeduplicateDiskReader(
+ const std::shared_ptr<Split>& split,
+ const std::shared_ptr<const RoaringBitmap64>& offset_deletions) {
+ if (!offset_deletions) {
+ return Status::Invalid("deduplicate real-time split does not pin an offset DV");
+ }
+ auto raw_read = dynamic_cast<RawFileSplitRead*>(split_reads_.front().get());
+ if (!raw_read) {
+ return Status::NotImplemented(
+ "deduplicate real-time offset-DV overlay only supports raw append reads");
+ }
+
+ std::shared_ptr<DataSplitImpl> data_split;
+ std::optional<std::vector<Range>> row_ranges;
+ if (auto indexed_split = std::dynamic_pointer_cast<IndexedSplitImpl>(split)) {
+ PAIMON_RETURN_NOT_OK(indexed_split->Validate());
+ if (!indexed_split->Scores().empty()) {
+ return Status::NotImplemented(
+ "deduplicate real-time reads do not support scored indexed splits yet");
+ }
+ data_split = std::dynamic_pointer_cast<DataSplitImpl>(indexed_split->GetDataSplit());
+ row_ranges = indexed_split->RowRanges();
+ } else {
+ data_split = std::dynamic_pointer_cast<DataSplitImpl>(split);
+ }
+ if (!data_split) {
+ return Status::Invalid("deduplicate real-time disk split is not a data split");
+ }
+
+ if (offset_deletions->IsEmpty()) {
+ return raw_read->CreateReader(data_split->Partition(), data_split->Bucket(),
+ data_split->DataFiles(), data_split->DeletionFiles(),
+ row_ranges);
+ }
+
+ std::shared_ptr<arrow::Schema> read_schema = context_->GetReadSchema();
+ const std::string& offset_name = SpecialFields::RealtimeOffset().Name();
+ const bool expose_offset = read_schema->GetFieldIndex(offset_name) >= 0;
+ if (!expose_offset) {
+ arrow::FieldVector fields = read_schema->fields();
+ PAIMON_ASSIGN_OR_RAISE(DataField offset_field,
+ context_->GetTableSchema()->GetField(offset_name));
+ fields.push_back(DataField::ConvertDataFieldToArrowField(offset_field));
+ read_schema = arrow::schema(std::move(fields), read_schema->metadata());
+ }
+
+ // TODO(xinyu.lxy): Build a file-specific visibility predicate for each data file. Use the
+ // `_REALTIME_OFFSET` min/max statistics in DataFileMeta to remove deleted offsets that cannot
+ // intersect that file, instead of pushing the complete bucket offset DV to every file.
+ std::vector<Literal> deleted_offsets;
+ deleted_offsets.reserve(offset_deletions->Cardinality());
+ for (RoaringBitmap64::Iterator offset = offset_deletions->Begin();
+ offset != offset_deletions->End(); ++offset) {
+ deleted_offsets.emplace_back(static_cast<int64_t>(*offset));
+ }
+ const int32_t offset_position = read_schema->GetFieldIndex(offset_name);
+ std::shared_ptr<Predicate> visibility_predicate =
+ PredicateBuilder::NotIn(offset_position, offset_name, FieldType::BIGINT, deleted_offsets);
+ std::shared_ptr<Predicate> predicate = visibility_predicate;
+ if (context_->GetPredicate()) {
+ PAIMON_ASSIGN_OR_RAISE(predicate, PredicateBuilder::And({context_->GetPredicate(),
+ std::move(visibility_predicate)}));
+ }
+
+ DeletionVector::Factory committed_factory = DeletionVector::CreateFactory(
+ context_->GetCoreOptions().GetFileSystem(),
+ DeletionVector::CreateDeletionFileMap(data_split->DataFiles(), data_split->DeletionFiles()),
+ context_->GetMemoryPool());
+ PAIMON_ASSIGN_OR_RAISE(
+ std::unique_ptr<BatchReader> reader,
+ raw_read->CreateReader(data_split->Partition(), data_split->Bucket(),
+ data_split->DataFiles(), std::move(committed_factory), row_ranges,
+ read_schema, predicate));
+ if (!expose_offset) {
+ reader = std::make_unique<RemoveRealtimeOffsetBatchReader>(std::move(reader));
+ }
+ return reader;
+}
+
Result<std::unique_ptr<CountReader>> AppendOnlyTableRead::CreateCountReader(
const std::vector<std::shared_ptr<Split>>& splits) {
for (const std::shared_ptr<Split>& split : splits) {
diff --git a/src/paimon/core/table/source/append_only_table_read.h b/src/paimon/core/table/source/append_only_table_read.h
index 79eacbc1..0999774c 100644
--- a/src/paimon/core/table/source/append_only_table_read.h
+++ b/src/paimon/core/table/source/append_only_table_read.h
@@ -37,6 +37,7 @@ class FileStorePathFactory;
class InternalReadContext;
class MemoryPool;
class RealtimeSplit;
+class RoaringBitmap64;
class AppendOnlyTableRead : public TableRead {
public:
@@ -60,6 +61,10 @@ class AppendOnlyTableRead : public TableRead {
Result<std::unique_ptr<BatchReader>> CreateDiskReader(const std::shared_ptr<Split>& split);
+ Result<std::unique_ptr<BatchReader>> CreateDeduplicateDiskReader(
+ const std::shared_ptr<Split>& split,
+ const std::shared_ptr<const RoaringBitmap64>& offset_deletions);
+
std::vector<std::unique_ptr<SplitRead>> split_reads_;
std::shared_ptr<InternalReadContext> context_;
};
diff --git a/src/paimon/core/table/source/realtime_table_scan.cpp b/src/paimon/core/table/source/realtime_table_scan.cpp
index 5cfb26c6..42a7171b 100644
--- a/src/paimon/core/table/source/realtime_table_scan.cpp
+++ b/src/paimon/core/table/source/realtime_table_scan.cpp
@@ -179,7 +179,7 @@ Result<std::vector<std::shared_ptr<Split>>> RealtimeTableScan::CreateRealtimeSpl
}
RealtimePartitionBucketView& memory = memory_iter->second;
- if (!pk_table_) {
+ if (!pk_table_ && memory.mode != RealtimeStoreMode::DEDUPLICATE) {
// Append tables can schedule all but the tail disk split independently. The tail split
// carries the immutable memory view so disk and memory are still concatenated by one
// RealtimeSplit without collapsing the whole partition-bucket into one scheduling unit.
diff --git a/src/paimon/core/table/source/table_read.cpp b/src/paimon/core/table/source/table_read.cpp
index 77e20120..7dd10e36 100644
--- a/src/paimon/core/table/source/table_read.cpp
+++ b/src/paimon/core/table/source/table_read.cpp
@@ -35,6 +35,7 @@
#include "paimon/core/core_options.h"
#include "paimon/core/operation/internal_read_context.h"
#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/schema_validation.h"
#include "paimon/core/schema/table_schema.h"
#include "paimon/core/table/format/format_table_loader.h"
#include "paimon/core/table/format/format_table_read.h"
@@ -105,6 +106,7 @@ Result<std::unique_ptr<TableRead>> CreateTableRead(
if (internal_context->GetRealtimeContext() && !core_options.RealtimeEnabled()) {
return Status::Invalid("real-time read requires realtime.enabled=true");
}
+ PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options));
auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields());
PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> external_paths,
core_options.CreateExternalPaths());
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index 7ede285f..21d338f2 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -381,6 +381,8 @@ Result<std::unique_ptr<TableScan>> NewDataTableScan(
CoreOptions::FromMap(options, context->GetSpecificFileSystem(), {}));
core_options.WithCache(context->GetCache());
+ PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options));
+
PAIMON_RETURN_NOT_OK(
ValidateRealtimeScan(*table_schema, core_options, *context, read_optimized));
// validate options
diff --git a/src/paimon/core/utils/commit_increment.h b/src/paimon/core/utils/commit_increment.h
index 51bfef1c..dbb1bf2b 100644
--- a/src/paimon/core/utils/commit_increment.h
+++ b/src/paimon/core/utils/commit_increment.h
@@ -21,6 +21,7 @@
#include <memory>
#include <optional>
+#include <utility>
#include "paimon/core/compact/compact_deletion_file.h"
#include "paimon/core/io/compact_increment.h"
@@ -29,6 +30,8 @@
namespace paimon {
+class RoaringBitmap64;
+
// Changes to commit.
class CommitIncrement {
public:
@@ -66,11 +69,20 @@ class CommitIncrement {
realtime_offset_range_ = offset_range;
}
+ const std::shared_ptr<const RoaringBitmap64>& GetRealtimeDeletedOffsets() const {
+ return realtime_deleted_offsets_;
+ }
+
+ void SetRealtimeDeletedOffsets(std::shared_ptr<const RoaringBitmap64> deleted_offsets) {
+ realtime_deleted_offsets_ = std::move(deleted_offsets);
+ }
+
private:
DataIncrement data_increment_;
CompactIncrement compact_increment_;
std::shared_ptr<CompactDeletionFile> compact_deletion_file_;
std::optional<OffsetRange> realtime_offset_range_;
+ std::shared_ptr<const RoaringBitmap64> realtime_deleted_offsets_;
};
} // namespace paimon
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index ee598ea4..074dc9b9 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -50,6 +50,8 @@
#include "paimon/common/utils/path_util.h"
#include "paimon/common/utils/scope_guard.h"
#include "paimon/core/core_options.h"
+#include "paimon/core/deletionvectors/bitmap_deletion_vector.h"
+#include "paimon/core/deletionvectors/deletion_vectors_index_file.h"
#include "paimon/core/operation/commit/realtime_commit_properties.h"
#include "paimon/core/operation/metrics/commit_metrics.h"
#include "paimon/core/realtime/realtime_context_impl.h"
@@ -82,6 +84,7 @@
#include "paimon/scan_context.h"
#include "paimon/table/source/table_read.h"
#include "paimon/table/source/table_scan.h"
+#include "paimon/testing/mock/mock_index_path_factory.h"
#include "paimon/testing/utils/read_result_collector.h"
#include "paimon/testing/utils/test_helper.h"
#include "paimon/testing/utils/testharness.h"
@@ -215,6 +218,26 @@ class QueryTrackingRealtimeStore final : public DelegatingRealtimeStore {
std::shared_ptr<std::weak_ptr<RealtimeReadView>> query_view_;
};
+// Publishes a competing snapshot after the attempt has already materialized its DVs.
+class InterleavingSnapshotFileSystem final : public LocalFileSystem {
+ public:
+ InterleavingSnapshotFileSystem(std::string snapshot_path, std::function<Status()> interleave)
+ : snapshot_path_(std::move(snapshot_path)), interleave_(std::move(interleave)) {}
+
+ Status AtomicStore(const std::string& path, const std::string& content) override {
+ if (path == snapshot_path_ && interleave_) {
+ std::function<Status()> interleave = std::move(interleave_);
+ PAIMON_RETURN_NOT_OK(interleave());
+ return Status::IOError("injected snapshot publication conflict");
+ }
+ return FileSystem::AtomicStore(path, content);
+ }
+
+ private:
+ std::string snapshot_path_;
+ std::function<Status()> interleave_;
+};
+
class SnapshotConflictFileSystem final : public LocalFileSystem {
public:
explicit SnapshotConflictFileSystem(std::string conflict_snapshot_path)
@@ -503,6 +526,22 @@ class RealtimeWriteInteTest : public ::testing::Test {
table_primary_keys, options_, /*ignore_if_exists=*/false));
}
+ void ConfigureRealtimeDeduplicateTable(const std::vector<std::string>& partition_keys = {}) {
+ fields_ = {
+ arrow::field(SpecialFields::RealtimeOffset().Name(), arrow::int64(),
+ /*nullable=*/false),
+ arrow::field("id", arrow::int64(), /*nullable=*/false),
+ arrow::field("payload", arrow::utf8()),
+ arrow::field("pt", arrow::utf8()),
+ };
+ schema_ = arrow::schema(fields_);
+ options_[Options::FILE_FORMAT] = "parquet";
+ options_[Options::DELETION_VECTORS_ENABLED] = "true";
+ options_[Options::REALTIME_DEDUPLICATE_KEY_FIELDS] = "id";
+ options_["file-index.bitmap.columns"] = SpecialFields::RealtimeOffset().Name();
+ CreateTable(partition_keys);
+ }
+
Result<std::unique_ptr<FileStoreWrite>> CreateRealtimeWriter(
const std::shared_ptr<RealtimeContext>& realtime_context) const {
return CreateRealtimeWriter(realtime_context, /*temp_directory=*/"");
@@ -588,9 +627,12 @@ class RealtimeWriteInteTest : public ::testing::Test {
}
json += "]";
- PAIMON_ASSIGN_OR_RAISE(
- std::unique_ptr<RealtimeSchemaLayout> schema_layout,
- RealtimeSchemaLayout::Create(RealtimeStoreMode::APPEND_ONLY, schema_));
+ const RealtimeStoreMode store_mode =
+ options_.count(Options::REALTIME_DEDUPLICATE_KEY_FIELDS) > 0
+ ? RealtimeStoreMode::DEDUPLICATE
+ : RealtimeStoreMode::APPEND_ONLY;
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RealtimeSchemaLayout> schema_layout,
+ RealtimeSchemaLayout::Create(store_mode, schema_));
const std::shared_ptr<arrow::Schema>& realtime_schema = schema_layout->InputSchema();
PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
arrow::ipc::internal::json::ArrayFromJSON(
@@ -1249,6 +1291,964 @@ TEST_F(RealtimeWriteInteTest, TestAppendCommitAndRead) {
FinalizeCommitAndCheck(writer.get(), /*realtime_commits=*/{}, /*prepare_identifier=*/0, rows);
}
+TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateWritesDeletionVector) {
+ ConfigureRealtimeDeduplicateTable();
+
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+ RealtimeContext::Create());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+ CreateRealtimeWriter(realtime_context));
+
+ const std::vector<Row> first_rows = {{1, "old-one", "p0"}, {2, "two", "p0"}};
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
+ MakeBatch(first_rows, /*partitioned=*/false));
+ ASSERT_OK(writer->Write(std::move(first_batch)));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+ ASSERT_EQ(1, first_progress.size());
+ ASSERT_EQ(OffsetRange(0, 2), first_progress[0].offset_range);
+ std::shared_ptr<CommitMessageImpl> first_message =
+ std::dynamic_pointer_cast<CommitMessageImpl>(first_progress[0].commit_message);
+ ASSERT_NE(nullptr, first_message);
+ ASSERT_EQ(1, first_message->GetNewFilesIncrement().NewFiles().size());
+ ASSERT_TRUE(first_message->GetCompactIncrement().NewIndexFiles().empty());
+ ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id,
+ Commit(first_progress, /*commit_identifier=*/0));
+
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> first_rows_before_refresh, ReadRows(realtime_context));
+ ASSERT_EQ(first_rows, first_rows_before_refresh);
+
+ ASSERT_OK(writer->RefreshCommittedSnapshot(first_snapshot_id));
+
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> first_rows_after_refresh, ReadRows(realtime_context));
+ ASSERT_EQ(first_rows, first_rows_after_refresh);
+
+ const std::vector<Row> second_rows = {{1, "new-one", "p0"}, {3, "three", "p0"}};
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
+ MakeBatch(second_rows, /*partitioned=*/false));
+ ASSERT_OK(writer->Write(std::move(second_batch)));
+
+ const std::vector<Row> expected_rows = {
+ {2, "two", "p0"}, {1, "new-one", "p0"}, {3, "three", "p0"}};
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, realtime_rows);
+
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+ ASSERT_EQ(1, second_progress.size());
+ ASSERT_EQ(OffsetRange(2, 4), second_progress[0].offset_range);
+ std::shared_ptr<CommitMessageImpl> second_message =
+ std::dynamic_pointer_cast<CommitMessageImpl>(second_progress[0].commit_message);
+ ASSERT_NE(nullptr, second_message);
+ ASSERT_EQ(1, second_message->GetNewFilesIncrement().NewFiles().size());
+ ASSERT_TRUE(second_message->GetCompactIncrement().NewIndexFiles().empty());
+ ASSERT_NE(nullptr, second_progress[0].deleted_offsets);
+ ASSERT_FALSE(second_progress[0].deleted_offsets->IsEmpty());
+
+ ASSERT_OK_AND_ASSIGN(int64_t second_snapshot_id,
+ Commit(second_progress, /*commit_identifier=*/1));
+
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_refresh, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, rows_before_refresh);
+
+ ASSERT_OK(writer->RefreshCommittedSnapshot(second_snapshot_id));
+
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_refresh, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, rows_after_refresh);
+ ASSERT_OK(writer->Close());
+ writer.reset();
+ realtime_context.reset();
+
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
+ ASSERT_EQ(expected_rows, disk_rows);
+}
+
+TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCommitRetryMergesLatestDv) {
+ ConfigureRealtimeDeduplicateTable();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+ RealtimeContext::Create());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+ CreateRealtimeWriter(realtime_context));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<RecordBatch> initial_batch,
+ MakeBatch({Row{1, "old-one", "p0"}, Row{2, "two", "p0"}, Row{3, "three", "p0"}},
+ /*partitioned=*/false));
+ ASSERT_OK(writer->Write(std::move(initial_batch)));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> initial_progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+ ASSERT_EQ(1, initial_progress.size());
+ ASSERT_NE(nullptr, initial_progress[0].deleted_offsets);
+ ASSERT_TRUE(initial_progress[0].deleted_offsets->IsEmpty());
+ ASSERT_OK_AND_ASSIGN(int64_t initial_snapshot,
+ Commit(initial_progress, /*commit_identifier=*/0));
+ ASSERT_OK(writer->RefreshCommittedSnapshot(initial_snapshot));
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> update_batch,
+ MakeBatch({Row{1, "new-one", "p0"}}, /*partitioned=*/false));
+ ASSERT_OK(writer->Write(std::move(update_batch)));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+ ASSERT_EQ(1, progress.size());
+ ASSERT_NE(nullptr, progress[0].deleted_offsets);
+ ASSERT_EQ("{0}", progress[0].deleted_offsets->ToString());
+ auto initial_message =
+ std::dynamic_pointer_cast<CommitMessageImpl>(initial_progress[0].commit_message);
+ auto update_message = std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
+ ASSERT_NE(nullptr, initial_message);
+ ASSERT_NE(nullptr, update_message);
+ ASSERT_TRUE(update_message->GetCompactIncrement().IsEmpty());
+ std::vector<RealtimeCommitProgress> missing_deletions = progress;
+ missing_deletions[0].deleted_offsets.reset();
+ ASSERT_NOK_WITH_MSG(Commit(missing_deletions, /*commit_identifier=*/1),
+ "requires a deleted-offset bitmap");
+ const std::vector<std::shared_ptr<DataFileMeta>> initial_files = NewFiles(initial_progress);
+ ASSERT_EQ(1, initial_files.size());
+
+ // A concurrent physical DV update deletes a different row from the same file and bucket.
+ auto index_paths =
+ std::make_shared<MockIndexPathFactory>(PathUtil::JoinPath(table_path_, "index"));
+ DeletionVectorsIndexFile dv_file(dir_->GetFileSystem(), index_paths, /*bitmap64=*/false, pool_);
+ RoaringBitmap32 deleted_positions;
+ deleted_positions.Add(1);
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<IndexFileMeta> concurrent_dv,
+ dv_file.WriteSingleFile({{initial_files[0]->file_name,
+ std::make_shared<BitmapDeletionVector>(deleted_positions)}}));
+ auto concurrent_message = std::make_shared<CommitMessageImpl>(
+ initial_message->Partition(), /*bucket=*/0, /*total_buckets=*/1,
+ DataIncrement({}, {}, {}, {concurrent_dv}, {}), CompactIncrement({}, {}, {}, {}, {}));
+ CommitContextBuilder concurrent_builder(table_path_, "concurrent-dv-writer");
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> concurrent_context,
+ concurrent_builder.SetOptions(options_).Finish());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> concurrent_commit,
+ FileStoreCommit::Create(std::move(concurrent_context)));
+ bool interleaved = false;
+ auto fs = std::make_shared<InterleavingSnapshotFileSystem>(
+ PathUtil::JoinPath(table_path_, "snapshot/snapshot-2"), [&]() -> Status {
+ interleaved = true;
+ return concurrent_commit->Commit({concurrent_message}, /*commit_identifier=*/0,
+ /*watermark=*/std::nullopt);
+ });
+ CommitContextBuilder builder(table_path_, commit_user_);
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> context,
+ builder.SetOptions(options_).WithFileSystem(fs).Finish());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
+ FileStoreCommit::Create(std::move(context)));
+ ASSERT_OK_AND_ASSIGN(
+ int64_t snapshot_id,
+ commit->CommitWithProgress(progress, /*commit_identifier=*/1, /*watermark=*/std::nullopt));
+ ASSERT_TRUE(interleaved);
+ ASSERT_EQ(3, snapshot_id);
+ ASSERT_OK_AND_ASSIGN(uint64_t attempts, commit->GetCommitMetrics()->GetCounter(
+ CommitMetrics::LAST_COMMIT_ATTEMPTS));
+ ASSERT_EQ(2, attempts);
+ ASSERT_EQ("{0}", progress[0].deleted_offsets->ToString());
+ const std::vector<Row> expected_rows = {{3, "three", "p0"}, {1, "new-one", "p0"}};
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
+ ASSERT_EQ(expected_rows, disk_rows);
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_refresh, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, rows_before_refresh);
+ ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_refresh, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, rows_after_refresh);
+ ASSERT_OK_AND_ASSIGN(
+ int64_t retried_snapshot,
+ commit->CommitWithProgress(progress, /*commit_identifier=*/1, /*watermark=*/std::nullopt));
+ ASSERT_EQ(snapshot_id, retried_snapshot);
+ ASSERT_OK(writer->Close());
+}
+
+// TODO(xinyu.lxy): Enable when realtime deduplicate compaction is supported.
+TEST_F(RealtimeWriteInteTest, DISABLED_TestRealtimeDeduplicateCommitRetryRemapsCompactedOffsets) {
+ ConfigureRealtimeDeduplicateTable();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+ RealtimeContext::Create());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+ CreateRealtimeWriter(realtime_context));
+ for (int64_t id = 1; id <= 2; ++id) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({Row{id, "value", "p0"}}, /*partitioned=*/false));
+ ASSERT_OK(writer->Write(std::move(batch)));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/id - 1));
+ ASSERT_OK_AND_ASSIGN(int64_t snapshot, Commit(progress, /*commit_identifier=*/id - 1));
+ ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot));
+ }
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> delete_batch,
+ MakeBatch({Row{2, "ignored", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
+ {RecordBatch::RowKind::DELETE}));
+ ASSERT_OK(writer->Write(std::move(delete_batch)));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+ ASSERT_EQ(1, progress.size());
+ ASSERT_NE(nullptr, progress[0].deleted_offsets);
+ ASSERT_EQ("{1}", progress[0].deleted_offsets->ToString());
+ bool interleaved = false;
+ auto fs = std::make_shared<InterleavingSnapshotFileSystem>(
+ PathUtil::JoinPath(table_path_, "snapshot/snapshot-3"), [&]() -> Status {
+ PAIMON_ASSIGN_OR_RAISE(Snapshot compacted,
+ CompactAndCommit({}, /*bucket=*/0, /*commit_identifier=*/1));
+ if (compacted.Id() != 3 ||
+ compacted.GetCommitKind() != Snapshot::CommitKind::Compact()) {
+ return Status::Invalid("expected competing compaction snapshot");
+ }
+ interleaved = true;
+ return Status::OK();
+ });
+ CommitContextBuilder builder(table_path_, commit_user_);
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> context,
+ builder.SetOptions(options_).WithFileSystem(fs).Finish());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
+ FileStoreCommit::Create(std::move(context)));
+ ASSERT_OK_AND_ASSIGN(
+ int64_t snapshot_id,
+ commit->CommitWithProgress(progress, /*commit_identifier=*/2, /*watermark=*/std::nullopt));
+ ASSERT_TRUE(interleaved);
+ ASSERT_EQ(4, snapshot_id);
+ ASSERT_OK_AND_ASSIGN(uint64_t attempts, commit->GetCommitMetrics()->GetCounter(
+ CommitMetrics::LAST_COMMIT_ATTEMPTS));
+ ASSERT_EQ(2, attempts);
+ ASSERT_EQ("{1}", progress[0].deleted_offsets->ToString());
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
+ ASSERT_EQ((std::vector<Row>{{1, "value", "p0"}}), disk_rows);
+ ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateWriterHandoffRestoresCommittedFiles) {
+ ConfigureRealtimeDeduplicateTable();
+
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+ RealtimeContext::Create());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
+ CreateRealtimeWriter(realtime_context));
+ const std::vector<Row> initial_rows = {{1, "old-one", "p0"}, {2, "two", "p0"}};
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> initial_batch,
+ MakeBatch(initial_rows, /*partitioned=*/false));
+ ASSERT_OK(first_writer->Write(std::move(initial_batch)));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> initial_progress,
+ first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+ ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(initial_progress, /*commit_identifier=*/0));
+ ASSERT_EQ(1, snapshot_id);
+ ASSERT_OK(first_writer->Close());
+ first_writer.reset();
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
+ CreateRealtimeWriter(realtime_context));
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> restored_rows, ReadRows(realtime_context));
+ ASSERT_EQ(initial_rows, restored_rows);
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> update_batch,
+ MakeBatch({{1, "new-one", "p0"}}, /*partitioned=*/false));
+ ASSERT_OK(second_writer->Write(std::move(update_batch)));
+ const std::vector<Row> expected_rows = {{2, "two", "p0"}, {1, "new-one", "p0"}};
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> updated_rows, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, updated_rows);
+
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> update_progress,
+ second_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+ ASSERT_EQ(1, update_progress.size());
+ std::shared_ptr<CommitMessageImpl> update_message =
+ std::dynamic_pointer_cast<CommitMessageImpl>(update_progress[0].commit_message);
+ ASSERT_NE(nullptr, update_message);
+ ASSERT_TRUE(update_message->GetCompactIncrement().NewIndexFiles().empty());
+ ASSERT_NE(nullptr, update_progress[0].deleted_offsets);
+ ASSERT_FALSE(update_progress[0].deleted_offsets->IsEmpty());
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_prepare, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, rows_after_prepare);
+ ASSERT_OK(second_writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateWriterHandoffRetainsPreparedFiles) {
+ ConfigureRealtimeDeduplicateTable();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+ RealtimeContext::Create());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
+ CreateRealtimeWriter(realtime_context));
+ const std::vector<Row> initial_rows = {
+ {1, "old-one", "p0"}, {2, "keep-two", "p0"}, {3, "delete-three", "p0"}};
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> initial_batch,
+ MakeBatch(initial_rows, /*partitioned=*/false));
+ ASSERT_OK(first_writer->Write(std::move(initial_batch)));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+ first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+ ASSERT_EQ(1, progress.size());
+ ASSERT_EQ(OffsetRange(0, 3), progress[0].offset_range);
+ std::shared_ptr<CommitMessageImpl> initial_message =
+ std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
+ ASSERT_NE(nullptr, initial_message);
+ ASSERT_EQ(1, initial_message->GetNewFilesIncrement().NewFiles().size());
+ ASSERT_TRUE(initial_message->GetCompactIncrement().NewIndexFiles().empty());
+ ASSERT_OK(first_writer->Close());
+ first_writer.reset();
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
+ CreateRealtimeWriter(realtime_context));
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> restored_rows, ReadRows(realtime_context));
+ ASSERT_EQ(initial_rows, restored_rows);
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<RecordBatch> update_batch,
+ MakeBatch({Row{1, "new-one", "p0"}, Row{3, "ignored", "p0"}, Row{4, "four", "p0"}},
+ /*partitioned=*/false, /*bucket=*/0,
+ {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE,
+ RecordBatch::RowKind::INSERT}));
+ ASSERT_OK(second_writer->Write(std::move(update_batch)));
+ const std::vector<Row> expected_rows = {
+ {2, "keep-two", "p0"}, {1, "new-one", "p0"}, {4, "four", "p0"}};
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_prepare, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, rows_before_prepare);
+
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> update_progress,
+ second_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+ ASSERT_EQ(1, update_progress.size());
+ ASSERT_EQ(OffsetRange(3, 6), update_progress[0].offset_range);
+ std::shared_ptr<CommitMessageImpl> update_message =
+ std::dynamic_pointer_cast<CommitMessageImpl>(update_progress[0].commit_message);
+ ASSERT_NE(nullptr, update_message);
+ ASSERT_EQ(1, update_message->GetNewFilesIncrement().NewFiles().size());
+ ASSERT_EQ(2, update_message->GetNewFilesIncrement().NewFiles()[0]->row_count);
+ ASSERT_TRUE(update_message->GetCompactIncrement().NewIndexFiles().empty());
+ ASSERT_NE(nullptr, update_progress[0].deleted_offsets);
+ ASSERT_FALSE(update_progress[0].deleted_offsets->IsEmpty());
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_prepare, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, rows_after_prepare);
+
+ progress.push_back(std::move(update_progress[0]));
+ ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/1));
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_refresh, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, rows_before_refresh);
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
+ ASSERT_EQ(expected_rows, disk_rows);
+ ASSERT_OK(second_writer->RefreshCommittedSnapshot(snapshot_id));
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_refresh, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, rows_after_refresh);
+ ASSERT_OK(second_writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateWriterHandoffRetainsDvAcrossRefresh) {
+ for (bool prepare_before_refresh : {false, true}) {
+ SCOPED_TRACE(prepare_before_refresh);
+ dir_ = UniqueTestDirectory::Create("local");
+ ASSERT_NE(nullptr, dir_);
+ table_path_ = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
+ ResetExternalOffset(/*partition=*/{}, /*bucket=*/0, /*next_offset=*/0);
+ ConfigureRealtimeDeduplicateTable();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+ RealtimeContext::Create());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
+ CreateRealtimeWriter(realtime_context));
+ const std::vector<Row> initial_rows = {
+ {1, "one", "p0"}, {2, "two", "p0"}, {3, "three", "p0"}};
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> initial_batch,
+ MakeBatch(initial_rows, /*partitioned=*/false));
+ ASSERT_OK(first_writer->Write(std::move(initial_batch)));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> initial_progress,
+ first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+ ASSERT_EQ(1, initial_progress.size());
+ auto initial_message =
+ std::dynamic_pointer_cast<CommitMessageImpl>(initial_progress[0].commit_message);
+ ASSERT_NE(nullptr, initial_message);
+ ASSERT_EQ(1, initial_message->GetNewFilesIncrement().NewFiles().size());
+ ASSERT_OK_AND_ASSIGN(int64_t initial_snapshot_id,
+ Commit(initial_progress, /*commit_identifier=*/0));
+ ASSERT_OK(first_writer->RefreshCommittedSnapshot(initial_snapshot_id));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_delete,
+ MakeBatch({initial_rows[0]}, /*partitioned=*/false, /*bucket=*/0,
+ {RecordBatch::RowKind::DELETE}));
+ ASSERT_OK(first_writer->Write(std::move(first_delete)));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_progress,
+ first_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+ ASSERT_OK(first_writer->Close());
+ first_writer.reset();
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
+ CreateRealtimeWriter(realtime_context));
+ const Row fourth_row = {4, "four", "p0"};
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> insert_batch,
+ MakeBatch({fourth_row}, /*partitioned=*/false));
+ // Create the bucket writer before the first writer's deletion reaches the snapshot.
+ ASSERT_OK(second_writer->Write(std::move(insert_batch)));
+ std::vector<RealtimeCommitProgress> second_progress;
+ if (prepare_before_refresh) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_delete,
+ MakeBatch({initial_rows[1]}, /*partitioned=*/false, /*bucket=*/0,
+ {RecordBatch::RowKind::DELETE}));
+ ASSERT_OK(second_writer->Write(std::move(second_delete)));
+ ASSERT_OK_AND_ASSIGN(second_progress,
+ second_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+ }
+ ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id,
+ Commit(first_progress, /*commit_identifier=*/1));
+ ASSERT_OK(second_writer->RefreshCommittedSnapshot(first_snapshot_id));
+ const std::vector<Row> rows_after_refresh =
+ prepare_before_refresh ? std::vector<Row>{initial_rows[2], fourth_row}
+ : std::vector<Row>{initial_rows[1], initial_rows[2], fourth_row};
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> refreshed_rows, ReadRows(realtime_context));
+ ASSERT_EQ(rows_after_refresh, refreshed_rows);
+ if (!prepare_before_refresh) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_delete,
+ MakeBatch({initial_rows[1]}, /*partitioned=*/false, /*bucket=*/0,
+ {RecordBatch::RowKind::DELETE}));
+ ASSERT_OK(second_writer->Write(std::move(second_delete)));
+ ASSERT_OK_AND_ASSIGN(second_progress,
+ second_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+ }
+ const std::vector<Row> expected_rows = {initial_rows[2], fourth_row};
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> prepared_rows, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, prepared_rows);
+ ASSERT_OK(Commit(second_progress, /*commit_identifier=*/2));
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
+ ASSERT_EQ(expected_rows, disk_rows);
+
+ // Another prepare must preserve deletions from the message prepared before refresh,
+ // even though its commit has not yet been refreshed into this context.
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> third_delete,
+ MakeBatch({initial_rows[2]}, /*partitioned=*/false, /*bucket=*/0,
+ {RecordBatch::RowKind::DELETE}));
+ ASSERT_OK(second_writer->Write(std::move(third_delete)));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> third_progress,
+ second_writer->PrepareCommitWithProgress(/*commit_identifier=*/3));
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_commit, ReadRows(realtime_context));
+ ASSERT_EQ(std::vector<Row>{fourth_row}, rows_before_commit);
+ ASSERT_OK_AND_ASSIGN(int64_t final_snapshot_id,
+ Commit(third_progress, /*commit_identifier=*/3));
+ ASSERT_OK_AND_ASSIGN(disk_rows, ReadRows());
+ ASSERT_EQ(std::vector<Row>{fourth_row}, disk_rows);
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_refresh, ReadRows(realtime_context));
+ ASSERT_EQ(std::vector<Row>{fourth_row}, rows_before_refresh);
+ ASSERT_OK(second_writer->RefreshCommittedSnapshot(final_snapshot_id));
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows, ReadRows(realtime_context));
+ ASSERT_EQ(std::vector<Row>{fourth_row}, final_rows);
+ ASSERT_OK(second_writer->Close());
+ }
+}
+
+TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateWithinOneBatch) {
+ ConfigureRealtimeDeduplicateTable();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+ RealtimeContext::Create());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+ CreateRealtimeWriter(realtime_context));
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<RecordBatch> batch,
+ MakeBatch({Row{1, "old-one", "p0"}, Row{2, "two", "p0"}, Row{1, "new-one", "p0"}},
+ /*partitioned=*/false));
+ ASSERT_OK(writer->Write(std::move(batch)));
+ const std::vector<Row> expected_rows = {{2, "two", "p0"}, {1, "new-one", "p0"}};
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, realtime_rows);
+
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+ ASSERT_EQ(1, progress.size());
+ ASSERT_EQ(OffsetRange(0, 3), progress[0].offset_range);
+ std::shared_ptr<CommitMessageImpl> message =
+ std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
+ ASSERT_NE(nullptr, message);
+ ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
+ ASSERT_EQ(2, message->GetNewFilesIncrement().NewFiles()[0]->row_count);
+ ASSERT_TRUE(message->GetCompactIncrement().NewIndexFiles().empty());
+
+ ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/0));
+ ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+ ASSERT_OK(writer->Close());
+ writer.reset();
+ realtime_context.reset();
+
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
+ ASSERT_EQ(expected_rows, disk_rows);
+}
+
+TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateAcrossSealedSegments) {
+ ConfigureRealtimeDeduplicateTable();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+ RealtimeContext::Create());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+ CreateRealtimeWriter(realtime_context));
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
+ MakeBatch({Row{1, "old-one", "p0"}, Row{2, "two", "p0"}},
+ /*partitioned=*/false));
+ ASSERT_OK(writer->Write(std::move(first_batch)));
+ ASSERT_OK(writer->Seal());
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
+ MakeBatch({Row{1, "new-one", "p0"}, Row{3, "three", "p0"}},
+ /*partitioned=*/false));
+ ASSERT_OK(writer->Write(std::move(second_batch)));
+ const std::vector<Row> expected_rows = {
+ {2, "two", "p0"}, {1, "new-one", "p0"}, {3, "three", "p0"}};
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_prepare, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, rows_before_prepare);
+
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+ ASSERT_EQ(1, progress.size());
+ ASSERT_EQ(OffsetRange(0, 4), progress[0].offset_range);
+ std::shared_ptr<CommitMessageImpl> message =
+ std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
+ ASSERT_NE(nullptr, message);
+ ASSERT_EQ(2, message->GetNewFilesIncrement().NewFiles().size());
+ ASSERT_TRUE(message->GetCompactIncrement().NewIndexFiles().empty());
+ ASSERT_NE(nullptr, progress[0].deleted_offsets);
+ ASSERT_FALSE(progress[0].deleted_offsets->IsEmpty());
+
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_prepare, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, rows_after_prepare);
+
+ ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/0));
+ ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_refresh, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, rows_after_refresh);
+ ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateAcrossPreparedSegments) {
+ ConfigureRealtimeDeduplicateTable();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+ RealtimeContext::Create());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+ CreateRealtimeWriter(realtime_context));
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
+ MakeBatch({Row{1, "old-one", "p0"}, Row{2, "two", "p0"}},
+ /*partitioned=*/false));
+ ASSERT_OK(writer->Write(std::move(first_batch)));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+ ASSERT_EQ(1, progress.size());
+ ASSERT_EQ(OffsetRange(0, 2), progress[0].offset_range);
+ ASSERT_NE(nullptr, progress[0].deleted_offsets);
+ ASSERT_TRUE(progress[0].deleted_offsets->IsEmpty());
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
+ MakeBatch({Row{1, "new-one", "p0"}, Row{3, "three", "p0"}},
+ /*partitioned=*/false));
+ ASSERT_OK(writer->Write(std::move(second_batch)));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+ ASSERT_EQ(1, second_progress.size());
+ ASSERT_EQ(OffsetRange(2, 4), second_progress[0].offset_range);
+ ASSERT_NE(nullptr, second_progress[0].deleted_offsets);
+ ASSERT_EQ("{0}", second_progress[0].deleted_offsets->ToString());
+ ASSERT_TRUE(progress[0].deleted_offsets->IsEmpty());
+ std::shared_ptr<CommitMessageImpl> second_message =
+ std::dynamic_pointer_cast<CommitMessageImpl>(second_progress[0].commit_message);
+ ASSERT_NE(nullptr, second_message);
+ ASSERT_EQ(1, second_message->GetNewFilesIncrement().NewFiles().size());
+ ASSERT_TRUE(second_message->GetCompactIncrement().NewIndexFiles().empty());
+
+ progress.push_back(std::move(second_progress[0]));
+ ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/1));
+ const std::vector<Row> expected_rows = {
+ {2, "two", "p0"}, {1, "new-one", "p0"}, {3, "three", "p0"}};
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_refresh, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, rows_before_refresh);
+ ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_refresh, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, rows_after_refresh);
+ ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateDeletesCommittedKey) {
+ ConfigureRealtimeDeduplicateTable();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+ RealtimeContext::Create());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+ CreateRealtimeWriter(realtime_context));
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<RecordBatch> insert_batch,
+ MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}}, /*partitioned=*/false));
+ ASSERT_OK(writer->Write(std::move(insert_batch)));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> insert_progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+ ASSERT_OK_AND_ASSIGN(int64_t insert_snapshot_id,
+ Commit(insert_progress, /*commit_identifier=*/0));
+ ASSERT_OK(writer->RefreshCommittedSnapshot(insert_snapshot_id));
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> delete_batch,
+ MakeBatch({Row{1, "ignored", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
+ {RecordBatch::RowKind::DELETE}));
+ ASSERT_OK(writer->Write(std::move(delete_batch)));
+ const std::vector<Row> expected_rows = {{2, "two", "p0"}};
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, realtime_rows);
+
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> delete_progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+ ASSERT_EQ(1, delete_progress.size());
+ ASSERT_EQ(OffsetRange(2, 3), delete_progress[0].offset_range);
+ std::shared_ptr<CommitMessageImpl> delete_message =
+ std::dynamic_pointer_cast<CommitMessageImpl>(delete_progress[0].commit_message);
+ ASSERT_NE(nullptr, delete_message);
+ ASSERT_TRUE(delete_message->GetNewFilesIncrement().NewFiles().empty());
+ ASSERT_TRUE(delete_message->GetCompactIncrement().NewIndexFiles().empty());
+ ASSERT_NE(nullptr, delete_progress[0].deleted_offsets);
+ ASSERT_FALSE(delete_progress[0].deleted_offsets->IsEmpty());
+
+ ASSERT_OK_AND_ASSIGN(int64_t delete_snapshot_id,
+ Commit(delete_progress, /*commit_identifier=*/1));
+ ASSERT_OK(writer->RefreshCommittedSnapshot(delete_snapshot_id));
+ ASSERT_OK(writer->Close());
+ writer.reset();
+ realtime_context.reset();
+
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
+ ASSERT_EQ(expected_rows, disk_rows);
+}
+
+TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateInsertThenDeleteInOneSegment) {
+ ConfigureRealtimeDeduplicateTable();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+ RealtimeContext::Create());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+ CreateRealtimeWriter(realtime_context));
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({Row{1, "one", "p0"}, Row{1, "ignored", "p0"}},
+ /*partitioned=*/false, /*bucket=*/0,
+ {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE}));
+ ASSERT_OK(writer->Write(std::move(batch)));
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
+ ASSERT_TRUE(realtime_rows.empty());
+
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+ ASSERT_EQ(1, progress.size());
+ ASSERT_EQ(OffsetRange(0, 2), progress[0].offset_range);
+ std::shared_ptr<CommitMessageImpl> message =
+ std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
+ ASSERT_NE(nullptr, message);
+ ASSERT_TRUE(message->IsEmpty());
+
+ ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/0));
+ ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+ ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
+ ASSERT_EQ(2, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_refresh, ReadRows(realtime_context));
+ ASSERT_TRUE(rows_after_refresh.empty());
+ ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateUpdateBeforeAndAfter) {
+ ConfigureRealtimeDeduplicateTable();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+ RealtimeContext::Create());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+ CreateRealtimeWriter(realtime_context));
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> insert_batch,
+ MakeBatch({Row{1, "old-one", "p0"}, Row{2, "two", "p0"}},
+ /*partitioned=*/false));
+ ASSERT_OK(writer->Write(std::move(insert_batch)));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> insert_progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+ ASSERT_OK_AND_ASSIGN(int64_t insert_snapshot_id,
+ Commit(insert_progress, /*commit_identifier=*/0));
+ ASSERT_OK(writer->RefreshCommittedSnapshot(insert_snapshot_id));
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<RecordBatch> update_batch,
+ MakeBatch({Row{1, "ignored-before", "p0"}, Row{1, "new-one", "p0"}},
+ /*partitioned=*/false, /*bucket=*/0,
+ {RecordBatch::RowKind::UPDATE_BEFORE, RecordBatch::RowKind::UPDATE_AFTER}));
+ ASSERT_OK(writer->Write(std::move(update_batch)));
+ const std::vector<Row> expected_rows = {{2, "two", "p0"}, {1, "new-one", "p0"}};
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, realtime_rows);
+
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> update_progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+ ASSERT_EQ(1, update_progress.size());
+ ASSERT_EQ(OffsetRange(2, 4), update_progress[0].offset_range);
+ std::shared_ptr<CommitMessageImpl> update_message =
+ std::dynamic_pointer_cast<CommitMessageImpl>(update_progress[0].commit_message);
+ ASSERT_NE(nullptr, update_message);
+ ASSERT_EQ(1, update_message->GetNewFilesIncrement().NewFiles().size());
+ ASSERT_TRUE(update_message->GetCompactIncrement().NewIndexFiles().empty());
+ ASSERT_NE(nullptr, update_progress[0].deleted_offsets);
+ ASSERT_FALSE(update_progress[0].deleted_offsets->IsEmpty());
+
+ ASSERT_OK_AND_ASSIGN(int64_t update_snapshot_id,
+ Commit(update_progress, /*commit_identifier=*/1));
+ ASSERT_OK(writer->RefreshCommittedSnapshot(update_snapshot_id));
+ ASSERT_OK(writer->Close());
+ writer.reset();
+ realtime_context.reset();
+
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
+ ASSERT_EQ(expected_rows, disk_rows);
+}
+
+TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateAcrossMultipleFileIndexStorageForms) {
+ options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
+ ConfigureRealtimeDeduplicateTable();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+ RealtimeContext::Create());
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> embedded_writer,
+ CreateRealtimeWriter(realtime_context));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> embedded_batch,
+ MakeBatch({Row{1, "old-one", "p0"}, Row{2, "two", "p0"}},
+ /*partitioned=*/false));
+ ASSERT_OK(embedded_writer->Write(std::move(embedded_batch)));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> embedded_progress,
+ embedded_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+ std::vector<std::shared_ptr<DataFileMeta>> embedded_files = NewFiles(embedded_progress);
+ ASSERT_EQ(1, embedded_files.size());
+ ASSERT_NE(nullptr, embedded_files[0]->embedded_index);
+ // The only extra file is the deduplicate key-offset sidecar.
+ ASSERT_EQ(1, embedded_files[0]->extra_files.size());
+ ASSERT_OK_AND_ASSIGN(int64_t embedded_snapshot_id,
+ Commit(embedded_progress, /*commit_identifier=*/0));
+ ASSERT_OK(embedded_writer->RefreshCommittedSnapshot(embedded_snapshot_id));
+ ASSERT_OK(embedded_writer->Close());
+
+ options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> external_writer,
+ CreateRealtimeWriter(realtime_context));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> external_batch,
+ MakeBatch({Row{3, "old-three", "p0"}, Row{4, "four", "p0"}},
+ /*partitioned=*/false));
+ ASSERT_OK(external_writer->Write(std::move(external_batch)));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> external_progress,
+ external_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+ std::vector<std::shared_ptr<DataFileMeta>> external_files = NewFiles(external_progress);
+ ASSERT_EQ(1, external_files.size());
+ ASSERT_EQ(nullptr, external_files[0]->embedded_index);
+ // The external bitmap index precedes the deduplicate key-offset sidecar.
+ ASSERT_EQ(2, external_files[0]->extra_files.size());
+ ASSERT_OK_AND_ASSIGN(int64_t external_snapshot_id,
+ Commit(external_progress, /*commit_identifier=*/1));
+ ASSERT_OK(external_writer->RefreshCommittedSnapshot(external_snapshot_id));
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<RecordBatch> update_batch,
+ MakeBatch({Row{1, "new-one", "p0"}, Row{3, "new-three", "p0"}, Row{5, "five", "p0"}},
+ /*partitioned=*/false));
+ ASSERT_OK(external_writer->Write(std::move(update_batch)));
+ const std::vector<Row> expected_rows = {{2, "two", "p0"},
+ {4, "four", "p0"},
+ {1, "new-one", "p0"},
+ {3, "new-three", "p0"},
+ {5, "five", "p0"}};
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_prepare, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, rows_before_prepare);
+
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> update_progress,
+ external_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
+ ASSERT_EQ(1, update_progress.size());
+ std::shared_ptr<CommitMessageImpl> update_message =
+ std::dynamic_pointer_cast<CommitMessageImpl>(update_progress[0].commit_message);
+ ASSERT_NE(nullptr, update_message);
+ ASSERT_EQ(1, update_message->GetNewFilesIncrement().NewFiles().size());
+ ASSERT_TRUE(update_message->GetCompactIncrement().NewIndexFiles().empty());
+ ASSERT_NE(nullptr, update_progress[0].deleted_offsets);
+ ASSERT_FALSE(update_progress[0].deleted_offsets->IsEmpty());
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_prepare, ReadRows(realtime_context));
+ ASSERT_EQ(expected_rows, rows_after_prepare);
+
+ ASSERT_OK_AND_ASSIGN(int64_t update_snapshot_id,
+ Commit(update_progress, /*commit_identifier=*/2));
+ ASSERT_OK(external_writer->RefreshCommittedSnapshot(update_snapshot_id));
+ ASSERT_OK(external_writer->Close());
+ external_writer.reset();
+ realtime_context.reset();
+
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
+ ASSERT_EQ(expected_rows, disk_rows);
+}
+
+TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateReadViewIsStableAcrossWriteAndRefresh) {
+ ConfigureRealtimeDeduplicateTable();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+ RealtimeContext::Create());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+ CreateRealtimeWriter(realtime_context));
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
+ MakeBatch({Row{1, "old-one", "p0"}, Row{2, "old-two", "p0"}},
+ /*partitioned=*/false));
+ ASSERT_OK(writer->Write(std::move(base_batch)));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+ ASSERT_OK_AND_ASSIGN(int64_t base_snapshot_id, Commit(base_progress, /*commit_identifier=*/0));
+ ASSERT_OK(writer->RefreshCommittedSnapshot(base_snapshot_id));
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_update,
+ MakeBatch({Row{1, "new-one", "p0"}, Row{3, "three", "p0"}},
+ /*partitioned=*/false));
+ ASSERT_OK(writer->Write(std::move(first_update)));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> old_plan,
+ CreatePlan(realtime_context, /*predicate=*/nullptr));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> old_plan_across_refresh,
+ CreatePlan(realtime_context, /*predicate=*/nullptr));
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_update,
+ MakeBatch({Row{2, "new-two", "p0"}, Row{4, "four", "p0"}},
+ /*partitioned=*/false));
+ ASSERT_OK(writer->Write(std::move(second_update)));
+
+ const std::vector<Row> old_expected_rows = {
+ {2, "old-two", "p0"}, {1, "new-one", "p0"}, {3, "three", "p0"}};
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> old_plan_rows, ReadRows(old_plan, realtime_context));
+ ASSERT_EQ(old_expected_rows, old_plan_rows);
+
+ const std::vector<Row> latest_expected_rows = {
+ {1, "new-one", "p0"}, {3, "three", "p0"}, {2, "new-two", "p0"}, {4, "four", "p0"}};
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> latest_rows_before_prepare, ReadRows(realtime_context));
+ ASSERT_EQ(latest_expected_rows, latest_rows_before_prepare);
+
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+ ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/1));
+ ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
+
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> pinned_rows_after_refresh,
+ ReadRows(old_plan_across_refresh, realtime_context));
+ ASSERT_EQ(old_expected_rows, pinned_rows_after_refresh);
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> latest_rows_after_refresh, ReadRows(realtime_context));
+ ASSERT_EQ(latest_expected_rows, latest_rows_after_refresh);
+ ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateIsolatesPartitionsAndBuckets) {
+ options_[Options::BUCKET] = "2";
+ ConfigureRealtimeDeduplicateTable(/*partition_keys=*/{"pt"});
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+ RealtimeContext::Create());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+ CreateRealtimeWriter(realtime_context));
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p0b0_batch,
+ MakeBatch({Row{1, "p0-b0-old", "p0"}, Row{2, "p0-b0-keep", "p0"}},
+ /*partitioned=*/true, /*bucket=*/0));
+ ASSERT_OK(writer->Write(std::move(p0b0_batch)));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p0b1_batch,
+ MakeBatch({Row{1, "p0-b1-old", "p0"}, Row{3, "p0-b1-keep", "p0"}},
+ /*partitioned=*/true, /*bucket=*/1));
+ ASSERT_OK(writer->Write(std::move(p0b1_batch)));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p1b0_batch,
+ MakeBatch({Row{1, "p1-b0-old", "p1"}, Row{4, "p1-b0-keep", "p1"}},
+ /*partitioned=*/true, /*bucket=*/0));
+ ASSERT_OK(writer->Write(std::move(p1b0_batch)));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+ ASSERT_EQ(3, base_progress.size());
+ ASSERT_OK_AND_ASSIGN(int64_t base_snapshot_id, Commit(base_progress, /*commit_identifier=*/0));
+ ASSERT_OK(writer->RefreshCommittedSnapshot(base_snapshot_id));
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<RecordBatch> update_batch,
+ MakeBatch({Row{1, "p0-b0-new", "p0"}}, /*partitioned=*/true, /*bucket=*/0));
+ ASSERT_OK(writer->Write(std::move(update_batch)));
+ const std::vector<Row> expected_rows = {{2, "p0-b0-keep", "p0"}, {1, "p0-b0-new", "p0"},
+ {1, "p0-b1-old", "p0"}, {3, "p0-b1-keep", "p0"},
+ {1, "p1-b0-old", "p1"}, {4, "p1-b0-keep", "p1"}};
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_prepare, ReadRows(realtime_context));
+ ASSERT_EQ(std::multiset<Row>(expected_rows.begin(), expected_rows.end()),
+ std::multiset<Row>(rows_before_prepare.begin(), rows_before_prepare.end()));
+
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> update_progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+ ASSERT_EQ(1, update_progress.size());
+ ASSERT_EQ(RealtimePartitionBucket({{"pt", "p0"}}, /*bucket=*/0),
+ update_progress[0].partition_bucket);
+ ASSERT_EQ(OffsetRange(2, 3), update_progress[0].offset_range);
+ std::shared_ptr<CommitMessageImpl> update_message =
+ std::dynamic_pointer_cast<CommitMessageImpl>(update_progress[0].commit_message);
+ ASSERT_NE(nullptr, update_message);
+ ASSERT_TRUE(update_message->GetCompactIncrement().NewIndexFiles().empty());
+ ASSERT_NE(nullptr, update_progress[0].deleted_offsets);
+ ASSERT_FALSE(update_progress[0].deleted_offsets->IsEmpty());
+
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_prepare, ReadRows(realtime_context));
+ ASSERT_EQ(std::multiset<Row>(expected_rows.begin(), expected_rows.end()),
+ std::multiset<Row>(rows_after_prepare.begin(), rows_after_prepare.end()));
+
+ ASSERT_OK_AND_ASSIGN(int64_t update_snapshot_id,
+ Commit(update_progress, /*commit_identifier=*/1));
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_refresh, ReadRows(realtime_context));
+ ASSERT_EQ(std::multiset<Row>(expected_rows.begin(), expected_rows.end()),
+ std::multiset<Row>(rows_before_refresh.begin(), rows_before_refresh.end()));
+ ASSERT_OK(writer->RefreshCommittedSnapshot(update_snapshot_id));
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_refresh, ReadRows(realtime_context));
+ ASSERT_EQ(std::multiset<Row>(expected_rows.begin(), expected_rows.end()),
+ std::multiset<Row>(rows_after_refresh.begin(), rows_after_refresh.end()));
+
+ ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
+ ASSERT_EQ(3, committed_offsets.size());
+ ASSERT_EQ(3, committed_offsets.at(RealtimePartitionBucket({{"pt", "p0"}}, /*bucket=*/0)));
+ ASSERT_EQ(2, committed_offsets.at(RealtimePartitionBucket({{"pt", "p0"}}, /*bucket=*/1)));
+ ASSERT_EQ(2, committed_offsets.at(RealtimePartitionBucket({{"pt", "p1"}}, /*bucket=*/0)));
+ ASSERT_OK(writer->Close());
+}
+
+TEST_F(RealtimeWriteInteTest,
+ TestRealtimeDeduplicatePredicatePushdownAcrossDataFileSealedAndBuilding) {
+ options_[Options::WRITE_BATCH_SIZE] = "1";
+ options_[Options::REALTIME_STORE_STATS_MODE] = "full";
+ options_["parquet.page.size"] = "1";
+ options_["parquet.enable.dictionary"] = "false";
+ options_["parquet.write.enable-page-index"] = "true";
+ options_["parquet.write.max-row-group-length"] = "1";
+ options_["parquet.read.enable-page-index-filter"] = "true";
+ ConfigureRealtimeDeduplicateTable();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
+ RealtimeContext::Create());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
+ CreateRealtimeWriter(realtime_context));
+
+ auto write_one = [&](const Row& row) -> Status {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({row}, /*partitioned=*/false));
+ return writer->Write(std::move(batch));
+ };
+
+ ASSERT_OK(write_one({1, "match", "p0"}));
+ ASSERT_OK(write_one({2, "data-file-other", "p0"}));
+ ASSERT_OK(write_one({7, "match", "p0"}));
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> data_file_progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
+ ASSERT_OK_AND_ASSIGN(int64_t data_file_snapshot_id,
+ Commit(data_file_progress, /*commit_identifier=*/0));
+ ASSERT_OK(writer->RefreshCommittedSnapshot(data_file_snapshot_id));
+
+ ASSERT_OK(write_one({3, "match", "p0"}));
+ ASSERT_OK(write_one({4, "sealed-other", "p0"}));
+ ASSERT_OK(write_one({7, "replacement-other", "p0"}));
+ ASSERT_OK(writer->Seal());
+
+ ASSERT_OK(write_one({5, "match", "p0"}));
+ ASSERT_OK(write_one({6, "building-other", "p0"}));
+
+ const std::string matching_payload = "match";
+ std::shared_ptr<Predicate> scan_predicate = PredicateBuilder::Equal(
+ /*field_index=*/2, /*field_name=*/"payload", FieldType::STRING,
+ Literal(FieldType::STRING, matching_payload.data(), matching_payload.size()));
+ std::shared_ptr<Predicate> read_predicate = PredicateBuilder::Equal(
+ /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
+ Literal(FieldType::STRING, matching_payload.data(), matching_payload.size()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, CreatePlan(realtime_context, scan_predicate));
+
+ // Disable residual filtering so the result proves that the predicate reached the data file
+ // reader and both the sealed and building in-memory segments.
+ ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, ReadRows(plan, realtime_context, read_predicate,
+ /*enable_predicate_filter=*/false));
+ ASSERT_EQ((std::vector<Row>{{1, "match", "p0"}, {3, "match", "p0"}, {5, "match", "p0"}}), rows);
+ ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> final_progress,
+ writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
+ ASSERT_EQ(1, final_progress.size());
+ std::shared_ptr<CommitMessageImpl> final_message =
+ std::dynamic_pointer_cast<CommitMessageImpl>(final_progress[0].commit_message);
+ ASSERT_NE(nullptr, final_message);
+ ASSERT_TRUE(final_message->GetCompactIncrement().NewIndexFiles().empty());
+ ASSERT_NE(nullptr, final_progress[0].deleted_offsets);
+ ASSERT_FALSE(final_progress[0].deleted_offsets->IsEmpty());
+ ASSERT_OK(writer->Close());
+}
+
TEST_F(RealtimeWriteInteTest, TestRealtimeReadBatchSizeSplitsLargeAppendBatch) {
CreateTable(/*partition_keys=*/{});
ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,