You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ka...@apache.org on 2026/03/27 06:09:29 UTC
(doris) 01/01: [feature](iceberg) Implements iceberg update delete merge into functionality. (#60482)
This is an automated email from the ASF dual-hosted git repository.
kakachen pushed a commit to branch cherry-pick-60482_4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1b84cf854060ae75a8bc9501b6b812bdd0869ffc
Author: Qi Chen <ch...@selectdb.com>
AuthorDate: Thu Mar 26 14:17:51 2026 +0800
[feature](iceberg) Implements iceberg update delete merge into functionality. (#60482)
[feature] (iceberg) Implements iceberg update & delete & merge into.
1. **Nereids Planner**:
- Introduced parser and planner support for `DELETE`, `UPDATE`, and
`MERGE INTO` targeting Iceberg tables.
- Added specific plan nodes and rules (`IcebergDeleteCommand`,
`IcebergMergeCommand`, `IcebergUpdateCommand`,
`Logical/PhysicalIcebergDeleteSink`,
`Logical/PhysicalIcebergMergeSink`).
2. **Frontend & Transaction**:
- Handled Iceberg DML transactional context and snapshot commits via
`IcebergTransaction`.
- Implemented conflict detection and resolution utilities for concurrent
modifications (`IcebergConflictDetectionFilterUtils`).
- Added support for Iceberg metadata columns (e.g., `_file_path`,
`_pos`) via `IcebergMetadataColumn` to accurately locate deleting
records.
3. **Backend Execution Sinks**:
- Introduced `VIcebergDeleteSink` and `VIcebergMergeSink` sink
operators.
- Added `VIcebergDeleteFileWriter` mechanism to support writing Position
Delete files natively inside Doris BE.
- Implemented routing and data shuffling logic via `MergePartitioner`
and `IcebergPartitionFunction` to properly align input rows with
Iceberg's partitioning constraints during writes.
4. **Scanner & Execution Decoupling**:
- Adjusted `FileScanner` and `GenericReader` to natively support
row-level operations (like generating `$row_id`), feeding precise
coordinates for DML modifications.
5. **Quality Assurance**:
- Added comprehensive logic/physical plan tests (e.g.
`IcebergDDLAndDMLPlanTest`).
- Extensive coverage of the sink components in BE Unit Tests
(`viceberg_merge_sink_test.cpp`, `viceberg_delete_sink_test.cpp`, etc.).
- Introduced standard regression test suites covering basic/advanced
usages and edge cases for merge/update/delete
(`regression-test/suites/external_table_p0/iceberg/dml/*`).
---
be/src/common/consts.h | 1 +
be/src/exec/exchange/exchange_writer.cpp | 2 +-
be/src/exec/operator/exchange_sink_operator.cpp | 28 +-
be/src/exec/operator/exchange_sink_operator.h | 2 +
.../exec/operator/iceberg_delete_sink_operator.cpp | 33 +
.../exec/operator/iceberg_delete_sink_operator.h | 87 +++
.../exec/operator/iceberg_merge_sink_operator.cpp | 35 +
be/src/exec/operator/iceberg_merge_sink_operator.h | 86 +++
be/src/exec/operator/operator.cpp | 6 +
be/src/exec/partitioner/partitioner.h | 28 +
be/src/exec/pipeline/pipeline_fragment_context.cpp | 18 +
be/src/exec/scan/file_scanner.cpp | 38 +-
be/src/exec/scan/file_scanner.h | 2 +
be/src/exec/sink/sink_common.h | 41 +
be/src/exec/sink/viceberg_delete_sink.cpp | 501 ++++++++++++
be/src/exec/sink/viceberg_delete_sink.h | 154 ++++
be/src/exec/sink/viceberg_merge_sink.cpp | 341 +++++++++
be/src/exec/sink/viceberg_merge_sink.h | 96 +++
.../writer/iceberg/viceberg_delete_file_writer.cpp | 155 ++++
.../writer/iceberg/viceberg_delete_file_writer.h | 123 +++
.../sink/writer/iceberg/viceberg_table_writer.cpp | 18 +
.../sink/writer/iceberg/viceberg_table_writer.h | 4 +
be/src/format/orc/vorc_reader.cpp | 144 +++-
be/src/format/orc/vorc_reader.h | 16 +-
be/src/format/parquet/vparquet_group_reader.cpp | 152 +++-
be/src/format/parquet/vparquet_group_reader.h | 13 +
be/src/format/parquet/vparquet_reader.cpp | 17 +
be/src/format/parquet/vparquet_reader.h | 4 +
be/src/format/table/deletion_vector_reader.h | 2 +
be/src/format/table/iceberg_reader.cpp | 25 +
be/src/format/table/iceberg_reader.h | 8 +
.../transformer/iceberg_partition_function.cpp | 401 ++++++++++
.../transformer/iceberg_partition_function.h | 101 +++
be/src/format/transformer/merge_partitioner.cpp | 365 +++++++++
be/src/format/transformer/merge_partitioner.h | 71 ++
be/src/format/transformer/writer_assigner.h | 127 ++++
be/test/core/value/merge_partitioner_test.cpp | 351 +++++++++
be/test/exec/exchange/exchange_writer_test.cpp | 66 ++
be/test/exec/sink/viceberg_delete_sink_test.cpp | 485 ++++++++++++
be/test/exec/sink/viceberg_merge_sink_test.cpp | 318 ++++++++
.../main/java/org/apache/doris/catalog/Column.java | 1 +
.../org/apache/doris/datasource/ExternalTable.java | 20 +-
.../apache/doris/datasource/FileQueryScanNode.java | 16 +-
.../IcebergConflictDetectionFilterUtils.java | 336 ++++++++
.../datasource/iceberg/IcebergExternalTable.java | 24 +-
.../datasource/iceberg/IcebergMergeOperation.java | 39 +
.../datasource/iceberg/IcebergMetadataColumn.java | 122 +++
.../datasource/iceberg/IcebergNereidsUtils.java | 134 +++-
.../doris/datasource/iceberg/IcebergRowId.java | 68 ++
.../datasource/iceberg/IcebergTransaction.java | 419 +++++++++-
.../doris/datasource/iceberg/IcebergUtils.java | 61 ++
.../iceberg/helper/IcebergWriterHelper.java | 73 ++
.../datasource/iceberg/source/IcebergScanNode.java | 21 +-
.../datasource/iceberg/source/IcebergSplit.java | 2 +
.../glue/translator/PhysicalPlanTranslator.java | 71 ++
.../nereids/properties/DistributionSpecMerge.java | 189 +++++
.../nereids/properties/RequestPropertyDeriver.java | 24 +
.../org/apache/doris/nereids/rules/RuleSet.java | 4 +
.../org/apache/doris/nereids/rules/RuleType.java | 4 +
.../nereids/rules/analysis/BindExpression.java | 97 +++
.../rules/expression/ExpressionRewrite.java | 9 +
...ebergDeleteSinkToPhysicalIcebergDeleteSink.java | 49 ++
...IcebergMergeSinkToPhysicalIcebergMergeSink.java | 49 ++
.../apache/doris/nereids/trees/plans/PlanType.java | 4 +
.../trees/plans/commands/DeleteFromCommand.java | 35 +
.../trees/plans/commands/ExplainCommand.java | 57 +-
.../trees/plans/commands/IcebergDeleteCommand.java | 288 +++++++
.../trees/plans/commands/IcebergMergeCommand.java | 519 +++++++++++++
.../trees/plans/commands/IcebergUpdateCommand.java | 305 ++++++++
.../trees/plans/commands/UpdateCommand.java | 31 +
.../commands/delete/DeleteCommandContext.java | 53 ++
.../commands/insert/IcebergDeleteExecutor.java | 105 +++
.../commands/insert/IcebergMergeExecutor.java | 85 +++
.../plans/commands/merge/MergeIntoCommand.java | 21 +-
.../plans/logical/LogicalIcebergDeleteSink.java | 151 ++++
.../plans/logical/LogicalIcebergMergeSink.java | 150 ++++
.../plans/physical/PhysicalIcebergDeleteSink.java | 175 +++++
.../plans/physical/PhysicalIcebergMergeSink.java | 338 +++++++++
.../nereids/trees/plans/visitor/SinkVisitor.java | 20 +
.../org/apache/doris/planner/DataPartition.java | 126 ++-
.../apache/doris/planner/IcebergDeleteSink.java | 139 ++++
.../org/apache/doris/planner/IcebergMergeSink.java | 179 +++++
.../java/org/apache/doris/qe/ConnectContext.java | 27 +
.../java/org/apache/doris/qe/SessionVariable.java | 12 +
.../datasource/hive/HiveDDLAndDMLPlanTest.java | 1 +
.../IcebergConflictDetectionFilterUtilsTest.java | 198 +++++
.../iceberg/IcebergDDLAndDMLPlanTest.java | 843 +++++++++++++++++++++
.../datasource/iceberg/IcebergDeletePlanTest.java | 539 +++++++++++++
.../iceberg/IcebergHiddenColumnTest.java | 82 ++
.../iceberg/IcebergMetadataColumnTest.java | 88 +++
.../doris/datasource/iceberg/IcebergUtilsTest.java | 15 +
.../iceberg/helper/IcebergWriterHelperTest.java | 153 ++++
.../plans/ExplainIcebergDeleteCommandTest.java | 241 ++++++
.../plans/commands/IcebergDeleteCommandTest.java | 62 ++
.../plans/commands/IcebergUpdateCommandTest.java | 137 ++++
.../commands/delete/DeleteCommandContextTest.java | 44 ++
gensrc/thrift/DataSinks.thrift | 52 ++
gensrc/thrift/Partitions.thrift | 26 +-
gensrc/thrift/PlanNodes.thrift | 2 +
.../dml/test_iceberg_merge_into_advanced.out | 135 ++++
.../iceberg/dml/test_iceberg_merge_into_basic.out | 33 +
.../dml/test_iceberg_update_delete_advanced.out | 97 +++
.../dml/test_iceberg_update_delete_basic.out | 61 ++
.../dml/test_iceberg_update_delete_basic.out.org | 77 ++
.../dml/test_iceberg_merge_into_advanced.groovy | 255 +++++++
.../dml/test_iceberg_merge_into_basic.groovy | 177 +++++
.../dml/test_iceberg_update_delete_advanced.groovy | 196 +++++
.../dml/test_iceberg_update_delete_basic.groovy | 153 ++++
108 files changed, 12724 insertions(+), 60 deletions(-)
diff --git a/be/src/common/consts.h b/be/src/common/consts.h
index 8e67f0c81fd..77d2acadf48 100644
--- a/be/src/common/consts.h
+++ b/be/src/common/consts.h
@@ -30,6 +30,7 @@ const std::string ROW_STORE_COL = "__DORIS_ROW_STORE_COL__";
const std::string DYNAMIC_COLUMN_NAME = "__DORIS_DYNAMIC_COL__";
const std::string PARTIAL_UPDATE_AUTO_INC_COL = "__PARTIAL_UPDATE_AUTO_INC_COLUMN__";
const std::string VIRTUAL_COLUMN_PREFIX = "__DORIS_VIRTUAL_COL__";
+const std::string ICEBERG_ROWID_COL = "__DORIS_ICEBERG_ROWID_COL__";
/// The maximum precision representable by a 4-byte decimal (Decimal4Value)
constexpr int MAX_DECIMAL32_PRECISION = 9;
diff --git a/be/src/exec/exchange/exchange_writer.cpp b/be/src/exec/exchange/exchange_writer.cpp
index 98437f74102..b712349d530 100644
--- a/be/src/exec/exchange/exchange_writer.cpp
+++ b/be/src/exec/exchange/exchange_writer.cpp
@@ -128,12 +128,12 @@ Status ExchangeOlapWriter::_write_impl(RuntimeState* state, Block* block, bool e
}
Status ExchangeTrivialWriter::write(RuntimeState* state, Block* block, bool eos) {
- auto rows = block->rows();
{
SCOPED_TIMER(_local_state.split_block_hash_compute_timer());
RETURN_IF_ERROR(_partitioner->do_partitioning(state, block));
}
{
+ auto rows = block->rows();
SCOPED_TIMER(_local_state.distribute_rows_into_channels_timer());
const auto& channel_ids = _partitioner->get_channel_ids();
diff --git a/be/src/exec/operator/exchange_sink_operator.cpp b/be/src/exec/operator/exchange_sink_operator.cpp
index 4cd23ba48b7..ffbe0780d87 100644
--- a/be/src/exec/operator/exchange_sink_operator.cpp
+++ b/be/src/exec/operator/exchange_sink_operator.cpp
@@ -41,6 +41,7 @@
#include "exec/sink/scale_writer_partitioning_exchanger.hpp"
#include "exec/sink/tablet_sink_hash_partitioner.h"
#include "exprs/vexpr.h"
+#include "format/transformer/merge_partitioner.h"
#include "runtime/runtime_profile.h"
#include "util/uid_util.h"
@@ -176,6 +177,20 @@ Status ExchangeSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& inf
RETURN_IF_ERROR(_partitioner->prepare(state, p._row_desc));
custom_profile()->add_info_string(
"Partitioner", fmt::format("ScaleWriterPartitioner({})", _partition_count));
+ } else if (_part_type == TPartitionType::MERGE_PARTITIONED) {
+ if (!p._has_merge_partition_info) {
+ return Status::InternalError("Merge partition info is missing");
+ }
+ _partition_count = channels.size();
+ const bool use_new_shuffle_hash_method =
+ _state->query_options().__isset.enable_new_shuffle_hash_method &&
+ _state->query_options().enable_new_shuffle_hash_method;
+ _partitioner = std::make_unique<MergePartitioner>(_partition_count, p._merge_partition_info,
+ use_new_shuffle_hash_method);
+ RETURN_IF_ERROR(_partitioner->init({}));
+ RETURN_IF_ERROR(_partitioner->prepare(state, p._row_desc));
+ custom_profile()->add_info_string("Partitioner",
+ fmt::format("MergePartitioner({})", _partition_count));
}
return Status::OK();
@@ -256,7 +271,8 @@ Status ExchangeSinkLocalState::open(RuntimeState* state) {
if (_part_type == TPartitionType::HASH_PARTITIONED ||
_part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED ||
_part_type == TPartitionType::HIVE_TABLE_SINK_HASH_PARTITIONED ||
- _part_type == TPartitionType::OLAP_TABLE_SINK_HASH_PARTITIONED) {
+ _part_type == TPartitionType::OLAP_TABLE_SINK_HASH_PARTITIONED ||
+ _part_type == TPartitionType::MERGE_PARTITIONED) {
RETURN_IF_ERROR(_partitioner->open(state));
}
return Status::OK();
@@ -299,13 +315,18 @@ ExchangeSinkOperatorX::ExchangeSinkOperatorX(
sink.output_partition.type == TPartitionType::OLAP_TABLE_SINK_HASH_PARTITIONED ||
sink.output_partition.type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED ||
sink.output_partition.type == TPartitionType::HIVE_TABLE_SINK_HASH_PARTITIONED ||
- sink.output_partition.type == TPartitionType::HIVE_TABLE_SINK_UNPARTITIONED);
+ sink.output_partition.type == TPartitionType::HIVE_TABLE_SINK_UNPARTITIONED ||
+ sink.output_partition.type == TPartitionType::MERGE_PARTITIONED);
#endif
_name = "ExchangeSinkOperatorX";
_pool = std::make_shared<ObjectPool>();
if (sink.__isset.output_tuple_id) {
_output_tuple_id = sink.output_tuple_id;
}
+ if (sink.output_partition.__isset.merge_partition_info) {
+ _merge_partition_info = sink.output_partition.merge_partition_info;
+ _has_merge_partition_info = true;
+ }
if (_part_type != TPartitionType::UNPARTITIONED) {
// if the destinations only one dest, we need to use broadcast
@@ -494,8 +515,9 @@ Status ExchangeSinkOperatorX::sink(RuntimeState* state, Block* block, bool eos)
(local_state.current_channel_idx + 1) % local_state.channels.size();
} else if (_part_type == TPartitionType::HASH_PARTITIONED ||
_part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED ||
+ _part_type == TPartitionType::OLAP_TABLE_SINK_HASH_PARTITIONED ||
_part_type == TPartitionType::HIVE_TABLE_SINK_HASH_PARTITIONED ||
- _part_type == TPartitionType::OLAP_TABLE_SINK_HASH_PARTITIONED) {
+ _part_type == TPartitionType::MERGE_PARTITIONED) {
RETURN_IF_ERROR(local_state._writer->write(state, block, eos));
} else if (_part_type == TPartitionType::HIVE_TABLE_SINK_UNPARTITIONED) {
// Control the number of channels according to the flow, thereby controlling the number of table sink writers.
diff --git a/be/src/exec/operator/exchange_sink_operator.h b/be/src/exec/operator/exchange_sink_operator.h
index d30acdf93e8..9824e3aaf88 100644
--- a/be/src/exec/operator/exchange_sink_operator.h
+++ b/be/src/exec/operator/exchange_sink_operator.h
@@ -243,6 +243,8 @@ private:
RuntimeState* _state = nullptr;
const std::vector<TExpr> _texprs;
+ TMergePartitionInfo _merge_partition_info;
+ bool _has_merge_partition_info = false;
const RowDescriptor& _row_desc;
TTupleId _output_tuple_id = -1;
diff --git a/be/src/exec/operator/iceberg_delete_sink_operator.cpp b/be/src/exec/operator/iceberg_delete_sink_operator.cpp
new file mode 100644
index 00000000000..5bb478c54e6
--- /dev/null
+++ b/be/src/exec/operator/iceberg_delete_sink_operator.cpp
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "iceberg_delete_sink_operator.h"
+
+#include "common/status.h"
+
+namespace doris {
+#include "common/compile_check_begin.h"
+Status IcebergDeleteSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) {
+ RETURN_IF_ERROR(Base::init(state, info));
+ SCOPED_TIMER(exec_time_counter());
+ SCOPED_TIMER(_init_timer);
+ auto& p = _parent->cast<Parent>();
+ RETURN_IF_ERROR(_writer->init_properties(p._pool));
+ return Status::OK();
+}
+
+} // namespace doris
diff --git a/be/src/exec/operator/iceberg_delete_sink_operator.h b/be/src/exec/operator/iceberg_delete_sink_operator.h
new file mode 100644
index 00000000000..f9cbbd60e31
--- /dev/null
+++ b/be/src/exec/operator/iceberg_delete_sink_operator.h
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "exec/sink/viceberg_delete_sink.h"
+#include "operator.h"
+
+namespace doris {
+#include "common/compile_check_begin.h"
+
+class IcebergDeleteSinkOperatorX;
+
+class IcebergDeleteSinkLocalState final
+ : public AsyncWriterSink<VIcebergDeleteSink, IcebergDeleteSinkOperatorX> {
+public:
+ using Base = AsyncWriterSink<VIcebergDeleteSink, IcebergDeleteSinkOperatorX>;
+ using Parent = IcebergDeleteSinkOperatorX;
+ ENABLE_FACTORY_CREATOR(IcebergDeleteSinkLocalState);
+ IcebergDeleteSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state)
+ : Base(parent, state) {};
+ Status init(RuntimeState* state, LocalSinkStateInfo& info) override;
+ Status open(RuntimeState* state) override {
+ SCOPED_TIMER(exec_time_counter());
+ SCOPED_TIMER(_open_timer);
+ return Base::open(state);
+ }
+ friend class IcebergDeleteSinkOperatorX;
+};
+
+class IcebergDeleteSinkOperatorX final : public DataSinkOperatorX<IcebergDeleteSinkLocalState> {
+public:
+ using Base = DataSinkOperatorX<IcebergDeleteSinkLocalState>;
+ IcebergDeleteSinkOperatorX(ObjectPool* pool, int operator_id, const RowDescriptor& row_desc,
+ const std::vector<TExpr>& t_output_expr)
+ : Base(operator_id, 0, 0),
+ _row_desc(row_desc),
+ _t_output_expr(t_output_expr),
+ _pool(pool) {};
+
+ Status init(const TDataSink& thrift_sink) override {
+ RETURN_IF_ERROR(Base::init(thrift_sink));
+ // From the thrift expressions create the real exprs.
+ RETURN_IF_ERROR(VExpr::create_expr_trees(_t_output_expr, _output_vexpr_ctxs));
+ return Status::OK();
+ }
+
+ Status prepare(RuntimeState* state) override {
+ RETURN_IF_ERROR(Base::prepare(state));
+ RETURN_IF_ERROR(VExpr::prepare(_output_vexpr_ctxs, state, _row_desc));
+ return VExpr::open(_output_vexpr_ctxs, state);
+ }
+
+ Status sink(RuntimeState* state, Block* in_block, bool eos) override {
+ auto& local_state = get_local_state(state);
+ SCOPED_TIMER(local_state.exec_time_counter());
+ COUNTER_UPDATE(local_state.rows_input_counter(), (int64_t)in_block->rows());
+ return local_state.sink(state, in_block, eos);
+ }
+
+private:
+ friend class IcebergDeleteSinkLocalState;
+ template <typename Writer, typename Parent>
+ requires(std::is_base_of_v<AsyncResultWriter, Writer>)
+ friend class AsyncWriterSink;
+ const RowDescriptor& _row_desc;
+ VExprContextSPtrs _output_vexpr_ctxs;
+ const std::vector<TExpr>& _t_output_expr;
+ ObjectPool* _pool = nullptr;
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris
diff --git a/be/src/exec/operator/iceberg_merge_sink_operator.cpp b/be/src/exec/operator/iceberg_merge_sink_operator.cpp
new file mode 100644
index 00000000000..dc077125528
--- /dev/null
+++ b/be/src/exec/operator/iceberg_merge_sink_operator.cpp
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "iceberg_merge_sink_operator.h"
+
+#include "common/status.h"
+
+namespace doris {
+#include "common/compile_check_begin.h"
+
+Status IcebergMergeSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) {
+ RETURN_IF_ERROR(Base::init(state, info));
+ SCOPED_TIMER(exec_time_counter());
+ SCOPED_TIMER(_init_timer);
+ auto& p = _parent->cast<Parent>();
+ RETURN_IF_ERROR(_writer->init_properties(p._pool, p._row_desc));
+ return Status::OK();
+}
+
+#include "common/compile_check_end.h"
+} // namespace doris
diff --git a/be/src/exec/operator/iceberg_merge_sink_operator.h b/be/src/exec/operator/iceberg_merge_sink_operator.h
new file mode 100644
index 00000000000..362d7ad7c45
--- /dev/null
+++ b/be/src/exec/operator/iceberg_merge_sink_operator.h
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "exec/sink/viceberg_merge_sink.h"
+#include "operator.h"
+
+namespace doris {
+#include "common/compile_check_begin.h"
+
+class IcebergMergeSinkOperatorX;
+
+class IcebergMergeSinkLocalState final
+ : public AsyncWriterSink<VIcebergMergeSink, IcebergMergeSinkOperatorX> {
+public:
+ using Base = AsyncWriterSink<VIcebergMergeSink, IcebergMergeSinkOperatorX>;
+ using Parent = IcebergMergeSinkOperatorX;
+ ENABLE_FACTORY_CREATOR(IcebergMergeSinkLocalState);
+ IcebergMergeSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state)
+ : Base(parent, state) {};
+ Status init(RuntimeState* state, LocalSinkStateInfo& info) override;
+ Status open(RuntimeState* state) override {
+ SCOPED_TIMER(exec_time_counter());
+ SCOPED_TIMER(_open_timer);
+ return Base::open(state);
+ }
+ friend class IcebergMergeSinkOperatorX;
+};
+
+class IcebergMergeSinkOperatorX final : public DataSinkOperatorX<IcebergMergeSinkLocalState> {
+public:
+ using Base = DataSinkOperatorX<IcebergMergeSinkLocalState>;
+ IcebergMergeSinkOperatorX(ObjectPool* pool, int operator_id, const RowDescriptor& row_desc,
+ const std::vector<TExpr>& t_output_expr)
+ : Base(operator_id, 0, 0),
+ _row_desc(row_desc),
+ _t_output_expr(t_output_expr),
+ _pool(pool) {};
+
+ Status init(const TDataSink& thrift_sink) override {
+ RETURN_IF_ERROR(Base::init(thrift_sink));
+ RETURN_IF_ERROR(VExpr::create_expr_trees(_t_output_expr, _output_vexpr_ctxs));
+ return Status::OK();
+ }
+
+ Status prepare(RuntimeState* state) override {
+ RETURN_IF_ERROR(Base::prepare(state));
+ RETURN_IF_ERROR(VExpr::prepare(_output_vexpr_ctxs, state, _row_desc));
+ return VExpr::open(_output_vexpr_ctxs, state);
+ }
+
+ Status sink(RuntimeState* state, Block* in_block, bool eos) override {
+ auto& local_state = get_local_state(state);
+ SCOPED_TIMER(local_state.exec_time_counter());
+ COUNTER_UPDATE(local_state.rows_input_counter(), (int64_t)in_block->rows());
+ return local_state.sink(state, in_block, eos);
+ }
+
+private:
+ friend class IcebergMergeSinkLocalState;
+ template <typename Writer, typename Parent>
+ requires(std::is_base_of_v<AsyncResultWriter, Writer>)
+ friend class AsyncWriterSink;
+ const RowDescriptor& _row_desc;
+ VExprContextSPtrs _output_vexpr_ctxs;
+ const std::vector<TExpr>& _t_output_expr;
+ ObjectPool* _pool = nullptr;
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris
diff --git a/be/src/exec/operator/operator.cpp b/be/src/exec/operator/operator.cpp
index 2a55fef7142..16cc969276d 100644
--- a/be/src/exec/operator/operator.cpp
+++ b/be/src/exec/operator/operator.cpp
@@ -42,6 +42,8 @@
#include "exec/operator/hashjoin_build_sink.h"
#include "exec/operator/hashjoin_probe_operator.h"
#include "exec/operator/hive_table_sink_operator.h"
+#include "exec/operator/iceberg_delete_sink_operator.h"
+#include "exec/operator/iceberg_merge_sink_operator.h"
#include "exec/operator/iceberg_table_sink_operator.h"
#include "exec/operator/jdbc_scan_operator.h"
#include "exec/operator/jdbc_table_sink_operator.h"
@@ -789,6 +791,8 @@ DECLARE_OPERATOR(HiveTableSinkLocalState)
DECLARE_OPERATOR(TVFTableSinkLocalState)
DECLARE_OPERATOR(IcebergTableSinkLocalState)
DECLARE_OPERATOR(SpillIcebergTableSinkLocalState)
+DECLARE_OPERATOR(IcebergDeleteSinkLocalState)
+DECLARE_OPERATOR(IcebergMergeSinkLocalState)
DECLARE_OPERATOR(MCTableSinkLocalState)
DECLARE_OPERATOR(AnalyticSinkLocalState)
DECLARE_OPERATOR(BlackholeSinkLocalState)
@@ -903,6 +907,8 @@ template class AsyncWriterSink<doris::VTabletWriterV2, OlapTableSinkV2OperatorX>
template class AsyncWriterSink<doris::VHiveTableWriter, HiveTableSinkOperatorX>;
template class AsyncWriterSink<doris::VIcebergTableWriter, IcebergTableSinkOperatorX>;
template class AsyncWriterSink<doris::VIcebergTableWriter, SpillIcebergTableSinkOperatorX>;
+template class AsyncWriterSink<doris::VIcebergDeleteSink, IcebergDeleteSinkOperatorX>;
+template class AsyncWriterSink<doris::VIcebergMergeSink, IcebergMergeSinkOperatorX>;
template class AsyncWriterSink<doris::VMCTableWriter, MCTableSinkOperatorX>;
template class AsyncWriterSink<doris::VTVFTableWriter, TVFTableSinkOperatorX>;
diff --git a/be/src/exec/partitioner/partitioner.h b/be/src/exec/partitioner/partitioner.h
index 4eaed892cb8..15c2b578ef8 100644
--- a/be/src/exec/partitioner/partitioner.h
+++ b/be/src/exec/partitioner/partitioner.h
@@ -56,6 +56,34 @@ protected:
const HashValType _partition_count;
};
+class PartitionFunction {
+public:
+ using HashValType = PartitionerBase::HashValType;
+
+ virtual ~PartitionFunction() = default;
+
+ virtual Status init(const std::vector<TExpr>& texprs) = 0;
+
+ virtual Status prepare(RuntimeState* state, const RowDescriptor& row_desc) = 0;
+
+ virtual Status open(RuntimeState* state) = 0;
+
+ virtual Status close(RuntimeState* state) = 0;
+
+ virtual Status get_partitions(RuntimeState* state, Block* block, size_t partition_count,
+ std::vector<HashValType>& partitions) const = 0;
+
+ virtual HashValType partition_count() const = 0;
+
+ virtual Status clone(RuntimeState* state,
+ std::unique_ptr<PartitionFunction>& function) const = 0;
+};
+
+enum class ShuffleHashMethod {
+ CRC32,
+ CRC32C,
+};
+
template <typename ChannelIds>
class Crc32HashPartitioner : public PartitionerBase {
public:
diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp
index 3e3b0fe0778..dc08bbbd989 100644
--- a/be/src/exec/pipeline/pipeline_fragment_context.cpp
+++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp
@@ -64,6 +64,8 @@
#include "exec/operator/hashjoin_build_sink.h"
#include "exec/operator/hashjoin_probe_operator.h"
#include "exec/operator/hive_table_sink_operator.h"
+#include "exec/operator/iceberg_delete_sink_operator.h"
+#include "exec/operator/iceberg_merge_sink_operator.h"
#include "exec/operator/iceberg_table_sink_operator.h"
#include "exec/operator/jdbc_scan_operator.h"
#include "exec/operator/jdbc_table_sink_operator.h"
@@ -1096,6 +1098,22 @@ Status PipelineFragmentContext::_create_data_sink(ObjectPool* pool, const TDataS
}
break;
}
+ case TDataSinkType::ICEBERG_DELETE_SINK: {
+ if (!thrift_sink.__isset.iceberg_delete_sink) {
+ return Status::InternalError("Missing iceberg delete sink.");
+ }
+ _sink = std::make_shared<IcebergDeleteSinkOperatorX>(pool, next_sink_operator_id(),
+ row_desc, output_exprs);
+ break;
+ }
+ case TDataSinkType::ICEBERG_MERGE_SINK: {
+ if (!thrift_sink.__isset.iceberg_merge_sink) {
+ return Status::InternalError("Missing iceberg merge sink.");
+ }
+ _sink = std::make_shared<IcebergMergeSinkOperatorX>(pool, next_sink_operator_id(), row_desc,
+ output_exprs);
+ break;
+ }
case TDataSinkType::MAXCOMPUTE_TABLE_SINK: {
if (!thrift_sink.__isset.max_compute_table_sink) {
return Status::InternalError("Missing max compute table sink.");
diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp
index cd8a57126ba..a547ee4e7c8 100644
--- a/be/src/exec/scan/file_scanner.cpp
+++ b/be/src/exec/scan/file_scanner.cpp
@@ -35,6 +35,7 @@
#include "common/compiler_util.h" // IWYU pragma: keep
#include "common/config.h"
+#include "common/consts.h"
#include "common/logging.h"
#include "common/status.h"
#include "core/block/column_with_type_and_name.h"
@@ -456,6 +457,12 @@ Status FileScanner::_get_block_wrapped(RuntimeState* state, Block* block, bool*
// For query job, simply set _src_block_ptr to block.
size_t read_rows = 0;
RETURN_IF_ERROR(_init_src_block(block));
+ if (_need_iceberg_rowid_column && _current_range.__isset.table_format_params &&
+ _current_range.table_format_params.table_format_type == "iceberg") {
+ if (auto* iceberg_reader = dynamic_cast<IcebergTableReader*>(_cur_reader.get())) {
+ iceberg_reader->set_row_id_column_position(_iceberg_rowid_column_pos);
+ }
+ }
{
SCOPED_TIMER(_get_block_timer);
@@ -529,8 +536,22 @@ Status FileScanner::_init_src_block(Block* block) {
if (!_is_load) {
_src_block_ptr = block;
+ bool update_name_to_idx = _src_block_name_to_idx.empty();
+ _iceberg_rowid_column_pos = -1;
+ if (_need_iceberg_rowid_column && _current_range.__isset.table_format_params &&
+ _current_range.table_format_params.table_format_type == "iceberg") {
+ int row_id_idx = block->get_position_by_name(BeConsts::ICEBERG_ROWID_COL);
+ if (row_id_idx >= 0) {
+ _iceberg_rowid_column_pos = row_id_idx;
+ if (!update_name_to_idx &&
+ !_src_block_name_to_idx.contains(BeConsts::ICEBERG_ROWID_COL)) {
+ update_name_to_idx = true;
+ }
+ }
+ }
+
// Build name to index map only once on first call
- if (_src_block_name_to_idx.empty()) {
+ if (update_name_to_idx) {
_src_block_name_to_idx = block->get_name_to_pos_map();
}
return Status::OK();
@@ -1221,6 +1242,9 @@ Status FileScanner::_init_parquet_reader(std::unique_ptr<ParquetReader>&& parque
std::unique_ptr<IcebergParquetReader> iceberg_reader = IcebergParquetReader::create_unique(
std::move(parquet_reader), _profile, _state, *_params, range, _kv_cache,
_io_ctx.get(), file_meta_cache_ptr);
+ if (_need_iceberg_rowid_column) {
+ iceberg_reader->set_need_row_id_column(true);
+ }
init_status = iceberg_reader->init_reader(
_file_col_names, &_src_block_name_to_idx, _push_down_conjuncts,
slot_id_to_predicates, _real_tuple_desc, _default_val_row_desc.get(),
@@ -1336,6 +1360,9 @@ Status FileScanner::_init_orc_reader(std::unique_ptr<OrcReader>&& orc_reader,
std::move(orc_reader), _profile, _state, *_params, range, _kv_cache, _io_ctx.get(),
file_meta_cache_ptr);
+ if (_need_iceberg_rowid_column) {
+ iceberg_reader->set_need_row_id_column(true);
+ }
init_status = iceberg_reader->init_reader(
_file_col_names, &_src_block_name_to_idx, _push_down_conjuncts, _real_tuple_desc,
_default_val_row_desc.get(), _col_name_to_slot_id,
@@ -1423,6 +1450,11 @@ Status FileScanner::_set_fill_or_truncate_columns(bool need_to_get_parsed_schema
_slot_lower_name_to_col_type.clear();
std::unordered_map<std::string, DataTypePtr> name_to_col_type;
RETURN_IF_ERROR(_cur_reader->get_columns(&name_to_col_type, &_missing_cols));
+ if (_need_iceberg_rowid_column && _current_range.__isset.table_format_params &&
+ _current_range.table_format_params.table_format_type == "iceberg") {
+ _missing_cols.erase(BeConsts::ICEBERG_ROWID_COL);
+ _missing_cols.erase(to_lower(BeConsts::ICEBERG_ROWID_COL));
+ }
for (const auto& [col_name, col_type] : name_to_col_type) {
auto col_name_lower = to_lower(col_name);
if (_partition_col_descs.contains(col_name_lower)) {
@@ -1667,6 +1699,10 @@ Status FileScanner::_init_expr_ctxes() {
_row_id_column_iterator_pair.second = _default_val_row_desc->get_column_id(slot_id);
continue;
}
+ if (it->second->col_name() == BeConsts::ICEBERG_ROWID_COL) {
+ _need_iceberg_rowid_column = true;
+ continue;
+ }
if (slot_info.is_file_slot) {
_is_file_slot.emplace(slot_id);
diff --git a/be/src/exec/scan/file_scanner.h b/be/src/exec/scan/file_scanner.h
index e8c4adf3c14..5f68599d1ed 100644
--- a/be/src/exec/scan/file_scanner.h
+++ b/be/src/exec/scan/file_scanner.h
@@ -229,6 +229,8 @@ private:
std::pair<std::shared_ptr<RowIdColumnIteratorV2>, int> _row_id_column_iterator_pair = {nullptr,
-1};
+ bool _need_iceberg_rowid_column = false;
+ int _iceberg_rowid_column_pos = -1;
int64_t _last_bytes_read_from_local = 0;
int64_t _last_bytes_read_from_remote = 0;
diff --git a/be/src/exec/sink/sink_common.h b/be/src/exec/sink/sink_common.h
new file mode 100644
index 00000000000..e0bf0ad5fbf
--- /dev/null
+++ b/be/src/exec/sink/sink_common.h
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstdint>
+
+namespace doris {
+
+constexpr int8_t kInsertOperation = 1;
+constexpr int8_t kDeleteOperation = 2;
+constexpr int8_t kUpdateOperation = 3;
+constexpr int8_t kUpdateInsertOperation = 4;
+constexpr int8_t kUpdateDeleteOperation = 5;
+
+// The standard operation column name for merge sinks
+constexpr const char* kOperationColumnName = "operation";
+
+inline bool is_delete_op(int8_t op) {
+ return op == kDeleteOperation || op == kUpdateDeleteOperation || op == kUpdateOperation;
+}
+
+inline bool is_insert_op(int8_t op) {
+ return op == kInsertOperation || op == kUpdateInsertOperation || op == kUpdateOperation;
+}
+
+} // namespace doris
diff --git a/be/src/exec/sink/viceberg_delete_sink.cpp b/be/src/exec/sink/viceberg_delete_sink.cpp
new file mode 100644
index 00000000000..fd280af8be6
--- /dev/null
+++ b/be/src/exec/sink/viceberg_delete_sink.cpp
@@ -0,0 +1,501 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exec/sink/viceberg_delete_sink.h"
+
+#include <fmt/format.h>
+
+#include "common/logging.h"
+#include "core/block/column_with_type_and_name.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_string.h"
+#include "core/column/column_struct.h"
+#include "core/column/column_vector.h"
+#include "core/data_type/data_type_factory.hpp"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_struct.h"
+#include "exprs/vexpr.h"
+#include "format/transformer/vfile_format_transformer.h"
+#include "runtime/runtime_state.h"
+#include "util/string_util.h"
+#include "util/uid_util.h"
+
+namespace doris {
+
+VIcebergDeleteSink::VIcebergDeleteSink(const TDataSink& t_sink,
+ const VExprContextSPtrs& output_exprs,
+ std::shared_ptr<Dependency> dep,
+ std::shared_ptr<Dependency> fin_dep)
+ : AsyncResultWriter(output_exprs, dep, fin_dep), _t_sink(t_sink) {
+ DCHECK(_t_sink.__isset.iceberg_delete_sink);
+}
+
+Status VIcebergDeleteSink::init_properties(ObjectPool* pool) {
+ const auto& delete_sink = _t_sink.iceberg_delete_sink;
+
+ _delete_type = delete_sink.delete_type;
+ if (_delete_type != TFileContent::POSITION_DELETES) {
+ return Status::NotSupported("Iceberg delete only supports position delete files");
+ }
+
+ // Get file format settings
+ if (delete_sink.__isset.file_format) {
+ _file_format_type = delete_sink.file_format;
+ }
+
+ if (delete_sink.__isset.compress_type) {
+ _compress_type = delete_sink.compress_type;
+ }
+
+ // Get output path and table location
+ if (delete_sink.__isset.output_path) {
+ _output_path = delete_sink.output_path;
+ }
+
+ if (delete_sink.__isset.table_location) {
+ _table_location = delete_sink.table_location;
+ }
+
+ // Get Hadoop configuration
+ if (delete_sink.__isset.hadoop_config) {
+ _hadoop_conf.insert(delete_sink.hadoop_config.begin(), delete_sink.hadoop_config.end());
+ }
+
+ if (delete_sink.__isset.file_type) {
+ _file_type = delete_sink.file_type;
+ }
+
+ if (delete_sink.__isset.broker_addresses) {
+ _broker_addresses.assign(delete_sink.broker_addresses.begin(),
+ delete_sink.broker_addresses.end());
+ }
+
+ // Get partition information
+ if (delete_sink.__isset.partition_spec_id) {
+ _partition_spec_id = delete_sink.partition_spec_id;
+ }
+
+ if (delete_sink.__isset.partition_data_json) {
+ _partition_data_json = delete_sink.partition_data_json;
+ }
+
+ return Status::OK();
+}
+
+Status VIcebergDeleteSink::open(RuntimeState* state, RuntimeProfile* profile) {
+ _state = state;
+
+ // Initialize counters
+ _written_rows_counter = ADD_COUNTER(profile, "RowsWritten", TUnit::UNIT);
+ _send_data_timer = ADD_TIMER(profile, "SendDataTime");
+ _write_delete_files_timer = ADD_TIMER(profile, "WriteDeleteFilesTime");
+ _delete_file_count_counter = ADD_COUNTER(profile, "DeleteFileCount", TUnit::UNIT);
+ _open_timer = ADD_TIMER(profile, "OpenTime");
+ _close_timer = ADD_TIMER(profile, "CloseTime");
+
+ SCOPED_TIMER(_open_timer);
+
+ RETURN_IF_ERROR(_init_position_delete_output_exprs());
+
+ LOG(INFO) << fmt::format("VIcebergDeleteSink opened: delete_type={}, output_path={}",
+ to_string(_delete_type), _output_path);
+
+ return Status::OK();
+}
+
+Status VIcebergDeleteSink::write(RuntimeState* state, Block& block) {
+ SCOPED_TIMER(_send_data_timer);
+
+ if (block.rows() == 0) {
+ return Status::OK();
+ }
+
+ _row_count += block.rows();
+
+ if (_delete_type != TFileContent::POSITION_DELETES) {
+ return Status::NotSupported("Iceberg delete only supports position delete files");
+ }
+
+ // Extract $row_id column and group by file_path
+ RETURN_IF_ERROR(_collect_position_deletes(block, _file_deletions));
+
+ if (_written_rows_counter) {
+ COUNTER_UPDATE(_written_rows_counter, block.rows());
+ }
+
+ return Status::OK();
+}
+
+Status VIcebergDeleteSink::close(Status close_status) {
+ SCOPED_TIMER(_close_timer);
+
+ if (!close_status.ok()) {
+ LOG(WARNING) << fmt::format("VIcebergDeleteSink close with error: {}",
+ close_status.to_string());
+ return close_status;
+ }
+
+ if (_delete_type == TFileContent::POSITION_DELETES && !_file_deletions.empty()) {
+ SCOPED_TIMER(_write_delete_files_timer);
+ RETURN_IF_ERROR(_write_position_delete_files(_file_deletions));
+ }
+
+ // Update counters
+ if (_delete_file_count_counter) {
+ COUNTER_UPDATE(_delete_file_count_counter, _delete_file_count);
+ }
+
+ LOG(INFO) << fmt::format("VIcebergDeleteSink closed: rows={}, delete_files={}", _row_count,
+ _delete_file_count);
+
+ if (_state != nullptr) {
+ for (const auto& commit_data : _commit_data_list) {
+ _state->add_iceberg_commit_datas(commit_data);
+ }
+ }
+
+ return Status::OK();
+}
+
+int VIcebergDeleteSink::_get_row_id_column_index(const Block& block) {
+ // Find __DORIS_ICEBERG_ROWID_COL__ column in block
+ for (size_t i = 0; i < block.columns(); ++i) {
+ const auto& col_name = block.get_by_position(i).name;
+ if (col_name == doris::BeConsts::ICEBERG_ROWID_COL) {
+ return static_cast<int>(i);
+ }
+ }
+ return -1;
+}
+
+Status VIcebergDeleteSink::_collect_position_deletes(
+ const Block& block, std::map<std::string, IcebergFileDeletion>& file_deletions) {
+ // Find row id column
+ int row_id_col_idx = _get_row_id_column_index(block);
+ if (row_id_col_idx < 0) {
+ return Status::InternalError(
+ "__DORIS_ICEBERG_ROWID_COL__ column not found in block for position delete");
+ }
+
+ const auto& row_id_col = block.get_by_position(row_id_col_idx);
+ const IColumn* row_id_data = row_id_col.column.get();
+ const IDataType* row_id_type = row_id_col.type.get();
+ const auto* nullable_col = check_and_get_column<ColumnNullable>(row_id_data);
+ if (nullable_col != nullptr) {
+ row_id_data = nullable_col->get_nested_column_ptr().get();
+ }
+ const auto* nullable_type = check_and_get_data_type<DataTypeNullable>(row_id_type);
+ if (nullable_type != nullptr) {
+ row_id_type = nullable_type->get_nested_type().get();
+ }
+ const auto* struct_col = check_and_get_column<ColumnStruct>(row_id_data);
+ const auto* struct_type = check_and_get_data_type<DataTypeStruct>(row_id_type);
+ if (!struct_col || !struct_type) {
+ return Status::InternalError("__DORIS_ICEBERG_ROWID_COL__ column is not a struct column");
+ }
+
+ // __DORIS_ICEBERG_ROWID_COL__ struct:
+ // (file_path: STRING, row_position: BIGINT, partition_spec_id: INT, partition_data: STRING)
+ size_t field_count = struct_col->tuple_size();
+ if (field_count < 2) {
+ return Status::InternalError(
+ "__DORIS_ICEBERG_ROWID_COL__ struct must have at least 2 fields "
+ "(file_path, row_position)");
+ }
+
+ auto normalize = [](const std::string& name) { return doris::to_lower(name); };
+
+ int file_path_idx = -1;
+ int row_position_idx = -1;
+ int spec_id_idx = -1;
+ int partition_data_idx = -1;
+ const auto& field_names = struct_type->get_element_names();
+ for (size_t i = 0; i < field_names.size(); ++i) {
+ std::string name = normalize(field_names[i]);
+ if (file_path_idx < 0 && name == "file_path") {
+ file_path_idx = static_cast<int>(i);
+ } else if (row_position_idx < 0 && name == "row_position") {
+ row_position_idx = static_cast<int>(i);
+ } else if (spec_id_idx < 0 && name == "partition_spec_id") {
+ spec_id_idx = static_cast<int>(i);
+ } else if (partition_data_idx < 0 && name == "partition_data") {
+ partition_data_idx = static_cast<int>(i);
+ }
+ }
+
+ if (file_path_idx < 0 || row_position_idx < 0) {
+ return Status::InternalError(
+ "__DORIS_ICEBERG_ROWID_COL__ must contain standard fields file_path and "
+ "row_position");
+ }
+ if (field_count >= 3 && spec_id_idx < 0) {
+ return Status::InternalError(
+ "__DORIS_ICEBERG_ROWID_COL__ must use standard field name partition_spec_id");
+ }
+ if (field_count >= 4 && partition_data_idx < 0) {
+ return Status::InternalError(
+ "__DORIS_ICEBERG_ROWID_COL__ must use standard field name partition_data");
+ }
+
+ const auto* file_path_col = check_and_get_column<ColumnString>(
+ remove_nullable(struct_col->get_column_ptr(file_path_idx)).get());
+ const auto* row_position_col = check_and_get_column<ColumnVector<TYPE_BIGINT>>(
+ remove_nullable(struct_col->get_column_ptr(row_position_idx)).get());
+
+ if (!file_path_col || !row_position_col) {
+ return Status::InternalError(
+ "__DORIS_ICEBERG_ROWID_COL__ struct fields have incorrect types");
+ }
+
+ const ColumnVector<TYPE_INT>* spec_id_col = nullptr;
+ const ColumnString* partition_data_col = nullptr;
+ if (spec_id_idx >= 0 && spec_id_idx < static_cast<int>(field_count)) {
+ spec_id_col = check_and_get_column<ColumnVector<TYPE_INT>>(
+ remove_nullable(struct_col->get_column_ptr(spec_id_idx)).get());
+ if (!spec_id_col) {
+ return Status::InternalError(
+ "__DORIS_ICEBERG_ROWID_COL__ partition_spec_id has incorrect type");
+ }
+ }
+ if (partition_data_idx >= 0 && partition_data_idx < static_cast<int>(field_count)) {
+ partition_data_col = check_and_get_column<ColumnString>(
+ remove_nullable(struct_col->get_column_ptr(partition_data_idx)).get());
+ if (!partition_data_col) {
+ return Status::InternalError(
+ "__DORIS_ICEBERG_ROWID_COL__ partition_data has incorrect type");
+ }
+ }
+
+ // Group by file_path using roaring bitmap
+ for (size_t i = 0; i < block.rows(); ++i) {
+ std::string file_path = file_path_col->get_data_at(i).to_string();
+ int64_t row_position = row_position_col->get_element(i);
+ if (row_position < 0) {
+ return Status::InternalError("Invalid row_position {} in row_id column", row_position);
+ }
+
+ int32_t partition_spec_id = _partition_spec_id;
+ std::string partition_data_json = _partition_data_json;
+ if (spec_id_col != nullptr) {
+ partition_spec_id = spec_id_col->get_element(i);
+ }
+ if (partition_data_col != nullptr) {
+ partition_data_json = partition_data_col->get_data_at(i).to_string();
+ }
+
+ auto [iter, inserted] = file_deletions.emplace(
+ file_path, IcebergFileDeletion(partition_spec_id, partition_data_json));
+ if (!inserted) {
+ if (iter->second.partition_spec_id != partition_spec_id ||
+ iter->second.partition_data_json != partition_data_json) {
+ LOG(WARNING) << fmt::format(
+ "Mismatched partition info for file {}, existing spec_id={}, data={}, "
+ "new spec_id={}, data={}",
+ file_path, iter->second.partition_spec_id, iter->second.partition_data_json,
+ partition_spec_id, partition_data_json);
+ }
+ }
+ iter->second.rows_to_delete.add(static_cast<uint64_t>(row_position));
+ }
+
+ return Status::OK();
+}
+
+Status VIcebergDeleteSink::_write_position_delete_files(
+ const std::map<std::string, IcebergFileDeletion>& file_deletions) {
+ constexpr size_t kBatchSize = 4096;
+ for (const auto& [data_file_path, deletion] : file_deletions) {
+ if (deletion.rows_to_delete.isEmpty()) {
+ continue;
+ }
+ // Generate unique delete file path
+ std::string delete_file_path = _generate_delete_file_path(data_file_path);
+
+ // Create delete file writer
+ auto writer = VIcebergDeleteFileWriterFactory::create_writer(
+ TFileContent::POSITION_DELETES, delete_file_path, _file_format_type,
+ _compress_type);
+
+ // Build column names for position delete
+ std::vector<std::string> column_names = {"file_path", "pos"};
+
+ if (_position_delete_output_expr_ctxs.empty()) {
+ RETURN_IF_ERROR(_init_position_delete_output_exprs());
+ }
+
+ // Open writer
+ RETURN_IF_ERROR(writer->open(_state, _state->runtime_profile(),
+ _position_delete_output_expr_ctxs, column_names, _hadoop_conf,
+ _file_type, _broker_addresses));
+
+ // Build block with (file_path, pos) columns
+ std::vector<int64_t> positions;
+ positions.reserve(kBatchSize);
+ for (auto it = deletion.rows_to_delete.begin(); it != deletion.rows_to_delete.end(); ++it) {
+ positions.push_back(static_cast<int64_t>(*it));
+ if (positions.size() >= kBatchSize) {
+ Block delete_block;
+ RETURN_IF_ERROR(
+ _build_position_delete_block(data_file_path, positions, delete_block));
+ RETURN_IF_ERROR(writer->write(delete_block));
+ positions.clear();
+ }
+ }
+ if (!positions.empty()) {
+ Block delete_block;
+ RETURN_IF_ERROR(_build_position_delete_block(data_file_path, positions, delete_block));
+ RETURN_IF_ERROR(writer->write(delete_block));
+ }
+
+ // Set partition info on writer before close
+ writer->set_partition_info(deletion.partition_spec_id, deletion.partition_data_json);
+
+ // Close writer and collect commit data
+ TIcebergCommitData commit_data;
+ RETURN_IF_ERROR(writer->close(commit_data));
+
+ // Set referenced data file path
+ commit_data.__set_referenced_data_file_path(data_file_path);
+
+ _commit_data_list.push_back(commit_data);
+ _delete_file_count++;
+
+ VLOG(1) << fmt::format("Written position delete file: path={}, rows={}, referenced_file={}",
+ delete_file_path, commit_data.row_count, data_file_path);
+ }
+
+ return Status::OK();
+}
+
+Status VIcebergDeleteSink::_init_position_delete_output_exprs() {
+ if (!_position_delete_output_expr_ctxs.empty()) {
+ return Status::OK();
+ }
+
+ std::vector<TExpr> texprs;
+ texprs.reserve(2);
+
+ std::string empty_string;
+ TExprNode file_path_node =
+ create_texpr_node_from(&empty_string, PrimitiveType::TYPE_STRING, 0, 0);
+ file_path_node.__set_num_children(0);
+ file_path_node.__set_output_scale(0);
+ file_path_node.__set_is_nullable(false);
+ TExpr file_path_expr;
+ file_path_expr.nodes.emplace_back(std::move(file_path_node));
+ texprs.emplace_back(std::move(file_path_expr));
+
+ int64_t zero = 0;
+ TExprNode pos_node = create_texpr_node_from(&zero, PrimitiveType::TYPE_BIGINT, 0, 0);
+ pos_node.__set_num_children(0);
+ pos_node.__set_output_scale(0);
+ pos_node.__set_is_nullable(false);
+ TExpr pos_expr;
+ pos_expr.nodes.emplace_back(std::move(pos_node));
+ texprs.emplace_back(std::move(pos_expr));
+
+ RETURN_IF_ERROR(VExpr::create_expr_trees(texprs, _position_delete_output_expr_ctxs));
+ return Status::OK();
+}
+
+Status VIcebergDeleteSink::_build_position_delete_block(const std::string& file_path,
+ const std::vector<int64_t>& positions,
+ Block& output_block) {
+ // Create file_path column (repeated for each position)
+ auto file_path_col = ColumnString::create();
+ for (size_t i = 0; i < positions.size(); ++i) {
+ file_path_col->insert_data(file_path.data(), file_path.size());
+ }
+
+ // Create pos column
+ auto pos_col = ColumnVector<TYPE_BIGINT>::create();
+ pos_col->get_data().assign(positions.begin(), positions.end());
+
+ // Build block
+ output_block.insert(ColumnWithTypeAndName(std::move(file_path_col),
+ std::make_shared<DataTypeString>(), "file_path"));
+ output_block.insert(
+ ColumnWithTypeAndName(std::move(pos_col), std::make_shared<DataTypeInt64>(), "pos"));
+
+ return Status::OK();
+}
+
+std::string VIcebergDeleteSink::_get_file_extension() const {
+ std::string compress_name;
+ switch (_compress_type) {
+ case TFileCompressType::SNAPPYBLOCK: {
+ compress_name = ".snappy";
+ break;
+ }
+ case TFileCompressType::ZLIB: {
+ compress_name = ".zlib";
+ break;
+ }
+ case TFileCompressType::ZSTD: {
+ compress_name = ".zstd";
+ break;
+ }
+ default: {
+ compress_name = "";
+ break;
+ }
+ }
+
+ std::string file_format_name;
+ switch (_file_format_type) {
+ case TFileFormatType::FORMAT_PARQUET: {
+ file_format_name = ".parquet";
+ break;
+ }
+ case TFileFormatType::FORMAT_ORC: {
+ file_format_name = ".orc";
+ break;
+ }
+ default: {
+ file_format_name = "";
+ break;
+ }
+ }
+ return fmt::format("{}{}", compress_name, file_format_name);
+}
+
+std::string VIcebergDeleteSink::_generate_delete_file_path(
+ const std::string& referenced_data_file) {
+ // Generate unique delete file name using UUID
+ std::string uuid = generate_uuid_string();
+ std::string file_name;
+
+ std::string file_extension = _get_file_extension();
+ file_name =
+ fmt::format("delete_pos_{}_{}{}", uuid,
+ std::hash<std::string> {}(referenced_data_file) % 10000000, file_extension);
+
+ // Combine with output path or table location
+ std::string base_path = _output_path.empty() ? _table_location : _output_path;
+
+ // Ensure base path ends with /
+ if (!base_path.empty() && base_path.back() != '/') {
+ base_path += '/';
+ }
+
+ // Delete files are data files in Iceberg, write under data location
+ return fmt::format("{}{}", base_path, file_name);
+}
+
+} // namespace doris
diff --git a/be/src/exec/sink/viceberg_delete_sink.h b/be/src/exec/sink/viceberg_delete_sink.h
new file mode 100644
index 00000000000..9efcb1fe681
--- /dev/null
+++ b/be/src/exec/sink/viceberg_delete_sink.h
@@ -0,0 +1,154 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <gen_cpp/DataSinks_types.h>
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "common/status.h"
+#include "core/block/block.h"
+#include "core/value/bitmap_value.h"
+#include "exec/sink/writer/async_result_writer.h"
+#include "exec/sink/writer/iceberg/viceberg_delete_file_writer.h"
+#include "exprs/vexpr_fwd.h"
+#include "runtime/runtime_profile.h"
+
+namespace doris {
+
+class ObjectPool;
+class RuntimeState;
+class Dependency;
+
+class VIcebergDeleteFileWriter;
+
+struct IcebergFileDeletion {
+ IcebergFileDeletion() = default;
+ IcebergFileDeletion(int32_t spec_id, std::string data_json)
+ : partition_spec_id(spec_id), partition_data_json(std::move(data_json)) {}
+
+ int32_t partition_spec_id = 0;
+ std::string partition_data_json;
+ doris::detail::Roaring64Map rows_to_delete;
+};
+
+/**
+ * Sink for writing Iceberg position delete files.
+ *
+ * This sink receives blocks containing a $row_id column with
+ * (file_path, row_position, partition_spec_id, partition_data).
+ * It groups delete records by file_path and writes position delete files.
+ */
+class VIcebergDeleteSink final : public AsyncResultWriter {
+public:
+ VIcebergDeleteSink(const TDataSink& t_sink, const VExprContextSPtrs& output_exprs,
+ std::shared_ptr<Dependency> dep, std::shared_ptr<Dependency> fin_dep);
+
+ ~VIcebergDeleteSink() override = default;
+
+ Status init_properties(ObjectPool* pool);
+
+ Status open(RuntimeState* state, RuntimeProfile* profile) override;
+
+ Status write(RuntimeState* state, Block& block) override;
+
+ Status close(Status) override;
+
+private:
+ /**
+ * Extract $row_id column from block and group by file_path.
+ */
+ Status _collect_position_deletes(const Block& block,
+ std::map<std::string, IcebergFileDeletion>& file_deletions);
+
+ /**
+ * Write grouped position deletes to delete files
+ */
+ Status _write_position_delete_files(
+ const std::map<std::string, IcebergFileDeletion>& file_deletions);
+
+ /**
+ * Generate unique delete file path
+ */
+ std::string _generate_delete_file_path(const std::string& referenced_data_file = "");
+
+ /**
+ * Get $row_id column index from block
+ */
+ int _get_row_id_column_index(const Block& block);
+
+ /**
+ * Build a block for position delete (file_path, pos)
+ */
+ Status _build_position_delete_block(const std::string& file_path,
+ const std::vector<int64_t>& positions, Block& output_block);
+ Status _init_position_delete_output_exprs();
+ std::string _get_file_extension() const;
+
+ TDataSink _t_sink;
+ RuntimeState* _state = nullptr;
+
+ TFileContent::type _delete_type = TFileContent::POSITION_DELETES;
+
+ // Writers for delete files
+ std::vector<std::unique_ptr<VIcebergDeleteFileWriter>> _writers;
+
+ // Collected commit data from all writers
+ std::vector<TIcebergCommitData> _commit_data_list;
+ // TODO: All deletions are held in memory until close(). Consider flushing
+ // per-file when the upstream guarantees file_path ordering, or flushing
+ // when estimated memory exceeds a threshold, to reduce peak memory usage.
+ std::map<std::string, IcebergFileDeletion> _file_deletions;
+
+ // Hadoop configuration
+ std::map<std::string, std::string> _hadoop_conf;
+
+ // File format settings
+ TFileFormatType::type _file_format_type = TFileFormatType::FORMAT_PARQUET;
+ TFileCompressType::type _compress_type = TFileCompressType::SNAPPYBLOCK;
+
+ // Output directory for delete files
+ std::string _output_path;
+ std::string _table_location;
+
+ TFileType::type _file_type = TFileType::FILE_HDFS;
+ std::vector<TNetworkAddress> _broker_addresses;
+
+ // Partition information
+ int32_t _partition_spec_id = 0;
+ std::string _partition_data_json;
+
+ // Counters
+ size_t _row_count = 0;
+ size_t _delete_file_count = 0;
+
+ // Profile counters
+ RuntimeProfile::Counter* _written_rows_counter = nullptr;
+ RuntimeProfile::Counter* _send_data_timer = nullptr;
+ RuntimeProfile::Counter* _write_delete_files_timer = nullptr;
+ RuntimeProfile::Counter* _delete_file_count_counter = nullptr;
+ RuntimeProfile::Counter* _open_timer = nullptr;
+ RuntimeProfile::Counter* _close_timer = nullptr;
+
+ VExprContextSPtrs _position_delete_output_expr_ctxs;
+};
+
+} // namespace doris
diff --git a/be/src/exec/sink/viceberg_merge_sink.cpp b/be/src/exec/sink/viceberg_merge_sink.cpp
new file mode 100644
index 00000000000..167c51cd6fa
--- /dev/null
+++ b/be/src/exec/sink/viceberg_merge_sink.cpp
@@ -0,0 +1,341 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exec/sink/viceberg_merge_sink.h"
+
+#include <fmt/format.h>
+
+#include "common/consts.h"
+#include "common/exception.h"
+#include "common/logging.h"
+#include "core/block/block.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_vector.h"
+#include "exec/sink/sink_common.h"
+#include "exec/sink/viceberg_delete_sink.h"
+#include "exec/sink/writer/iceberg/viceberg_table_writer.h"
+#include "format/table/iceberg/schema.h"
+#include "format/table/iceberg/schema_parser.h"
+#include "runtime/runtime_state.h"
+#include "util/string_util.h"
+
+namespace doris {
+
+namespace {} // namespace
+
+VIcebergMergeSink::VIcebergMergeSink(const TDataSink& t_sink, const VExprContextSPtrs& output_exprs,
+ std::shared_ptr<Dependency> dep,
+ std::shared_ptr<Dependency> fin_dep)
+ : AsyncResultWriter(output_exprs, dep, fin_dep), _t_sink(t_sink) {
+ DCHECK(_t_sink.__isset.iceberg_merge_sink);
+}
+
+VIcebergMergeSink::~VIcebergMergeSink() = default;
+
+Status VIcebergMergeSink::init_properties(ObjectPool* pool, const RowDescriptor& row_desc) {
+ RETURN_IF_ERROR(_build_inner_sinks());
+
+ _table_writer = std::make_unique<VIcebergTableWriter>(_table_sink, _table_output_expr_ctxs,
+ nullptr, nullptr);
+ _delete_writer = std::make_unique<VIcebergDeleteSink>(_delete_sink, _delete_output_expr_ctxs,
+ nullptr, nullptr);
+ RETURN_IF_ERROR(_table_writer->init_properties(pool, row_desc));
+ RETURN_IF_ERROR(_delete_writer->init_properties(pool));
+ return Status::OK();
+}
+
+Status VIcebergMergeSink::open(RuntimeState* state, RuntimeProfile* profile) {
+ _state = state;
+
+ _written_rows_counter = ADD_COUNTER(profile, "RowsWritten", TUnit::UNIT);
+ _insert_rows_counter = ADD_COUNTER(profile, "InsertRows", TUnit::UNIT);
+ _delete_rows_counter = ADD_COUNTER(profile, "DeleteRows", TUnit::UNIT);
+ _send_data_timer = ADD_TIMER(profile, "SendDataTime");
+ _open_timer = ADD_TIMER(profile, "OpenTime");
+ _close_timer = ADD_TIMER(profile, "CloseTime");
+
+ SCOPED_TIMER(_open_timer);
+
+ RETURN_IF_ERROR(_prepare_output_layout());
+
+ RuntimeProfile* table_profile = profile->create_child("IcebergMergeTableWriter", true, true);
+ RuntimeProfile* delete_profile = profile->create_child("IcebergMergeDeleteWriter", true, true);
+
+ RETURN_IF_ERROR(_table_writer->open(state, table_profile));
+ RETURN_IF_ERROR(_delete_writer->open(state, delete_profile));
+
+ return Status::OK();
+}
+
+Status VIcebergMergeSink::write(RuntimeState* state, Block& block) {
+ SCOPED_TIMER(_send_data_timer);
+ if (block.rows() == 0) {
+ return Status::OK();
+ }
+
+ Block output_block;
+ RETURN_IF_ERROR(_projection_block(block, &output_block));
+ if (output_block.rows() == 0) {
+ return Status::OK();
+ }
+
+ _row_count += output_block.rows();
+
+ if (_operation_idx < 0 || _row_id_idx < 0) {
+ return Status::InternalError("Iceberg merge sink missing operation/row_id columns");
+ }
+
+ const auto& op_column = output_block.get_by_position(_operation_idx).column;
+ const auto* op_data = remove_nullable(op_column).get();
+
+ IColumn::Filter delete_filter(output_block.rows(), 0);
+ IColumn::Filter insert_filter(output_block.rows(), 0);
+ bool has_delete = false;
+ bool has_insert = false;
+ size_t delete_rows = 0;
+ size_t insert_rows = 0;
+
+ for (size_t i = 0; i < output_block.rows(); ++i) {
+ int8_t op = static_cast<int8_t>(op_data->get_int(i));
+ bool delete_op = is_delete_op(op);
+ bool insert_op = is_insert_op(op);
+ if (!delete_op && !insert_op) {
+ return Status::InternalError("Unknown Iceberg merge operation {}", op);
+ }
+ if (delete_op) {
+ delete_filter[i] = 1;
+ has_delete = true;
+ ++_delete_row_count;
+ ++delete_rows;
+ }
+ if (insert_op) {
+ insert_filter[i] = 1;
+ has_insert = true;
+ ++_insert_row_count;
+ ++insert_rows;
+ }
+ }
+
+ bool skip_io = false;
+#ifdef BE_TEST
+ skip_io = _skip_io;
+#endif
+
+ if (has_delete && !skip_io) {
+ Block delete_block = output_block;
+ std::vector<int> delete_indices {_row_id_idx};
+ delete_block.erase_not_in(delete_indices);
+ Block::filter_block_internal(&delete_block, delete_filter);
+ RETURN_IF_ERROR(_delete_writer->write(state, delete_block));
+ }
+
+ if (has_insert && !skip_io) {
+ if (_data_column_indices.empty()) {
+ return Status::InternalError("Iceberg merge sink has no data columns for insert");
+ }
+ Block insert_block = output_block;
+ insert_block.erase_not_in(_data_column_indices);
+ Block::filter_block_internal(&insert_block, insert_filter);
+ RETURN_IF_ERROR(_table_writer->write_prepared_block(insert_block));
+ }
+
+ if (_written_rows_counter != nullptr) {
+ COUNTER_UPDATE(_written_rows_counter, output_block.rows());
+ }
+ if (_insert_rows_counter != nullptr) {
+ COUNTER_UPDATE(_insert_rows_counter, insert_rows);
+ }
+ if (_delete_rows_counter != nullptr) {
+ COUNTER_UPDATE(_delete_rows_counter, delete_rows);
+ }
+
+ return Status::OK();
+}
+
+Status VIcebergMergeSink::close(Status close_status) {
+ SCOPED_TIMER(_close_timer);
+
+ if (!close_status.ok()) {
+ LOG(WARNING) << fmt::format("VIcebergMergeSink close with error: {}",
+ close_status.to_string());
+ if (_table_writer) {
+ static_cast<void>(_table_writer->close(close_status));
+ }
+ if (_delete_writer) {
+ static_cast<void>(_delete_writer->close(close_status));
+ }
+ return close_status;
+ }
+
+ Status table_status = Status::OK();
+ Status delete_status = Status::OK();
+ if (_table_writer) {
+ table_status = _table_writer->close(close_status);
+ }
+ if (_delete_writer) {
+ delete_status = _delete_writer->close(close_status);
+ }
+
+ if (_written_rows_counter != nullptr) {
+ COUNTER_SET(_written_rows_counter, static_cast<int64_t>(_row_count));
+ }
+ if (_insert_rows_counter != nullptr) {
+ COUNTER_SET(_insert_rows_counter, static_cast<int64_t>(_insert_row_count));
+ }
+ if (_delete_rows_counter != nullptr) {
+ COUNTER_SET(_delete_rows_counter, static_cast<int64_t>(_delete_row_count));
+ }
+
+ if (!table_status.ok()) {
+ return table_status;
+ }
+ return delete_status;
+}
+
+Status VIcebergMergeSink::_build_inner_sinks() {
+ if (!_t_sink.__isset.iceberg_merge_sink) {
+ return Status::InternalError("Missing iceberg merge sink config");
+ }
+
+ const auto& merge_sink = _t_sink.iceberg_merge_sink;
+
+ TIcebergTableSink table_sink;
+ if (merge_sink.__isset.db_name) {
+ table_sink.__set_db_name(merge_sink.db_name);
+ }
+ if (merge_sink.__isset.tb_name) {
+ table_sink.__set_tb_name(merge_sink.tb_name);
+ }
+ if (merge_sink.__isset.schema_json) {
+ table_sink.__set_schema_json(merge_sink.schema_json);
+ }
+ if (merge_sink.__isset.partition_specs_json) {
+ table_sink.__set_partition_specs_json(merge_sink.partition_specs_json);
+ }
+ if (merge_sink.__isset.partition_spec_id) {
+ table_sink.__set_partition_spec_id(merge_sink.partition_spec_id);
+ }
+ if (merge_sink.__isset.sort_fields) {
+ table_sink.__set_sort_fields(merge_sink.sort_fields);
+ }
+ if (merge_sink.__isset.file_format) {
+ table_sink.__set_file_format(merge_sink.file_format);
+ }
+ if (merge_sink.__isset.compression_type) {
+ table_sink.__set_compression_type(merge_sink.compression_type);
+ }
+ if (merge_sink.__isset.output_path) {
+ table_sink.__set_output_path(merge_sink.output_path);
+ }
+ if (merge_sink.__isset.original_output_path) {
+ table_sink.__set_original_output_path(merge_sink.original_output_path);
+ }
+ if (merge_sink.__isset.hadoop_config) {
+ table_sink.__set_hadoop_config(merge_sink.hadoop_config);
+ }
+ if (merge_sink.__isset.file_type) {
+ table_sink.__set_file_type(merge_sink.file_type);
+ }
+ if (merge_sink.__isset.broker_addresses) {
+ table_sink.__set_broker_addresses(merge_sink.broker_addresses);
+ }
+ _table_sink.__set_type(TDataSinkType::ICEBERG_TABLE_SINK);
+ _table_sink.__set_iceberg_table_sink(table_sink);
+
+ TIcebergDeleteSink delete_sink;
+ if (merge_sink.__isset.db_name) {
+ delete_sink.__set_db_name(merge_sink.db_name);
+ }
+ if (merge_sink.__isset.tb_name) {
+ delete_sink.__set_tb_name(merge_sink.tb_name);
+ }
+ if (merge_sink.__isset.delete_type) {
+ delete_sink.__set_delete_type(merge_sink.delete_type);
+ }
+ if (merge_sink.__isset.file_format) {
+ delete_sink.__set_file_format(merge_sink.file_format);
+ }
+ if (merge_sink.__isset.compression_type) {
+ delete_sink.__set_compress_type(merge_sink.compression_type);
+ }
+ if (merge_sink.__isset.output_path) {
+ delete_sink.__set_output_path(merge_sink.output_path);
+ }
+ if (merge_sink.__isset.table_location) {
+ delete_sink.__set_table_location(merge_sink.table_location);
+ }
+ if (merge_sink.__isset.hadoop_config) {
+ delete_sink.__set_hadoop_config(merge_sink.hadoop_config);
+ }
+ if (merge_sink.__isset.file_type) {
+ delete_sink.__set_file_type(merge_sink.file_type);
+ }
+ if (merge_sink.__isset.partition_spec_id_for_delete) {
+ delete_sink.__set_partition_spec_id(merge_sink.partition_spec_id_for_delete);
+ }
+ if (merge_sink.__isset.partition_data_json_for_delete) {
+ delete_sink.__set_partition_data_json(merge_sink.partition_data_json_for_delete);
+ }
+ if (merge_sink.__isset.broker_addresses) {
+ delete_sink.__set_broker_addresses(merge_sink.broker_addresses);
+ }
+ _delete_sink.__set_type(TDataSinkType::ICEBERG_DELETE_SINK);
+ _delete_sink.__set_iceberg_delete_sink(delete_sink);
+
+ return Status::OK();
+}
+
+Status VIcebergMergeSink::_prepare_output_layout() {
+ if (_vec_output_expr_ctxs.empty()) {
+ return Status::InternalError("Iceberg merge sink has empty output expressions");
+ }
+
+ std::string row_id_name = doris::to_lower(BeConsts::ICEBERG_ROWID_COL);
+ std::string op_name = doris::to_lower(kOperationColumnName);
+
+ _operation_idx = -1;
+ _row_id_idx = -1;
+ for (size_t i = 0; i < _vec_output_expr_ctxs.size(); ++i) {
+ std::string expr_name = doris::to_lower(_vec_output_expr_ctxs[i]->expr_name());
+ if (_operation_idx < 0 && expr_name == op_name) {
+ _operation_idx = static_cast<int>(i);
+ } else if (_row_id_idx < 0 && expr_name == row_id_name) {
+ _row_id_idx = static_cast<int>(i);
+ }
+ }
+
+ if (_operation_idx < 0) {
+ return Status::InternalError("Iceberg merge sink missing operation column");
+ }
+ if (_row_id_idx < 0) {
+ return Status::InternalError("Iceberg merge sink missing row_id column");
+ }
+
+ _data_column_indices.clear();
+ _table_output_expr_ctxs.clear();
+ for (size_t i = 0; i < _vec_output_expr_ctxs.size(); ++i) {
+ if (static_cast<int>(i) == _operation_idx || static_cast<int>(i) == _row_id_idx) {
+ continue;
+ }
+ _data_column_indices.push_back(static_cast<int>(i));
+ _table_output_expr_ctxs.emplace_back(_vec_output_expr_ctxs[i]);
+ }
+
+ return Status::OK();
+}
+
+} // namespace doris
diff --git a/be/src/exec/sink/viceberg_merge_sink.h b/be/src/exec/sink/viceberg_merge_sink.h
new file mode 100644
index 00000000000..d7f87b87012
--- /dev/null
+++ b/be/src/exec/sink/viceberg_merge_sink.h
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <gen_cpp/DataSinks_types.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "common/status.h"
+#include "exec/sink/writer/async_result_writer.h"
+#include "exprs/vexpr_fwd.h"
+#include "runtime/descriptors.h"
+#include "runtime/runtime_profile.h"
+
+namespace doris {
+
+class ObjectPool;
+class RuntimeState;
+class Dependency;
+
+class VIcebergDeleteSink;
+class VIcebergTableWriter;
+
+class VIcebergMergeSink final : public AsyncResultWriter {
+public:
+ VIcebergMergeSink(const TDataSink& t_sink, const VExprContextSPtrs& output_exprs,
+ std::shared_ptr<Dependency> dep, std::shared_ptr<Dependency> fin_dep);
+
+ ~VIcebergMergeSink() override;
+
+ Status init_properties(ObjectPool* pool, const RowDescriptor& row_desc);
+
+ Status open(RuntimeState* state, RuntimeProfile* profile) override;
+
+ Status write(RuntimeState* state, Block& block) override;
+
+ Status close(Status) override;
+
+#ifdef BE_TEST
+ void set_skip_io(bool skip) { _skip_io = skip; }
+#endif
+
+private:
+ Status _build_inner_sinks();
+ Status _prepare_output_layout();
+
+ TDataSink _t_sink;
+ TDataSink _table_sink;
+ TDataSink _delete_sink;
+
+ std::unique_ptr<VIcebergTableWriter> _table_writer;
+ std::unique_ptr<VIcebergDeleteSink> _delete_writer;
+
+ RuntimeState* _state = nullptr;
+
+ int _operation_idx = -1;
+ int _row_id_idx = -1;
+ std::vector<int> _data_column_indices;
+
+ VExprContextSPtrs _table_output_expr_ctxs;
+ VExprContextSPtrs _delete_output_expr_ctxs;
+
+ size_t _row_count = 0;
+ size_t _insert_row_count = 0;
+ size_t _delete_row_count = 0;
+
+ RuntimeProfile::Counter* _written_rows_counter = nullptr;
+ RuntimeProfile::Counter* _insert_rows_counter = nullptr;
+ RuntimeProfile::Counter* _delete_rows_counter = nullptr;
+ RuntimeProfile::Counter* _send_data_timer = nullptr;
+ RuntimeProfile::Counter* _open_timer = nullptr;
+ RuntimeProfile::Counter* _close_timer = nullptr;
+
+#ifdef BE_TEST
+ bool _skip_io = false;
+#endif
+};
+
+} // namespace doris
diff --git a/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.cpp
new file mode 100644
index 00000000000..a531cf342d6
--- /dev/null
+++ b/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.cpp
@@ -0,0 +1,155 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "viceberg_delete_file_writer.h"
+
+#include <fmt/format.h>
+
+#include "format/transformer/vorc_transformer.h"
+#include "format/transformer/vparquet_transformer.h"
+#include "io/file_factory.h"
+#include "runtime/runtime_state.h"
+
+namespace doris {
+
+VIcebergDeleteFileWriter::VIcebergDeleteFileWriter(TFileContent::type delete_type,
+ const std::string& output_path,
+ TFileFormatType::type file_format,
+ TFileCompressType::type compress_type)
+ : _delete_type(delete_type),
+ _output_path(output_path),
+ _file_format(file_format),
+ _compress_type(compress_type) {}
+
+VIcebergDeleteFileWriter::~VIcebergDeleteFileWriter() = default;
+
+Status VIcebergDeleteFileWriter::open(RuntimeState* state, RuntimeProfile* profile,
+ const VExprContextSPtrs& output_exprs,
+ const std::vector<std::string>& column_names,
+ const std::map<std::string, std::string>& hadoop_config,
+ TFileType::type file_type,
+ const std::vector<TNetworkAddress>& broker_addresses) {
+ if (_delete_type != TFileContent::POSITION_DELETES) {
+ return Status::NotSupported("Iceberg delete file writer only supports position deletes");
+ }
+
+ _state = state;
+
+ // Create file system and file writer
+ io::FSPropertiesRef fs_properties(file_type);
+ fs_properties.properties = &hadoop_config;
+ if (!broker_addresses.empty()) {
+ fs_properties.broker_addresses = &broker_addresses;
+ }
+
+ io::FileDescription file_description = {.path = _output_path, .fs_name {}};
+ _fs = DORIS_TRY(FileFactory::create_fs(fs_properties, file_description));
+
+ io::FileWriterOptions file_writer_options = {.used_by_s3_committer = false};
+ RETURN_IF_ERROR(_fs->create_file(file_description.path, &_file_writer, &file_writer_options));
+
+ // Create file format transformer based on format type
+ switch (_file_format) {
+ case TFileFormatType::FORMAT_PARQUET: {
+ TParquetCompressionType::type parquet_compression_type;
+ switch (_compress_type) {
+ case TFileCompressType::PLAIN:
+ parquet_compression_type = TParquetCompressionType::UNCOMPRESSED;
+ break;
+ case TFileCompressType::SNAPPYBLOCK:
+ parquet_compression_type = TParquetCompressionType::SNAPPY;
+ break;
+ case TFileCompressType::ZSTD:
+ parquet_compression_type = TParquetCompressionType::ZSTD;
+ break;
+ default:
+ return Status::InternalError("Unsupported compress type {} with parquet",
+ to_string(_compress_type));
+ }
+
+ ParquetFileOptions parquet_options = {parquet_compression_type,
+ TParquetVersion::PARQUET_1_0, false, false};
+ _file_format_transformer.reset(new VParquetTransformer(state, _file_writer.get(),
+ output_exprs, column_names, false,
+ parquet_options, nullptr, nullptr));
+ return _file_format_transformer->open();
+ }
+ case TFileFormatType::FORMAT_ORC: {
+ _file_format_transformer.reset(new VOrcTransformer(state, _file_writer.get(), output_exprs,
+ "", column_names, false, _compress_type,
+ nullptr));
+ return _file_format_transformer->open();
+ }
+ default:
+ return Status::InternalError("Unsupported file format type {}", to_string(_file_format));
+ }
+
+ return Status::OK();
+}
+
+Status VIcebergDeleteFileWriter::write(const Block& block) {
+ if (block.rows() == 0) {
+ return Status::OK();
+ }
+
+ // Write block using file format transformer
+ RETURN_IF_ERROR(_file_format_transformer->write(const_cast<Block&>(block)));
+ _written_rows += block.rows();
+
+ return Status::OK();
+}
+
+Status VIcebergDeleteFileWriter::close(TIcebergCommitData& commit_data) {
+ Status result_status;
+
+ if (_file_format_transformer != nullptr) {
+ result_status = _file_format_transformer->close();
+ if (!result_status.ok()) {
+ LOG(WARNING) << fmt::format("_file_format_transformer close failed, reason: {}",
+ result_status.to_string());
+ }
+ _file_size = _file_format_transformer->written_len();
+ }
+
+ // Fill commit data (use __set_ to mark optional fields as present)
+ commit_data.__set_file_path(_output_path);
+ commit_data.__set_row_count(_written_rows);
+ commit_data.__set_file_size(_file_size);
+ commit_data.__set_file_content(_delete_type);
+
+ // Set partition information (set via set_partition_info() before close)
+ commit_data.__set_partition_spec_id(_partition_spec_id);
+ if (!_partition_data_json.empty()) {
+ commit_data.__set_partition_data_json(_partition_data_json);
+ }
+
+ if (!_partition_values.empty()) {
+ commit_data.__set_partition_values(_partition_values);
+ }
+
+ return result_status;
+}
+
+// Factory method
+std::unique_ptr<VIcebergDeleteFileWriter> VIcebergDeleteFileWriterFactory::create_writer(
+ TFileContent::type delete_type, const std::string& output_path,
+ TFileFormatType::type file_format, TFileCompressType::type compress_type) {
+ return std::make_unique<VIcebergDeleteFileWriter>(delete_type, output_path, file_format,
+ compress_type);
+}
+
+} // namespace doris
diff --git a/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.h b/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.h
new file mode 100644
index 00000000000..e5de7143f2b
--- /dev/null
+++ b/be/src/exec/sink/writer/iceberg/viceberg_delete_file_writer.h
@@ -0,0 +1,123 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "common/status.h"
+#include "core/block/block.h"
+#include "exprs/vexpr_fwd.h"
+#include "gen_cpp/DataSinks_types.h"
+#include "io/fs/file_writer.h"
+
+namespace doris {
+
+class RuntimeState;
+class RuntimeProfile;
+class ObjectPool;
+
+namespace io {
+class FileSystem;
+}
+
+class VFileFormatTransformer;
+
+/**
+ * Writer for Iceberg Position Delete files.
+ *
+ * Schema: (file_path: string, pos: long)
+ * Each row contains the file path and row position to delete.
+ */
+class VIcebergDeleteFileWriter {
+public:
+ VIcebergDeleteFileWriter(TFileContent::type delete_type, const std::string& output_path,
+ TFileFormatType::type file_format,
+ TFileCompressType::type compress_type);
+
+ ~VIcebergDeleteFileWriter();
+
+ /**
+ * Open the writer with runtime state
+ */
+ Status open(RuntimeState* state, RuntimeProfile* profile, const VExprContextSPtrs& output_exprs,
+ const std::vector<std::string>& column_names,
+ const std::map<std::string, std::string>& hadoop_config, TFileType::type file_type,
+ const std::vector<TNetworkAddress>& broker_addresses = {});
+
+ /**
+ * Write a block of delete data.
+ * For Position Delete: block contains (file_path, pos) columns.
+ */
+ Status write(const Block& block);
+
+ /**
+ * Close the writer and return commit data
+ */
+ Status close(TIcebergCommitData& commit_data);
+
+ /**
+ * Set partition information for the delete file.
+ * Must be called before close() so that commit data includes correct partition info.
+ */
+ void set_partition_info(int32_t partition_spec_id, const std::string& partition_data_json) {
+ _partition_spec_id = partition_spec_id;
+ _partition_data_json = partition_data_json;
+ }
+
+ /**
+ * Get the number of rows written
+ */
+ int64_t get_written_rows() const { return _written_rows; }
+
+ /**
+ * Get the file size in bytes
+ */
+ int64_t get_file_size() const { return _file_size; }
+
+private:
+ TFileContent::type _delete_type;
+ std::string _output_path;
+ TFileFormatType::type _file_format;
+ TFileCompressType::type _compress_type = TFileCompressType::SNAPPYBLOCK;
+
+ int64_t _written_rows = 0;
+ int64_t _file_size = 0;
+
+ RuntimeState* _state = nullptr;
+ std::shared_ptr<io::FileSystem> _fs;
+ io::FileWriterPtr _file_writer;
+ std::unique_ptr<VFileFormatTransformer> _file_format_transformer;
+
+ int32_t _partition_spec_id = 0;
+ std::string _partition_data_json;
+ std::vector<std::string> _partition_values;
+};
+
+/**
+ * Factory class for creating delete file writers
+ */
+class VIcebergDeleteFileWriterFactory {
+public:
+ static std::unique_ptr<VIcebergDeleteFileWriter> create_writer(
+ TFileContent::type delete_type, const std::string& output_path,
+ TFileFormatType::type file_format, TFileCompressType::type compress_type);
+};
+
+} // namespace doris
diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp
index 33fbf49b960..fcba431f159 100644
--- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp
+++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp
@@ -45,6 +45,7 @@ VIcebergTableWriter::VIcebergTableWriter(const TDataSink& t_sink,
Status VIcebergTableWriter::open(RuntimeState* state, RuntimeProfile* profile) {
_state = state;
+ _operator_profile = profile;
// Get target file size from query options
// If value is 0 or not set, use config::iceberg_sink_max_file_size
@@ -88,6 +89,12 @@ Status VIcebergTableWriter::open(RuntimeState* state, RuntimeProfile* profile) {
partition_idx_set.insert(iceberg_partition_column.source_idx());
}
+ if (_vec_output_expr_ctxs.size() != _schema->columns().size()) {
+ return Status::InternalError(
+ "Iceberg table writer data columns {} do not match schema columns {}",
+ _vec_output_expr_ctxs.size(), _schema->columns().size());
+ }
+
for (int i = 0; i < _schema->columns().size(); ++i) {
_write_output_vexpr_ctxs.emplace_back(_vec_output_expr_ctxs[i]);
}
@@ -188,7 +195,18 @@ Status VIcebergTableWriter::write(RuntimeState* state, Block& block) {
RETURN_IF_ERROR(VExprContext::get_output_block_after_execute_exprs(_vec_output_expr_ctxs, block,
&output_block, false));
materialize_block_inplace(output_block);
+ return _write_prepared_block(output_block);
+}
+
+Status VIcebergTableWriter::write_prepared_block(Block& block) {
+ SCOPED_RAW_TIMER(&_send_data_ns);
+ if (block.rows() == 0) {
+ return Status::OK();
+ }
+ return _write_prepared_block(block);
+}
+Status VIcebergTableWriter::_write_prepared_block(Block& output_block) {
std::unordered_map<std::shared_ptr<IPartitionWriterBase>, IColumn::Filter> writer_positions;
_row_count += output_block.rows();
diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h
index 6ce6eadf30a..32d40695ce2 100644
--- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h
+++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h
@@ -55,6 +55,8 @@ public:
Status write(RuntimeState* state, Block& block) override;
+ Status write_prepared_block(Block& block);
+
Status close(Status) override;
// Getter for the current partition writer.
@@ -123,6 +125,8 @@ private:
Status _filter_block(doris::Block& block, const IColumn::Filter* filter,
doris::Block* output_block);
+ Status _write_prepared_block(Block& output_block);
+
// Currently it is a copy, maybe it is better to use move semantics to eliminate it.
TDataSink _t_sink;
RuntimeState* _state = nullptr;
diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp
index ba48aab565d..8a1f538dafb 100644
--- a/be/src/format/orc/vorc_reader.cpp
+++ b/be/src/format/orc/vorc_reader.cpp
@@ -47,6 +47,7 @@
#include "absl/strings/substitute.h"
#include "cctz/civil_time.h"
#include "cctz/time_zone.h"
+#include "common/consts.h"
#include "common/exception.h"
#include "core/block/block.h"
#include "core/block/column_with_type_and_name.h"
@@ -55,10 +56,13 @@
#include "core/column/column_const.h"
#include "core/column/column_map.h"
#include "core/column/column_nullable.h"
+#include "core/column/column_string.h"
#include "core/column/column_struct.h"
#include "core/data_type/data_type_array.h"
#include "core/data_type/data_type_map.h"
#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
#include "core/data_type/data_type_struct.h"
#include "core/data_type/define_primitive_type.h"
#include "core/data_type/primitive_type.h"
@@ -106,6 +110,63 @@ enum class FileCachePolicy : uint8_t;
namespace doris {
#include "common/compile_check_begin.h"
+
+namespace {
+Status build_iceberg_rowid_column(const DataTypePtr& type, const std::string& file_path,
+ int64_t start_row, size_t num_rows, int32_t partition_spec_id,
+ const std::string& partition_data_json,
+ MutableColumnPtr* column_out) {
+ if (type == nullptr || column_out == nullptr) {
+ return Status::InvalidArgument("Invalid iceberg rowid column type or output column");
+ }
+
+ MutableColumnPtr column = type->create_column();
+ ColumnNullable* nullable_col = check_and_get_column<ColumnNullable>(column.get());
+ ColumnStruct* struct_col = nullptr;
+ if (nullable_col != nullptr) {
+ struct_col =
+ check_and_get_column<ColumnStruct>(nullable_col->get_nested_column_ptr().get());
+ } else {
+ struct_col = check_and_get_column<ColumnStruct>(column.get());
+ }
+
+ if (struct_col == nullptr || struct_col->tuple_size() < 4) {
+ return Status::InternalError("Invalid iceberg rowid column structure");
+ }
+
+ auto& file_path_col = struct_col->get_column(0);
+ auto& row_pos_col = struct_col->get_column(1);
+ auto& spec_id_col = struct_col->get_column(2);
+ auto& partition_data_col = struct_col->get_column(3);
+
+ file_path_col.reserve(num_rows);
+ row_pos_col.reserve(num_rows);
+ spec_id_col.reserve(num_rows);
+ partition_data_col.reserve(num_rows);
+
+ for (size_t i = 0; i < num_rows; ++i) {
+ file_path_col.insert_data(file_path.data(), file_path.size());
+ }
+ for (size_t i = 0; i < num_rows; ++i) {
+ int64_t row_pos = start_row + static_cast<int64_t>(i);
+ row_pos_col.insert_data(reinterpret_cast<const char*>(&row_pos), sizeof(row_pos));
+ }
+ for (size_t i = 0; i < num_rows; ++i) {
+ int32_t spec_id = partition_spec_id;
+ spec_id_col.insert_data(reinterpret_cast<const char*>(&spec_id), sizeof(spec_id));
+ }
+ for (size_t i = 0; i < num_rows; ++i) {
+ partition_data_col.insert_data(partition_data_json.data(), partition_data_json.size());
+ }
+
+ if (nullable_col != nullptr) {
+ nullable_col->get_null_map_data().resize_fill(num_rows, 0);
+ }
+
+ *column_out = std::move(column);
+ return Status::OK();
+}
+} // namespace
// TODO: we need to determine it by test.
static constexpr uint32_t MAX_DICT_CODE_PREDICATE_TO_REWRITE = std::numeric_limits<uint32_t>::max();
static constexpr char EMPTY_STRING_FOR_OVERFLOW[ColumnString::MAX_STRINGS_OVERFLOW_SIZE] = "";
@@ -406,6 +467,16 @@ Status OrcReader::get_parsed_schema(std::vector<std::string>* col_names,
return Status::OK();
}
+void OrcReader::set_iceberg_rowid_params(const std::string& file_path, int32_t partition_spec_id,
+ const std::string& partition_data_json,
+ int row_id_column_pos) {
+ _iceberg_rowid_params.enabled = true;
+ _iceberg_rowid_params.file_path = file_path;
+ _iceberg_rowid_params.partition_spec_id = partition_spec_id;
+ _iceberg_rowid_params.partition_data_json = partition_data_json;
+ _iceberg_rowid_params.row_id_column_pos = row_id_column_pos;
+}
+
Status OrcReader::_init_read_columns() {
SCOPED_RAW_TIMER(&_statistics.init_column_time);
const auto& root_type = _reader->getType();
@@ -1394,10 +1465,9 @@ Status OrcReader::_fill_missing_columns(
return Status::OK();
}
-Status OrcReader::_fill_row_id_columns(Block* block) {
+Status OrcReader::_fill_row_id_columns(Block* block, int64_t start_row) {
if (_row_id_column_iterator_pair.first != nullptr) {
- RETURN_IF_ERROR(
- _row_id_column_iterator_pair.first->seek_to_ordinal(_row_reader->getRowNumber()));
+ RETURN_IF_ERROR(_row_id_column_iterator_pair.first->seek_to_ordinal(start_row));
size_t fill_size = _batch->numElements;
auto col = block->get_by_position(_row_id_column_iterator_pair.second)
@@ -1408,6 +1478,51 @@ Status OrcReader::_fill_row_id_columns(Block* block) {
return Status::OK();
}
+Status OrcReader::_append_iceberg_rowid_column(Block* block, size_t rows, int64_t start_row) {
+ if (!_iceberg_rowid_params.enabled) {
+ return Status::OK();
+ }
+
+ int row_id_idx = block->get_position_by_name(doris::BeConsts::ICEBERG_ROWID_COL);
+ if (row_id_idx >= 0) {
+ auto& col_with_type = block->get_by_position(static_cast<size_t>(row_id_idx));
+ MutableColumnPtr row_id_column;
+ RETURN_IF_ERROR(build_iceberg_rowid_column(
+ col_with_type.type, _iceberg_rowid_params.file_path, start_row, rows,
+ _iceberg_rowid_params.partition_spec_id, _iceberg_rowid_params.partition_data_json,
+ &row_id_column));
+ col_with_type.column = std::move(row_id_column);
+ } else {
+ DataTypes field_types;
+ field_types.push_back(std::make_shared<DataTypeString>());
+ field_types.push_back(std::make_shared<DataTypeInt64>());
+ field_types.push_back(std::make_shared<DataTypeInt32>());
+ field_types.push_back(std::make_shared<DataTypeString>());
+
+ std::vector<std::string> field_names = {"file_path", "row_position", "partition_spec_id",
+ "partition_data"};
+ auto row_id_type = std::make_shared<DataTypeStruct>(field_types, field_names);
+ MutableColumnPtr row_id_column;
+ RETURN_IF_ERROR(build_iceberg_rowid_column(
+ row_id_type, _iceberg_rowid_params.file_path, start_row, rows,
+ _iceberg_rowid_params.partition_spec_id, _iceberg_rowid_params.partition_data_json,
+ &row_id_column));
+ int insert_pos = _iceberg_rowid_params.row_id_column_pos;
+ if (insert_pos < 0 || insert_pos > static_cast<int>(block->columns())) {
+ insert_pos = static_cast<int>(block->columns());
+ }
+ block->insert(static_cast<size_t>(insert_pos),
+ ColumnWithTypeAndName(std::move(row_id_column), row_id_type,
+ doris::BeConsts::ICEBERG_ROWID_COL));
+ }
+
+ if (_col_name_to_block_idx != nullptr) {
+ *_col_name_to_block_idx = block->get_name_to_pos_map();
+ }
+
+ return Status::OK();
+}
+
void OrcReader::_init_system_properties() {
if (_scan_range.__isset.file_type) {
// for compatibility
@@ -2256,6 +2371,7 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
}
}
+ int64_t start_row = _row_reader->getRowNumber();
std::vector<orc::ColumnVectorBatch*> batch_vec;
_fill_batch_vec(batch_vec, _batch.get(), 0);
@@ -2283,7 +2399,8 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
RETURN_IF_ERROR(
_fill_missing_columns(block, _batch->numElements, _lazy_read_ctx.missing_columns));
- RETURN_IF_ERROR(_fill_row_id_columns(block));
+ RETURN_IF_ERROR(_fill_row_id_columns(block, start_row));
+ RETURN_IF_ERROR(_append_iceberg_rowid_column(block, block->rows(), start_row));
if (block->rows() == 0) {
RETURN_IF_ERROR(_convert_dict_cols_to_string_cols(block, nullptr));
@@ -2302,7 +2419,7 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
}
#endif
SCOPED_RAW_TIMER(&_statistics.predicate_filter_time);
- _execute_filter_position_delete_rowids(*_filter);
+ _execute_filter_position_delete_rowids(*_filter, start_row);
#ifndef NDEBUG
for (auto col : *block) {
col.column->sanity_check();
@@ -2354,7 +2471,7 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
_err_msg);
}
}
-
+ int64_t start_row = _row_reader->getRowNumber();
if (!_dict_cols_has_converted && !_dict_filter_cols.empty()) {
for (auto& dict_filter_cols : _dict_filter_cols) {
MutableColumnPtr dict_col_ptr = ColumnInt32::create();
@@ -2408,7 +2525,8 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
RETURN_IF_ERROR(
_fill_missing_columns(block, _batch->numElements, _lazy_read_ctx.missing_columns));
- RETURN_IF_ERROR(_fill_row_id_columns(block));
+ RETURN_IF_ERROR(_fill_row_id_columns(block, start_row));
+ RETURN_IF_ERROR(_append_iceberg_rowid_column(block, block->rows(), start_row));
if (block->rows() == 0) {
RETURN_IF_ERROR(_convert_dict_cols_to_string_cols(block, nullptr));
@@ -2460,18 +2578,18 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
Block::erase_useless_column(block, column_to_keep);
return _convert_dict_cols_to_string_cols(block, &batch_vec);
}
- _execute_filter_position_delete_rowids(result_filter);
+ _execute_filter_position_delete_rowids(result_filter, start_row);
RETURN_IF_CATCH_EXCEPTION(
Block::filter_block_internal(block, columns_to_filter, result_filter));
Block::erase_useless_column(block, column_to_keep);
} else {
if (_delete_rows_filter_ptr) {
- _execute_filter_position_delete_rowids(*_delete_rows_filter_ptr);
+ _execute_filter_position_delete_rowids(*_delete_rows_filter_ptr, start_row);
RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal(
block, columns_to_filter, (*_delete_rows_filter_ptr)));
} else if (_position_delete_ordered_rowids != nullptr) {
std::unique_ptr<IColumn::Filter> filter(new IColumn::Filter(block->rows(), 1));
- _execute_filter_position_delete_rowids(*filter);
+ _execute_filter_position_delete_rowids(*filter, start_row);
RETURN_IF_CATCH_EXCEPTION(
Block::filter_block_internal(block, columns_to_filter, (*filter)));
}
@@ -3240,6 +3358,7 @@ void ORCFileInputStream::_build_large_ranges_input_stripe_streams(
}
}
+<<<<<<< HEAD
void ORCFileInputStream::_collect_profile_before_close() {
if (_file_reader != nullptr) {
_file_reader->collect_profile_before_close();
@@ -3252,10 +3371,13 @@ void ORCFileInputStream::_collect_profile_before_close() {
}
void OrcReader::_execute_filter_position_delete_rowids(IColumn::Filter& filter) {
+=======
+void OrcReader::_execute_filter_position_delete_rowids(IColumn::Filter& filter, int64_t start_row) {
+>>>>>>> c74a1254c2 ([feature](iceberg) Implements iceberg update delete merge into functionality. (#60482))
if (_position_delete_ordered_rowids == nullptr) {
return;
}
- auto start = _row_reader->getRowNumber();
+ auto start = start_row;
auto nums = _batch->numElements;
auto l = std::lower_bound(_position_delete_ordered_rowids->begin(),
_position_delete_ordered_rowids->end(), start);
diff --git a/be/src/format/orc/vorc_reader.h b/be/src/format/orc/vorc_reader.h
index e4b53221cc5..be296f65b4d 100644
--- a/be/src/format/orc/vorc_reader.h
+++ b/be/src/format/orc/vorc_reader.h
@@ -208,6 +208,8 @@ public:
iterator_pair) {
_row_id_column_iterator_pair = iterator_pair;
}
+ void set_iceberg_rowid_params(const std::string& file_path, int32_t partition_spec_id,
+ const std::string& partition_data_json, int row_id_column_pos);
static bool inline is_hive1_col_name(const orc::Type* orc_type_ptr) {
for (uint64_t idx = 0; idx < orc_type_ptr->getSubtypeCount(); idx++) {
@@ -225,6 +227,14 @@ protected:
void _collect_profile_before_close() override;
private:
+ struct IcebergRowIdParams {
+ bool enabled = false;
+ std::string file_path;
+ int32_t partition_spec_id = 0;
+ std::string partition_data_json;
+ int row_id_column_pos = -1;
+ };
+
struct OrcProfile {
RuntimeProfile::Counter* read_time = nullptr;
RuntimeProfile::Counter* read_calls = nullptr;
@@ -312,7 +322,7 @@ private:
std::unique_ptr<orc::SearchArgumentBuilder>& builder);
bool _init_search_argument(const VExprSPtrs& exprs);
- void _execute_filter_position_delete_rowids(IColumn::Filter& filter);
+ void _execute_filter_position_delete_rowids(IColumn::Filter& filter, int64_t start_row);
void _fill_batch_vec(std::vector<orc::ColumnVectorBatch*>& result,
orc::ColumnVectorBatch* batch, int idx);
@@ -623,7 +633,8 @@ private:
return true;
}
- Status _fill_row_id_columns(Block* block);
+ Status _fill_row_id_columns(Block* block, int64_t start_row);
+ Status _append_iceberg_rowid_column(Block* block, size_t rows, int64_t start_row);
bool _seek_to_read_one_line() {
if (_read_by_rows) {
@@ -734,6 +745,7 @@ private:
std::pair<std::shared_ptr<segment_v2::RowIdColumnIteratorV2>, int>
_row_id_column_iterator_pair = {nullptr, -1};
+ IcebergRowIdParams _iceberg_rowid_params;
// Through this node, you can find the file column based on the table column.
std::shared_ptr<TableSchemaChangeHelper::Node> _table_info_node_ptr =
diff --git a/be/src/format/parquet/vparquet_group_reader.cpp b/be/src/format/parquet/vparquet_group_reader.cpp
index 06932beab7f..99961206b57 100644
--- a/be/src/format/parquet/vparquet_group_reader.cpp
+++ b/be/src/format/parquet/vparquet_group_reader.cpp
@@ -29,6 +29,7 @@
#include <ostream>
#include "common/config.h"
+#include "common/consts.h"
#include "common/logging.h"
#include "common/object_pool.h"
#include "common/status.h"
@@ -38,10 +39,13 @@
#include "core/column/column_const.h"
#include "core/column/column_nullable.h"
#include "core/column/column_string.h"
+#include "core/column/column_struct.h"
#include "core/column/column_vector.h"
#include "core/custom_allocator.h"
#include "core/data_type/data_type.h"
+#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_struct.h"
#include "core/data_type/define_primitive_type.h"
#include "core/pod_array.h"
#include "core/types.h"
@@ -73,6 +77,64 @@ struct IOContext;
namespace doris {
#include "common/compile_check_begin.h"
+
+namespace {
+Status build_iceberg_rowid_column(const DataTypePtr& type, const std::string& file_path,
+ const std::vector<rowid_t>& row_ids, int32_t partition_spec_id,
+ const std::string& partition_data_json,
+ MutableColumnPtr* column_out) {
+ if (type == nullptr || column_out == nullptr) {
+ return Status::InvalidArgument("Invalid iceberg rowid column type or output column");
+ }
+
+ MutableColumnPtr column = type->create_column();
+ ColumnNullable* nullable_col = check_and_get_column<ColumnNullable>(column.get());
+ ColumnStruct* struct_col = nullptr;
+ if (nullable_col != nullptr) {
+ struct_col =
+ check_and_get_column<ColumnStruct>(nullable_col->get_nested_column_ptr().get());
+ } else {
+ struct_col = check_and_get_column<ColumnStruct>(column.get());
+ }
+
+ if (struct_col == nullptr || struct_col->tuple_size() < 4) {
+ return Status::InternalError("Invalid iceberg rowid column structure");
+ }
+
+ size_t num_rows = row_ids.size();
+ auto& file_path_col = struct_col->get_column(0);
+ auto& row_pos_col = struct_col->get_column(1);
+ auto& spec_id_col = struct_col->get_column(2);
+ auto& partition_data_col = struct_col->get_column(3);
+
+ file_path_col.reserve(num_rows);
+ row_pos_col.reserve(num_rows);
+ spec_id_col.reserve(num_rows);
+ partition_data_col.reserve(num_rows);
+
+ for (size_t i = 0; i < num_rows; ++i) {
+ file_path_col.insert_data(file_path.data(), file_path.size());
+ }
+ for (size_t i = 0; i < num_rows; ++i) {
+ int64_t row_pos = static_cast<int64_t>(row_ids[i]);
+ row_pos_col.insert_data(reinterpret_cast<const char*>(&row_pos), sizeof(row_pos));
+ }
+ for (size_t i = 0; i < num_rows; ++i) {
+ int32_t spec_id = partition_spec_id;
+ spec_id_col.insert_data(reinterpret_cast<const char*>(&spec_id), sizeof(spec_id));
+ }
+ for (size_t i = 0; i < num_rows; ++i) {
+ partition_data_col.insert_data(partition_data_json.data(), partition_data_json.size());
+ }
+
+ if (nullable_col != nullptr) {
+ nullable_col->get_null_map_data().resize_fill(num_rows, 0);
+ }
+
+ *column_out = std::move(column);
+ return Status::OK();
+}
+} // namespace
const std::vector<int64_t> RowGroupReader::NO_DELETE = {};
static constexpr uint32_t MAX_DICT_CODE_PREDICATE_TO_REWRITE = std::numeric_limits<uint32_t>::max();
@@ -315,6 +377,7 @@ Status RowGroupReader::next_batch(Block* block, size_t batch_size, size_t* read_
RETURN_IF_ERROR(_fill_missing_columns(block, *read_rows, _lazy_read_ctx.missing_columns));
RETURN_IF_ERROR(_fill_row_id_columns(block, *read_rows, modify_row_ids));
+ RETURN_IF_ERROR(_append_iceberg_rowid_column(block, *read_rows, modify_row_ids));
Status st = VExprContext::filter_block(_lazy_read_ctx.conjuncts, block, block->columns());
*read_rows = block->rows();
@@ -331,6 +394,7 @@ Status RowGroupReader::next_batch(Block* block, size_t batch_size, size_t* read_
_fill_partition_columns(block, *read_rows, _lazy_read_ctx.partition_columns));
RETURN_IF_ERROR(_fill_missing_columns(block, *read_rows, _lazy_read_ctx.missing_columns));
RETURN_IF_ERROR(_fill_row_id_columns(block, *read_rows, false));
+ RETURN_IF_ERROR(_append_iceberg_rowid_column(block, *read_rows, false));
#ifndef NDEBUG
for (auto col : *block) {
@@ -516,6 +580,7 @@ Status RowGroupReader::_do_lazy_read(Block* block, size_t batch_size, size_t* re
RETURN_IF_ERROR(_fill_missing_columns(block, pre_read_rows,
_lazy_read_ctx.predicate_missing_columns));
RETURN_IF_ERROR(_fill_row_id_columns(block, pre_read_rows, false));
+ RETURN_IF_ERROR(_append_iceberg_rowid_column(block, pre_read_rows, false));
RETURN_IF_ERROR(_build_pos_delete_filter(pre_read_rows));
@@ -532,11 +597,18 @@ Status RowGroupReader::_do_lazy_read(Block* block, size_t batch_size, size_t* re
#endif
bool can_filter_all = false;
+ bool resize_first_column = _lazy_read_ctx.resize_first_column;
+ if (resize_first_column && _iceberg_rowid_params.enabled) {
+ int row_id_idx = block->get_position_by_name(doris::BeConsts::ICEBERG_ROWID_COL);
+ if (row_id_idx == 0) {
+ resize_first_column = false;
+ }
+ }
{
SCOPED_RAW_TIMER(&_predicate_filter_time);
// generate filter vector
- if (_lazy_read_ctx.resize_first_column) {
+ if (resize_first_column) {
// VExprContext.execute has an optimization, the filtering is executed when block->rows() > 0
// The following process may be tricky and time-consuming, but we have no other way.
block->get_by_position(0).column->assume_mutable()->resize(pre_read_rows);
@@ -590,6 +662,15 @@ Status RowGroupReader::_do_lazy_read(Block* block, size_t batch_size, size_t* re
.column->assume_mutable()
->clear();
}
+ if (_iceberg_rowid_params.enabled) {
+ int row_id_idx =
+ block->get_position_by_name(doris::BeConsts::ICEBERG_ROWID_COL);
+ if (row_id_idx >= 0) {
+ block->get_by_position(static_cast<size_t>(row_id_idx))
+ .column->assume_mutable()
+ ->clear();
+ }
+ }
Block::erase_useless_column(block, origin_column_num);
}
@@ -648,8 +729,17 @@ Status RowGroupReader::_do_lazy_read(Block* block, size_t batch_size, size_t* re
{
SCOPED_RAW_TIMER(&_predicate_filter_time);
if (filter_map.has_filter()) {
- RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal(
- block, _lazy_read_ctx.all_predicate_col_ids, result_filter));
+ std::vector<uint32_t> predicate_columns = _lazy_read_ctx.all_predicate_col_ids;
+ if (_iceberg_rowid_params.enabled) {
+ int row_id_idx = block->get_position_by_name(doris::BeConsts::ICEBERG_ROWID_COL);
+ if (row_id_idx >= 0 &&
+ std::find(predicate_columns.begin(), predicate_columns.end(),
+ static_cast<uint32_t>(row_id_idx)) == predicate_columns.end()) {
+ predicate_columns.push_back(static_cast<uint32_t>(row_id_idx));
+ }
+ }
+ RETURN_IF_CATCH_EXCEPTION(
+ Block::filter_block_internal(block, predicate_columns, result_filter));
Block::erase_useless_column(block, origin_column_num);
} else {
@@ -814,7 +904,7 @@ Status RowGroupReader::_read_empty_batch(size_t batch_size, size_t* read_rows, b
_position_delete_ctx.current_row_id = end_row_id;
*batch_eof = _position_delete_ctx.current_row_id == _position_delete_ctx.last_row_id;
- if (_row_id_column_iterator_pair.first != nullptr) {
+ if (_row_id_column_iterator_pair.first != nullptr || _iceberg_rowid_params.enabled) {
*modify_row_ids = true;
_current_batch_row_ids.clear();
_current_batch_row_ids.resize(*read_rows);
@@ -838,6 +928,10 @@ Status RowGroupReader::_read_empty_batch(size_t batch_size, size_t* read_rows, b
_remaining_rows = 0;
*batch_eof = true;
}
+ if (_iceberg_rowid_params.enabled) {
+ *modify_row_ids = true;
+ RETURN_IF_ERROR(_get_current_batch_row_id(*read_rows));
+ }
}
_total_read_rows += *read_rows;
return Status::OK();
@@ -886,6 +980,56 @@ Status RowGroupReader::_fill_row_id_columns(Block* block, size_t read_rows,
return Status::OK();
}
+Status RowGroupReader::_append_iceberg_rowid_column(Block* block, size_t read_rows,
+ bool is_current_row_ids) {
+ if (!_iceberg_rowid_params.enabled) {
+ return Status::OK();
+ }
+ if (!is_current_row_ids) {
+ RETURN_IF_ERROR(_get_current_batch_row_id(read_rows));
+ }
+
+ int row_id_idx = block->get_position_by_name(doris::BeConsts::ICEBERG_ROWID_COL);
+ if (row_id_idx >= 0) {
+ auto& col_with_type = block->get_by_position(static_cast<size_t>(row_id_idx));
+ MutableColumnPtr row_id_column;
+ RETURN_IF_ERROR(build_iceberg_rowid_column(
+ col_with_type.type, _iceberg_rowid_params.file_path, _current_batch_row_ids,
+ _iceberg_rowid_params.partition_spec_id, _iceberg_rowid_params.partition_data_json,
+ &row_id_column));
+ col_with_type.column = std::move(row_id_column);
+ } else {
+ DataTypes field_types;
+ field_types.push_back(std::make_shared<DataTypeString>());
+ field_types.push_back(std::make_shared<DataTypeInt64>());
+ field_types.push_back(std::make_shared<DataTypeInt32>());
+ field_types.push_back(std::make_shared<DataTypeString>());
+
+ std::vector<std::string> field_names = {"file_path", "row_position", "partition_spec_id",
+ "partition_data"};
+
+ auto row_id_type = std::make_shared<DataTypeStruct>(field_types, field_names);
+ MutableColumnPtr row_id_column;
+ RETURN_IF_ERROR(build_iceberg_rowid_column(
+ row_id_type, _iceberg_rowid_params.file_path, _current_batch_row_ids,
+ _iceberg_rowid_params.partition_spec_id, _iceberg_rowid_params.partition_data_json,
+ &row_id_column));
+ int insert_pos = _iceberg_rowid_params.row_id_column_pos;
+ if (insert_pos < 0 || insert_pos > static_cast<int>(block->columns())) {
+ insert_pos = static_cast<int>(block->columns());
+ }
+ block->insert(static_cast<size_t>(insert_pos),
+ ColumnWithTypeAndName(std::move(row_id_column), row_id_type,
+ doris::BeConsts::ICEBERG_ROWID_COL));
+ }
+
+ if (_col_name_to_block_idx != nullptr) {
+ *_col_name_to_block_idx = block->get_name_to_pos_map();
+ }
+
+ return Status::OK();
+}
+
Status RowGroupReader::_build_pos_delete_filter(size_t read_rows) {
if (!_position_delete_ctx.has_filter) {
_pos_delete_filter_ptr.reset(nullptr);
diff --git a/be/src/format/parquet/vparquet_group_reader.h b/be/src/format/parquet/vparquet_group_reader.h
index e1063b63a9c..cdc5e5b57b7 100644
--- a/be/src/format/parquet/vparquet_group_reader.h
+++ b/be/src/format/parquet/vparquet_group_reader.h
@@ -64,6 +64,13 @@ namespace doris {
class RowGroupReader : public ProfileCollector {
public:
+ struct IcebergRowIdParams {
+ bool enabled = false;
+ std::string file_path;
+ int32_t partition_spec_id = 0;
+ std::string partition_data_json;
+ int row_id_column_pos = -1;
+ };
std::shared_ptr<TableSchemaChangeHelper::Node> _table_info_node_ptr;
static const std::vector<int64_t> NO_DELETE;
@@ -182,6 +189,10 @@ public:
_row_id_column_iterator_pair = iterator_pair;
}
+ void set_iceberg_rowid_params(const IcebergRowIdParams& params) {
+ _iceberg_rowid_params = params;
+ }
+
void set_current_row_group_idx(RowGroupIndex row_group_idx) {
_current_row_group_idx = row_group_idx;
}
@@ -231,6 +242,7 @@ private:
Status _get_current_batch_row_id(size_t read_rows);
Status _fill_row_id_columns(Block* block, size_t read_rows, bool is_current_row_ids);
+ Status _append_iceberg_rowid_column(Block* block, size_t read_rows, bool is_current_row_ids);
io::FileReaderSPtr _file_reader;
std::unordered_map<std::string, std::unique_ptr<ParquetColumnReader>>
@@ -274,6 +286,7 @@ private:
std::vector<rowid_t> _current_batch_row_ids;
std::unordered_map<std::string, uint32_t>* _col_name_to_block_idx = nullptr;
+ IcebergRowIdParams _iceberg_rowid_params;
};
#include "common/compile_check_end.h"
diff --git a/be/src/format/parquet/vparquet_reader.cpp b/be/src/format/parquet/vparquet_reader.cpp
index c21c59cfeeb..d4c402202e6 100644
--- a/be/src/format/parquet/vparquet_reader.cpp
+++ b/be/src/format/parquet/vparquet_reader.cpp
@@ -579,6 +579,20 @@ Status ParquetReader::get_parsed_schema(std::vector<std::string>* col_names,
return Status::OK();
}
+void ParquetReader::set_iceberg_rowid_params(const std::string& file_path,
+ int32_t partition_spec_id,
+ const std::string& partition_data_json,
+ int row_id_column_pos) {
+ _iceberg_rowid_params.enabled = true;
+ _iceberg_rowid_params.file_path = file_path;
+ _iceberg_rowid_params.partition_spec_id = partition_spec_id;
+ _iceberg_rowid_params.partition_data_json = partition_data_json;
+ _iceberg_rowid_params.row_id_column_pos = row_id_column_pos;
+ if (_current_group_reader != nullptr) {
+ _current_group_reader->set_iceberg_rowid_params(_iceberg_rowid_params);
+ }
+}
+
Status ParquetReader::get_columns(std::unordered_map<std::string, DataTypePtr>* name_to_type,
std::unordered_set<std::string>* missing_cols) {
const auto& schema_desc = _file_metadata->schema();
@@ -774,6 +788,9 @@ Status ParquetReader::_next_row_group_reader() {
: group_file_reader,
_read_table_columns, _current_row_group_index.row_group_id, row_group, _ctz, _io_ctx,
position_delete_ctx, _lazy_read_ctx, _state, _column_ids, _filter_column_ids));
+ if (_iceberg_rowid_params.enabled) {
+ _current_group_reader->set_iceberg_rowid_params(_iceberg_rowid_params);
+ }
_row_group_eof = false;
_current_group_reader->set_current_row_group_idx(_current_row_group_index);
diff --git a/be/src/format/parquet/vparquet_reader.h b/be/src/format/parquet/vparquet_reader.h
index eac22cefd4d..9b2512873b6 100644
--- a/be/src/format/parquet/vparquet_reader.h
+++ b/be/src/format/parquet/vparquet_reader.h
@@ -162,6 +162,9 @@ public:
_row_id_column_iterator_pair = iterator_pair;
}
+ void set_iceberg_rowid_params(const std::string& file_path, int32_t partition_spec_id,
+ const std::string& partition_data_json, int row_id_column_pos);
+
bool count_read_rows() override { return true; }
protected:
@@ -342,6 +345,7 @@ private:
std::pair<std::shared_ptr<RowIdColumnIteratorV2>, int> _row_id_column_iterator_pair = {nullptr,
-1};
bool _filter_groups = true;
+ RowGroupReader::IcebergRowIdParams _iceberg_rowid_params;
std::set<uint64_t> _column_ids;
std::set<uint64_t> _filter_column_ids;
diff --git a/be/src/format/table/deletion_vector_reader.h b/be/src/format/table/deletion_vector_reader.h
index 1294a24a8c7..0663f3b2849 100644
--- a/be/src/format/table/deletion_vector_reader.h
+++ b/be/src/format/table/deletion_vector_reader.h
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+#pragma once
+
#include <cstdint>
#include <memory>
#include <string>
diff --git a/be/src/format/table/iceberg_reader.cpp b/be/src/format/table/iceberg_reader.cpp
index d87d6b72396..1f10345a075 100644
--- a/be/src/format/table/iceberg_reader.cpp
+++ b/be/src/format/table/iceberg_reader.cpp
@@ -99,6 +99,7 @@ IcebergTableReader::IcebergTableReader(std::unique_ptr<GenericReader> file_forma
Status IcebergTableReader::get_next_block_inner(Block* block, size_t* read_rows, bool* eof) {
RETURN_IF_ERROR(_expand_block_if_need(block));
+
RETURN_IF_ERROR(_file_format_reader->get_next_block(block, read_rows, eof));
if (_equality_delete_impls.size() > 0) {
@@ -127,6 +128,30 @@ Status IcebergTableReader::init_row_filters() {
return Status::OK();
}
+ // Initialize file information for $row_id generation
+ // Extract from table_desc which contains current file's metadata
+ if (_need_row_id_column) {
+ std::string file_path = table_desc.original_file_path;
+ int32_t partition_spec_id = 0;
+ std::string partition_data_json;
+ if (table_desc.__isset.partition_spec_id) {
+ partition_spec_id = table_desc.partition_spec_id;
+ }
+ if (table_desc.__isset.partition_data_json) {
+ partition_data_json = table_desc.partition_data_json;
+ }
+
+ if (auto* parquet_reader = dynamic_cast<ParquetReader*>(_file_format_reader.get())) {
+ parquet_reader->set_iceberg_rowid_params(file_path, partition_spec_id,
+ partition_data_json, _row_id_column_position);
+ } else if (auto* orc_reader = dynamic_cast<OrcReader*>(_file_format_reader.get())) {
+ orc_reader->set_iceberg_rowid_params(file_path, partition_spec_id, partition_data_json,
+ _row_id_column_position);
+ }
+ LOG(INFO) << "Initialized $row_id generation for file: " << file_path
+ << ", partition_spec_id: " << partition_spec_id;
+ }
+
std::vector<TIcebergDeleteFileDesc> position_delete_files;
std::vector<TIcebergDeleteFileDesc> equality_delete_files;
std::vector<TIcebergDeleteFileDesc> deletion_vector_files;
diff --git a/be/src/format/table/iceberg_reader.h b/be/src/format/table/iceberg_reader.h
index 2fc9ca4f9fd..1b5c154f9d9 100644
--- a/be/src/format/table/iceberg_reader.h
+++ b/be/src/format/table/iceberg_reader.h
@@ -74,6 +74,10 @@ public:
FileMetaCache* meta_cache);
~IcebergTableReader() override = default;
+ void set_need_row_id_column(bool need) { _need_row_id_column = need; }
+ bool need_row_id_column() const { return _need_row_id_column; }
+ void set_row_id_column_position(int position) { _row_id_column_position = position; }
+
Status init_row_filters() final;
Status get_next_block_inner(Block* block, size_t* read_rows, bool* eof) final;
@@ -99,6 +103,10 @@ protected:
std::string, std::unique_ptr<DeleteRows>, std::hash<std::string>, std::equal_to<>,
std::allocator<std::pair<const std::string, std::unique_ptr<DeleteRows>>>, 8,
std::mutex>;
+
+ // $row_id metadata column generation state
+ bool _need_row_id_column = false;
+ int _row_id_column_position = -1;
/**
* https://iceberg.apache.org/spec/#position-delete-files
* The rows in the delete file must be sorted by file_path then position to optimize filtering rows while scanning.
diff --git a/be/src/format/transformer/iceberg_partition_function.cpp b/be/src/format/transformer/iceberg_partition_function.cpp
new file mode 100644
index 00000000000..4417f97b223
--- /dev/null
+++ b/be/src/format/transformer/iceberg_partition_function.cpp
@@ -0,0 +1,401 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/transformer/iceberg_partition_function.h"
+
+#include "common/cast_set.h"
+#include "common/exception.h"
+#include "common/logging.h"
+#include "common/status.h"
+#include "core/column/column_const.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_struct.h"
+#include "core/data_type/data_type_struct.h"
+#include "exec/sink/writer/iceberg/partition_transformers.h"
+#include "format/table/iceberg/partition_spec.h"
+#include "util/string_util.h"
+
+namespace doris {
+#include "common/compile_check_begin.h"
+
+using HashValType = PartitionerBase::HashValType;
+
+static void initialize_shuffle_hashes(std::vector<HashValType>& hashes, size_t rows,
+ ShuffleHashMethod method) {
+ hashes.resize(rows);
+ if (method == ShuffleHashMethod::CRC32C) {
+ constexpr HashValType CRC32C_SHUFFLE_SEED = 0x9E3779B9U;
+ std::fill(hashes.begin(), hashes.end(), CRC32C_SHUFFLE_SEED);
+ } else {
+ std::fill(hashes.begin(), hashes.end(), 0);
+ }
+}
+
+static void update_shuffle_hashes(const ColumnPtr& column, const DataTypePtr& type,
+ HashValType* __restrict result, ShuffleHashMethod method) {
+ if (method == ShuffleHashMethod::CRC32C) {
+ column->update_crc32c_batch(result, nullptr);
+ } else {
+ column->update_crcs_with_value(result, type->get_primitive_type(),
+ cast_set<HashValType>(column->size()));
+ }
+}
+
+static void apply_shuffle_channel_ids(std::vector<HashValType>& hashes, size_t partition_count,
+ ShuffleHashMethod method) {
+ for (auto& h : hashes) {
+ if (method == ShuffleHashMethod::CRC32C) {
+ h = crc32c_shuffle_mix(h) % partition_count;
+ } else {
+ h = h % partition_count;
+ }
+ }
+}
+
+IcebergInsertPartitionFunction::IcebergInsertPartitionFunction(
+ HashValType partition_count, ShuffleHashMethod hash_method,
+ std::vector<TExpr> partition_exprs, std::vector<TIcebergPartitionField> partition_fields)
+ : _partition_count(partition_count),
+ _hash_method(hash_method),
+ _partition_exprs(std::move(partition_exprs)),
+ _partition_fields_spec(std::move(partition_fields)) {}
+
+Status IcebergInsertPartitionFunction::init(const std::vector<TExpr>& texprs) {
+ const auto& exprs = _partition_exprs.empty() ? texprs : _partition_exprs;
+ if (!exprs.empty()) {
+ RETURN_IF_ERROR(VExpr::create_expr_trees(exprs, _partition_expr_ctxs));
+ }
+ if (!_partition_fields_spec.empty()) {
+ _partition_fields.reserve(_partition_fields_spec.size());
+ for (const auto& field : _partition_fields_spec) {
+ VExprContextSPtr ctx;
+ RETURN_IF_ERROR(VExpr::create_expr_tree(field.source_expr, ctx));
+ InsertPartitionField insert_field;
+ insert_field.transform = field.transform;
+ insert_field.expr_ctx = std::move(ctx);
+ insert_field.source_id = field.__isset.source_id ? field.source_id : 0;
+ insert_field.name = field.__isset.name ? field.name : "";
+ _partition_fields.emplace_back(std::move(insert_field));
+ }
+ }
+ return Status::OK();
+}
+
+Status IcebergInsertPartitionFunction::prepare(RuntimeState* state, const RowDescriptor& row_desc) {
+ RETURN_IF_ERROR(VExpr::prepare(_partition_expr_ctxs, state, row_desc));
+ if (!_partition_fields.empty()) {
+ VExprContextSPtrs field_ctxs;
+ field_ctxs.reserve(_partition_fields.size());
+ for (const auto& field : _partition_fields) {
+ field_ctxs.emplace_back(field.expr_ctx);
+ }
+ RETURN_IF_ERROR(VExpr::prepare(field_ctxs, state, row_desc));
+ }
+ return Status::OK();
+}
+
+Status IcebergInsertPartitionFunction::open(RuntimeState* state) {
+ RETURN_IF_ERROR(VExpr::open(_partition_expr_ctxs, state));
+ if (!_partition_fields.empty()) {
+ VExprContextSPtrs field_ctxs;
+ field_ctxs.reserve(_partition_fields.size());
+ for (const auto& field : _partition_fields) {
+ field_ctxs.emplace_back(field.expr_ctx);
+ }
+ RETURN_IF_ERROR(VExpr::open(field_ctxs, state));
+ for (auto& field : _partition_fields) {
+ try {
+ doris::iceberg::PartitionField partition_field(field.source_id, 0, field.name,
+ field.transform);
+ field.transformer = PartitionColumnTransforms::create(
+ partition_field, field.expr_ctx->root()->data_type());
+ } catch (const doris::Exception& e) {
+ LOG(WARNING) << "Merge partitioning fallback to RR: " << e.what();
+ _fallback_to_random = true;
+ _partition_fields.clear();
+ break;
+ }
+ }
+ }
+ return Status::OK();
+}
+
+Status IcebergInsertPartitionFunction::get_partitions(RuntimeState* /*state*/, Block* block,
+ size_t partition_count,
+ std::vector<HashValType>& partitions) const {
+ if (_fallback_to_random) {
+ return Status::InternalError("Merge partitioning fallback to random");
+ }
+ if (partition_count == 0) {
+ return Status::InternalError("Partition count is zero");
+ }
+ if (!_partition_fields.empty()) {
+ RETURN_IF_ERROR(_compute_hashes_with_transform(block, partitions));
+ } else {
+ RETURN_IF_ERROR(_compute_hashes_with_exprs(block, partitions));
+ }
+ apply_shuffle_channel_ids(partitions, partition_count, _hash_method);
+ return Status::OK();
+}
+
+Status IcebergInsertPartitionFunction::clone(RuntimeState* state,
+ std::unique_ptr<PartitionFunction>& function) const {
+ auto* new_function = new IcebergInsertPartitionFunction(
+ _partition_count, _hash_method, _partition_exprs, _partition_fields_spec);
+ function.reset(new_function);
+ RETURN_IF_ERROR(
+ _clone_expr_ctxs(state, _partition_expr_ctxs, new_function->_partition_expr_ctxs));
+ if (!_partition_fields.empty()) {
+ VExprContextSPtrs src_field_ctxs;
+ src_field_ctxs.reserve(_partition_fields.size());
+ for (const auto& field : _partition_fields) {
+ src_field_ctxs.emplace_back(field.expr_ctx);
+ }
+ VExprContextSPtrs dst_field_ctxs;
+ RETURN_IF_ERROR(_clone_expr_ctxs(state, src_field_ctxs, dst_field_ctxs));
+ new_function->_partition_fields.reserve(dst_field_ctxs.size());
+ for (size_t i = 0; i < dst_field_ctxs.size(); ++i) {
+ InsertPartitionField field;
+ field.transform = _partition_fields[i].transform;
+ field.expr_ctx = dst_field_ctxs[i];
+ field.source_id = _partition_fields[i].source_id;
+ field.name = _partition_fields[i].name;
+ new_function->_partition_fields.emplace_back(std::move(field));
+ }
+ }
+ new_function->_fallback_to_random = _fallback_to_random;
+ return Status::OK();
+}
+
+Status IcebergInsertPartitionFunction::_compute_hashes_with_transform(
+ Block* block, std::vector<HashValType>& partitions) const {
+ const size_t rows = block->rows();
+ if (rows == 0) {
+ partitions.clear();
+ return Status::OK();
+ }
+ if (_partition_fields.empty()) {
+ return Status::InternalError("Merge partitioning insert fields are empty");
+ }
+
+ std::vector<int> results(_partition_fields.size());
+ for (size_t i = 0; i < _partition_fields.size(); ++i) {
+ RETURN_IF_ERROR(_partition_fields[i].expr_ctx->execute(block, &results[i]));
+ }
+
+ initialize_shuffle_hashes(partitions, rows, _hash_method);
+ auto* __restrict hash_values = partitions.data();
+ for (size_t i = 0; i < _partition_fields.size(); ++i) {
+ if (_partition_fields[i].transformer == nullptr) {
+ return Status::InternalError("Merge partitioning transform is not initialized");
+ }
+ ColumnWithTypeAndName transformed =
+ _partition_fields[i].transformer->apply(*block, results[i]);
+ const auto& [column, is_const] = unpack_if_const(transformed.column);
+ if (is_const) {
+ // A const column has the same value for all rows in this block,
+ // so it contributes an identical hash delta to every row and does
+ // not affect relative partition assignment. Actual partition
+ // placement is determined by downstream IcebergPartitionWriter.
+ continue;
+ }
+ update_shuffle_hashes(column, transformed.type, hash_values, _hash_method);
+ }
+ return Status::OK();
+}
+
+Status IcebergInsertPartitionFunction::_compute_hashes_with_exprs(
+ Block* block, std::vector<HashValType>& partitions) const {
+ const size_t rows = block->rows();
+ if (rows == 0) {
+ partitions.clear();
+ return Status::OK();
+ }
+ if (_partition_expr_ctxs.empty()) {
+ return Status::InternalError("Merge partitioning insert exprs are empty");
+ }
+
+ std::vector<int> results(_partition_expr_ctxs.size());
+ for (size_t i = 0; i < _partition_expr_ctxs.size(); ++i) {
+ RETURN_IF_ERROR(_partition_expr_ctxs[i]->execute(block, &results[i]));
+ }
+
+ initialize_shuffle_hashes(partitions, rows, _hash_method);
+ auto* __restrict hash_values = partitions.data();
+ for (size_t i = 0; i < results.size(); ++i) {
+ const auto& col_info = block->get_by_position(results[i]);
+ const auto& [column, is_const] = unpack_if_const(col_info.column);
+ if (is_const) {
+ // Same value for all rows — no effect on inter-row partitioning.
+ continue;
+ }
+ update_shuffle_hashes(column, col_info.type, hash_values, _hash_method);
+ }
+ return Status::OK();
+}
+
+Status IcebergInsertPartitionFunction::_clone_expr_ctxs(RuntimeState* state,
+ const VExprContextSPtrs& src,
+ VExprContextSPtrs& dst) const {
+ dst.resize(src.size());
+ for (size_t i = 0; i < src.size(); ++i) {
+ RETURN_IF_ERROR(src[i]->clone(state, dst[i]));
+ }
+ return Status::OK();
+}
+
+IcebergDeletePartitionFunction::IcebergDeletePartitionFunction(HashValType partition_count,
+ ShuffleHashMethod hash_method,
+ std::vector<TExpr> delete_exprs)
+ : _partition_count(partition_count),
+ _hash_method(hash_method),
+ _delete_exprs(std::move(delete_exprs)) {}
+
+Status IcebergDeletePartitionFunction::init(const std::vector<TExpr>& texprs) {
+ const auto& exprs = _delete_exprs.empty() ? texprs : _delete_exprs;
+ if (!exprs.empty()) {
+ RETURN_IF_ERROR(VExpr::create_expr_trees(exprs, _delete_partition_expr_ctxs));
+ }
+ return Status::OK();
+}
+
+Status IcebergDeletePartitionFunction::prepare(RuntimeState* state, const RowDescriptor& row_desc) {
+ return VExpr::prepare(_delete_partition_expr_ctxs, state, row_desc);
+}
+
+Status IcebergDeletePartitionFunction::open(RuntimeState* state) {
+ return VExpr::open(_delete_partition_expr_ctxs, state);
+}
+
+Status IcebergDeletePartitionFunction::get_partitions(RuntimeState* /*state*/, Block* block,
+ size_t partition_count,
+ std::vector<HashValType>& partitions) const {
+ if (partition_count == 0) {
+ return Status::InternalError("Partition count is zero");
+ }
+ RETURN_IF_ERROR(_compute_hashes(block, partitions));
+ apply_shuffle_channel_ids(partitions, partition_count, _hash_method);
+ return Status::OK();
+}
+
+Status IcebergDeletePartitionFunction::clone(RuntimeState* state,
+ std::unique_ptr<PartitionFunction>& function) const {
+ auto* new_function =
+ new IcebergDeletePartitionFunction(_partition_count, _hash_method, _delete_exprs);
+ function.reset(new_function);
+ return _clone_expr_ctxs(state, _delete_partition_expr_ctxs,
+ new_function->_delete_partition_expr_ctxs);
+}
+
+Status IcebergDeletePartitionFunction::_compute_hashes(Block* block,
+ std::vector<HashValType>& partitions) const {
+ const size_t rows = block->rows();
+ if (rows == 0) {
+ partitions.clear();
+ return Status::OK();
+ }
+ if (_delete_partition_expr_ctxs.empty()) {
+ return Status::InternalError("Merge partitioning delete exprs are empty");
+ }
+
+ std::vector<int> results(_delete_partition_expr_ctxs.size());
+ for (size_t i = 0; i < _delete_partition_expr_ctxs.size(); ++i) {
+ RETURN_IF_ERROR(_delete_partition_expr_ctxs[i]->execute(block, &results[i]));
+ }
+
+ initialize_shuffle_hashes(partitions, rows, _hash_method);
+ auto* __restrict hash_values = partitions.data();
+ for (size_t i = 0; i < results.size(); ++i) {
+ const auto& col_info = block->get_by_position(results[i]);
+ const auto& [column, is_const] = unpack_if_const(col_info.column);
+ if (is_const) {
+ // Same value for all rows — no effect on inter-row partitioning.
+ continue;
+ }
+ ColumnPtr hash_col = column;
+ DataTypePtr hash_type = col_info.type;
+ RETURN_IF_ERROR(_get_delete_hash_column(col_info, &hash_col, &hash_type));
+ update_shuffle_hashes(hash_col, hash_type, hash_values, _hash_method);
+ }
+ return Status::OK();
+}
+
+Status IcebergDeletePartitionFunction::_get_delete_hash_column(const ColumnWithTypeAndName& column,
+ ColumnPtr* out_column,
+ DataTypePtr* out_type) const {
+ ColumnPtr hash_col = column.column;
+ DataTypePtr hash_type = column.type;
+ if (auto* nullable_col = check_and_get_column<ColumnNullable>(hash_col.get())) {
+ hash_col = nullable_col->get_nested_column_ptr();
+ hash_type = remove_nullable(hash_type);
+ }
+ const auto* struct_col = check_and_get_column<ColumnStruct>(hash_col.get());
+ const auto* struct_type = check_and_get_data_type<DataTypeStruct>(hash_type.get());
+ if (!struct_col || !struct_type) {
+ *out_column = column.column;
+ *out_type = column.type;
+ return Status::OK();
+ }
+
+ int file_path_idx = _find_file_path_index(*struct_type);
+ if (file_path_idx < 0 || file_path_idx >= struct_col->tuple_size()) {
+ return Status::InternalError("Row id struct missing file_path field");
+ }
+ *out_column = struct_col->get_column_ptr(file_path_idx);
+ *out_type = struct_type->get_element(file_path_idx);
+ return Status::OK();
+}
+
+int IcebergDeletePartitionFunction::_find_file_path_index(const DataTypeStruct& struct_type) const {
+ auto normalize = [](const std::string& name) { return doris::to_lower(name); };
+ auto match_any = [](const std::string& name, std::initializer_list<const char*> candidates) {
+ for (const char* candidate : candidates) {
+ if (name == candidate) {
+ return true;
+ }
+ }
+ return false;
+ };
+
+ int file_path_idx = -1;
+ const auto& field_names = struct_type.get_element_names();
+ for (size_t i = 0; i < field_names.size(); ++i) {
+ std::string name = normalize(field_names[i]);
+ if (file_path_idx < 0 && match_any(name, {"file_path", "data_file_path", "path"})) {
+ file_path_idx = static_cast<int>(i);
+ break;
+ }
+ }
+
+ if (file_path_idx < 0 && !struct_type.get_elements().empty()) {
+ file_path_idx = 0;
+ }
+ return file_path_idx;
+}
+
+Status IcebergDeletePartitionFunction::_clone_expr_ctxs(RuntimeState* state,
+ const VExprContextSPtrs& src,
+ VExprContextSPtrs& dst) const {
+ dst.resize(src.size());
+ for (size_t i = 0; i < src.size(); ++i) {
+ RETURN_IF_ERROR(src[i]->clone(state, dst[i]));
+ }
+ return Status::OK();
+}
+
+#include "common/compile_check_end.h"
+} // namespace doris
diff --git a/be/src/format/transformer/iceberg_partition_function.h b/be/src/format/transformer/iceberg_partition_function.h
new file mode 100644
index 00000000000..216e313e923
--- /dev/null
+++ b/be/src/format/transformer/iceberg_partition_function.h
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <gen_cpp/Partitions_types.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "exec/partitioner/partitioner.h"
+#include "exec/sink/writer/iceberg/partition_transformers.h"
+
+namespace doris {
+#include "common/compile_check_begin.h"
+
+class IcebergInsertPartitionFunction final : public PartitionFunction {
+public:
+ IcebergInsertPartitionFunction(HashValType partition_count, ShuffleHashMethod hash_method,
+ std::vector<TExpr> partition_exprs,
+ std::vector<TIcebergPartitionField> partition_fields);
+
+ Status init(const std::vector<TExpr>& texprs) override;
+ Status prepare(RuntimeState* state, const RowDescriptor& row_desc) override;
+ Status open(RuntimeState* state) override;
+ Status close(RuntimeState* state) override { return Status::OK(); }
+ Status get_partitions(RuntimeState* state, Block* block, size_t partition_count,
+ std::vector<HashValType>& partitions) const override;
+ HashValType partition_count() const override { return _partition_count; }
+ Status clone(RuntimeState* state, std::unique_ptr<PartitionFunction>& function) const override;
+
+ bool fallback_to_random() const { return _fallback_to_random; }
+
+private:
+ struct InsertPartitionField {
+ std::string transform;
+ VExprContextSPtr expr_ctx;
+ std::unique_ptr<PartitionColumnTransform> transformer;
+ int32_t source_id = 0;
+ std::string name;
+ };
+
+ Status _compute_hashes_with_transform(Block* block, std::vector<HashValType>& partitions) const;
+ Status _compute_hashes_with_exprs(Block* block, std::vector<HashValType>& partitions) const;
+ Status _clone_expr_ctxs(RuntimeState* state, const VExprContextSPtrs& src,
+ VExprContextSPtrs& dst) const;
+
+ const HashValType _partition_count;
+ const ShuffleHashMethod _hash_method;
+ std::vector<TExpr> _partition_exprs;
+ std::vector<TIcebergPartitionField> _partition_fields_spec;
+ VExprContextSPtrs _partition_expr_ctxs;
+ std::vector<InsertPartitionField> _partition_fields;
+ bool _fallback_to_random = false;
+};
+
+class IcebergDeletePartitionFunction final : public PartitionFunction {
+public:
+ IcebergDeletePartitionFunction(HashValType partition_count, ShuffleHashMethod hash_method,
+ std::vector<TExpr> delete_exprs);
+
+ Status init(const std::vector<TExpr>& texprs) override;
+ Status prepare(RuntimeState* state, const RowDescriptor& row_desc) override;
+ Status open(RuntimeState* state) override;
+ Status close(RuntimeState* state) override { return Status::OK(); }
+ Status get_partitions(RuntimeState* state, Block* block, size_t partition_count,
+ std::vector<HashValType>& partitions) const override;
+ HashValType partition_count() const override { return _partition_count; }
+ Status clone(RuntimeState* state, std::unique_ptr<PartitionFunction>& function) const override;
+
+private:
+ Status _compute_hashes(Block* block, std::vector<HashValType>& partitions) const;
+ Status _get_delete_hash_column(const ColumnWithTypeAndName& column, ColumnPtr* out_column,
+ DataTypePtr* out_type) const;
+ int _find_file_path_index(const DataTypeStruct& struct_type) const;
+ Status _clone_expr_ctxs(RuntimeState* state, const VExprContextSPtrs& src,
+ VExprContextSPtrs& dst) const;
+
+ const HashValType _partition_count;
+ const ShuffleHashMethod _hash_method;
+ std::vector<TExpr> _delete_exprs;
+ VExprContextSPtrs _delete_partition_expr_ctxs;
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris
diff --git a/be/src/format/transformer/merge_partitioner.cpp b/be/src/format/transformer/merge_partitioner.cpp
new file mode 100644
index 00000000000..8da53d5b109
--- /dev/null
+++ b/be/src/format/transformer/merge_partitioner.cpp
@@ -0,0 +1,365 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/transformer/merge_partitioner.h"
+
+#include <algorithm>
+#include <cstdint>
+
+#include "common/cast_set.h"
+#include "common/config.h"
+#include "common/logging.h"
+#include "common/status.h"
+#include "core/block/block.h"
+#include "core/column/column_const.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_vector.h"
+#include "exec/sink/sink_common.h"
+#include "format/transformer/iceberg_partition_function.h"
+
+namespace doris {
+#include "common/compile_check_begin.h"
+
+namespace {
+int64_t scale_threshold_by_task(int64_t value, int task_num) {
+ if (task_num <= 0) {
+ return value;
+ }
+ int64_t scaled = value / task_num;
+ return scaled == 0 ? value : scaled;
+}
+} // namespace
+
+MergePartitioner::MergePartitioner(size_t partition_count, const TMergePartitionInfo& merge_info,
+ bool use_new_shuffle_hash_method)
+ : PartitionerBase(static_cast<HashValType>(partition_count)),
+ _merge_info(merge_info),
+ _use_new_shuffle_hash_method(use_new_shuffle_hash_method),
+ _insert_random(merge_info.insert_random) {}
+
+Status MergePartitioner::init(const std::vector<TExpr>& /*texprs*/) {
+ VExprContextSPtr op_ctx;
+ RETURN_IF_ERROR(VExpr::create_expr_tree(_merge_info.operation_expr, op_ctx));
+ _operation_expr_ctxs.emplace_back(std::move(op_ctx));
+
+ std::vector<TExpr> insert_exprs;
+ std::vector<TIcebergPartitionField> insert_fields;
+ if (_merge_info.__isset.insert_partition_exprs) {
+ insert_exprs = _merge_info.insert_partition_exprs;
+ }
+ if (_merge_info.__isset.insert_partition_fields) {
+ insert_fields = _merge_info.insert_partition_fields;
+ }
+ if (!insert_exprs.empty() || !insert_fields.empty()) {
+ _insert_partition_function = std::make_unique<IcebergInsertPartitionFunction>(
+ _partition_count, _hash_method(), std::move(insert_exprs),
+ std::move(insert_fields));
+ RETURN_IF_ERROR(_insert_partition_function->init({}));
+ }
+
+ if (_merge_info.__isset.delete_partition_exprs && !_merge_info.delete_partition_exprs.empty()) {
+ _delete_partition_function = std::make_unique<IcebergDeletePartitionFunction>(
+ _partition_count, _hash_method(), _merge_info.delete_partition_exprs);
+ RETURN_IF_ERROR(_delete_partition_function->init({}));
+ }
+ return Status::OK();
+}
+
+Status MergePartitioner::prepare(RuntimeState* state, const RowDescriptor& row_desc) {
+ RETURN_IF_ERROR(VExpr::prepare(_operation_expr_ctxs, state, row_desc));
+ if (_insert_partition_function != nullptr) {
+ RETURN_IF_ERROR(_insert_partition_function->prepare(state, row_desc));
+ }
+ if (_delete_partition_function != nullptr) {
+ RETURN_IF_ERROR(_delete_partition_function->prepare(state, row_desc));
+ }
+ return Status::OK();
+}
+
+Status MergePartitioner::open(RuntimeState* state) {
+ RETURN_IF_ERROR(VExpr::open(_operation_expr_ctxs, state));
+ if (_insert_partition_function != nullptr) {
+ RETURN_IF_ERROR(_insert_partition_function->open(state));
+ if (auto* insert_function =
+ dynamic_cast<IcebergInsertPartitionFunction*>(_insert_partition_function.get());
+ insert_function != nullptr && insert_function->fallback_to_random()) {
+ _insert_random = true;
+ }
+ }
+ if (_delete_partition_function != nullptr) {
+ RETURN_IF_ERROR(_delete_partition_function->open(state));
+ }
+ _init_insert_scaling(state);
+ return Status::OK();
+}
+
+Status MergePartitioner::close(RuntimeState* /*state*/) {
+ return Status::OK();
+}
+
+Status MergePartitioner::do_partitioning(RuntimeState* state, Block* block) const {
+ const size_t rows = block->rows();
+ if (rows == 0) {
+ _channel_ids.clear();
+ return Status::OK();
+ }
+
+ const size_t column_to_keep = block->columns();
+ if (_operation_expr_ctxs.empty()) {
+ return Status::InternalError("Merge partitioning missing operation expression");
+ }
+
+ int op_idx = -1;
+ RETURN_IF_ERROR(_operation_expr_ctxs[0]->execute(block, &op_idx));
+ if (op_idx < 0 || op_idx >= block->columns()) {
+ return Status::InternalError("Merge partitioning missing operation column");
+ }
+ if (op_idx >= cast_set<int>(column_to_keep)) {
+ return Status::InternalError("Merge partitioning requires operation column in input block");
+ }
+
+ const auto& op_column = block->get_by_position(op_idx).column;
+ const auto* op_data = remove_nullable(op_column).get();
+ std::vector<int8_t> ops(rows);
+ bool has_insert = false;
+ bool has_delete = false;
+ bool has_update = false;
+ for (size_t i = 0; i < rows; ++i) {
+ int8_t op = static_cast<int8_t>(op_data->get_int(i));
+ ops[i] = op;
+ if (is_insert_op(op)) {
+ has_insert = true;
+ }
+ if (is_delete_op(op)) {
+ has_delete = true;
+ }
+ if (op == kUpdateOperation) {
+ has_update = true;
+ }
+ }
+
+ if (has_insert && !_insert_random && _insert_partition_function == nullptr) {
+ return Status::InternalError("Merge partitioning insert exprs are empty");
+ }
+ if (has_delete && _delete_partition_function == nullptr) {
+ return Status::InternalError("Merge partitioning delete exprs are empty");
+ }
+
+ std::vector<uint32_t> insert_hashes;
+ std::vector<uint32_t> delete_hashes;
+ const size_t insert_partition_count =
+ _enable_insert_rebalance ? _insert_partition_count : _partition_count;
+ if (has_insert && !_insert_random) {
+ RETURN_IF_ERROR(_insert_partition_function->get_partitions(
+ state, block, insert_partition_count, insert_hashes));
+ }
+ if (has_delete) {
+ RETURN_IF_ERROR(_delete_partition_function->get_partitions(state, block, _partition_count,
+ delete_hashes));
+ }
+ if (has_insert) {
+ if (_insert_random) {
+ if (_non_partition_scaling_threshold > 0) {
+ _insert_data_processed += static_cast<int64_t>(block->bytes());
+ if (_insert_writer_count < static_cast<int>(_partition_count) &&
+ _insert_data_processed >=
+ _insert_writer_count * _non_partition_scaling_threshold) {
+ _insert_writer_count++;
+ }
+ } else {
+ _insert_writer_count = static_cast<int>(_partition_count);
+ }
+ } else if (_enable_insert_rebalance) {
+ _apply_insert_rebalance(ops, insert_hashes, block->bytes());
+ }
+ }
+
+ Block::erase_useless_column(block, column_to_keep);
+
+ _channel_ids.resize(rows);
+ for (size_t i = 0; i < rows; ++i) {
+ const int8_t op = ops[i];
+ if (op == kUpdateOperation) {
+ _channel_ids[i] = delete_hashes[i];
+ continue;
+ }
+ if (is_insert_op(op)) {
+ _channel_ids[i] = _insert_random ? _next_rr_channel() : insert_hashes[i];
+ } else if (is_delete_op(op)) {
+ _channel_ids[i] = delete_hashes[i];
+ } else {
+ return Status::InternalError("Unknown Iceberg merge operation {}", op);
+ }
+ }
+
+ if (has_update) {
+ for (size_t col_idx = 0; col_idx < block->columns(); ++col_idx) {
+ block->replace_by_position_if_const(col_idx);
+ }
+
+ MutableColumns mutable_columns = block->mutate_columns();
+ MutableColumnPtr& op_mut = mutable_columns[op_idx];
+ ColumnInt8* op_values_col = nullptr;
+ if (auto* nullable_col = check_and_get_column<ColumnNullable>(op_mut.get())) {
+ op_values_col =
+ check_and_get_column<ColumnInt8>(nullable_col->get_nested_column_ptr().get());
+ } else {
+ op_values_col = check_and_get_column<ColumnInt8>(op_mut.get());
+ }
+ if (op_values_col == nullptr) {
+ block->set_columns(std::move(mutable_columns));
+ return Status::InternalError("Merge operation column must be tinyint");
+ }
+ auto& op_values = op_values_col->get_data();
+ // First pass: collect update row indices and mark original rows as DELETE.
+ std::vector<size_t> update_rows;
+ for (size_t row = 0; row < rows; ++row) {
+ if (ops[row] != kUpdateOperation) {
+ continue;
+ }
+ op_values[row] = kUpdateDeleteOperation;
+ update_rows.push_back(row);
+ }
+ // Second pass: extract only the update rows into a temporary column,
+ // then batch-append from it. This avoids cloning the entire column.
+ for (size_t col_idx = 0; col_idx < mutable_columns.size(); ++col_idx) {
+ auto tmp = mutable_columns[col_idx]->clone_empty();
+ for (size_t row : update_rows) {
+ tmp->insert_from(*mutable_columns[col_idx], row);
+ }
+ mutable_columns[col_idx]->insert_range_from(*tmp, 0, tmp->size());
+ }
+ // Mark the newly appended rows as INSERT and assign their channels.
+ DCHECK(_insert_random || !insert_hashes.empty());
+ const size_t appended_update_begin = rows;
+ for (size_t idx = 0; idx < update_rows.size(); ++idx) {
+ const size_t row = update_rows[idx];
+ op_values[appended_update_begin + idx] = kUpdateInsertOperation;
+ const uint32_t insert_channel =
+ _insert_random ? _next_rr_channel() : insert_hashes[row];
+ _channel_ids.push_back(insert_channel);
+ }
+ block->set_columns(std::move(mutable_columns));
+ }
+
+ return Status::OK();
+}
+
+Status MergePartitioner::clone(RuntimeState* state, std::unique_ptr<PartitionerBase>& partitioner) {
+ auto* new_partitioner =
+ new MergePartitioner(_partition_count, _merge_info, _use_new_shuffle_hash_method);
+ partitioner.reset(new_partitioner);
+ RETURN_IF_ERROR(
+ _clone_expr_ctxs(state, _operation_expr_ctxs, new_partitioner->_operation_expr_ctxs));
+ if (_insert_partition_function != nullptr) {
+ RETURN_IF_ERROR(_insert_partition_function->clone(
+ state, new_partitioner->_insert_partition_function));
+ }
+ if (_delete_partition_function != nullptr) {
+ RETURN_IF_ERROR(_delete_partition_function->clone(
+ state, new_partitioner->_delete_partition_function));
+ }
+ new_partitioner->_insert_random = _insert_random;
+ new_partitioner->_rr_offset = _rr_offset;
+ return Status::OK();
+}
+
+void MergePartitioner::_apply_insert_rebalance(const std::vector<int8_t>& ops,
+ std::vector<uint32_t>& insert_hashes,
+ size_t block_bytes) const {
+ if (!_enable_insert_rebalance || _insert_writer_assigner == nullptr) {
+ return;
+ }
+ if (insert_hashes.empty() || _insert_partition_count == 0) {
+ return;
+ }
+ std::vector<uint8_t> mask(ops.size(), 0);
+ for (size_t i = 0; i < ops.size(); ++i) {
+ if (is_insert_op(ops[i])) {
+ mask[i] = 1;
+ }
+ }
+ _insert_writer_assigner->assign(insert_hashes, &mask, ops.size(), block_bytes, insert_hashes);
+}
+
+void MergePartitioner::_init_insert_scaling(RuntimeState* state) {
+ _enable_insert_rebalance = false;
+ _insert_partition_count = 0;
+ _insert_data_processed = 0;
+ _insert_writer_count = 1;
+ _insert_writer_assigner.reset();
+ _non_partition_scaling_threshold =
+ config::table_sink_non_partition_write_scaling_data_processed_threshold;
+
+ if (_partition_count == 0) {
+ return;
+ }
+ if (_insert_random) {
+ return;
+ }
+ if (_insert_partition_function == nullptr) {
+ return;
+ }
+
+ int max_partitions_per_writer =
+ config::table_sink_partition_write_max_partition_nums_per_writer;
+ if (max_partitions_per_writer <= 0) {
+ return;
+ }
+ _insert_partition_count = _partition_count * max_partitions_per_writer;
+ if (_insert_partition_count == 0) {
+ return;
+ }
+
+ int task_num = state == nullptr ? 0 : state->task_num();
+ int64_t min_partition_threshold = scale_threshold_by_task(
+ config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold,
+ task_num);
+ int64_t min_data_threshold = scale_threshold_by_task(
+ config::table_sink_partition_write_min_data_processed_rebalance_threshold, task_num);
+
+ _insert_writer_assigner = std::make_unique<SkewedWriterAssigner>(
+ static_cast<int>(_insert_partition_count), static_cast<int>(_partition_count), 1,
+ min_partition_threshold, min_data_threshold);
+ _enable_insert_rebalance = true;
+}
+
+uint32_t MergePartitioner::_next_rr_channel() const {
+ uint32_t writer_count = static_cast<uint32_t>(_partition_count);
+ if (_insert_random && _insert_writer_count > 0) {
+ writer_count = std::min<uint32_t>(static_cast<uint32_t>(_partition_count),
+ static_cast<uint32_t>(_insert_writer_count));
+ }
+ if (writer_count == 0) {
+ return 0;
+ }
+ const uint32_t channel = _rr_offset % writer_count;
+ _rr_offset = (_rr_offset + 1) % writer_count;
+ return channel;
+}
+
+Status MergePartitioner::_clone_expr_ctxs(RuntimeState* state, const VExprContextSPtrs& src,
+ VExprContextSPtrs& dst) const {
+ dst.resize(src.size());
+ for (size_t i = 0; i < src.size(); ++i) {
+ RETURN_IF_ERROR(src[i]->clone(state, dst[i]));
+ }
+ return Status::OK();
+}
+
+#include "common/compile_check_end.h"
+} // namespace doris
diff --git a/be/src/format/transformer/merge_partitioner.h b/be/src/format/transformer/merge_partitioner.h
new file mode 100644
index 00000000000..9cb36d9a5c6
--- /dev/null
+++ b/be/src/format/transformer/merge_partitioner.h
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <gen_cpp/Partitions_types.h>
+
+#include <string>
+
+#include "exec/partitioner/partitioner.h"
+#include "format/transformer/writer_assigner.h"
+
+namespace doris {
+#include "common/compile_check_begin.h"
+
+class MergePartitioner final : public PartitionerBase {
+public:
+ MergePartitioner(size_t partition_count, const TMergePartitionInfo& merge_info,
+ bool use_new_shuffle_hash_method);
+
+ Status init(const std::vector<TExpr>& texprs) override;
+ Status prepare(RuntimeState* state, const RowDescriptor& row_desc) override;
+ Status open(RuntimeState* state) override;
+ Status close(RuntimeState* state) override;
+ Status do_partitioning(RuntimeState* state, Block* block) const override;
+ const std::vector<HashValType>& get_channel_ids() const override { return _channel_ids; }
+ Status clone(RuntimeState* state, std::unique_ptr<PartitionerBase>& partitioner) override;
+
+private:
+ void _apply_insert_rebalance(const std::vector<int8_t>& ops,
+ std::vector<uint32_t>& insert_hashes, size_t block_bytes) const;
+ void _init_insert_scaling(RuntimeState* state);
+ uint32_t _next_rr_channel() const;
+ Status _clone_expr_ctxs(RuntimeState* state, const VExprContextSPtrs& src,
+ VExprContextSPtrs& dst) const;
+ ShuffleHashMethod _hash_method() const {
+ return _use_new_shuffle_hash_method ? ShuffleHashMethod::CRC32C : ShuffleHashMethod::CRC32;
+ }
+
+ TMergePartitionInfo _merge_info;
+ bool _use_new_shuffle_hash_method = false;
+ bool _insert_random = false;
+ bool _enable_insert_rebalance = false;
+ size_t _insert_partition_count = 0;
+ mutable int64_t _insert_data_processed = 0;
+ mutable int _insert_writer_count = 1;
+ int64_t _non_partition_scaling_threshold = 0;
+ VExprContextSPtrs _operation_expr_ctxs;
+ std::unique_ptr<PartitionFunction> _insert_partition_function;
+ std::unique_ptr<PartitionFunction> _delete_partition_function;
+ mutable std::unique_ptr<SkewedWriterAssigner> _insert_writer_assigner;
+ mutable std::vector<uint32_t> _channel_ids;
+ mutable uint32_t _rr_offset = 0;
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris
diff --git a/be/src/format/transformer/writer_assigner.h b/be/src/format/transformer/writer_assigner.h
new file mode 100644
index 00000000000..173b04867d6
--- /dev/null
+++ b/be/src/format/transformer/writer_assigner.h
@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+#include "exec/connector/skewed_partition_rebalancer.h"
+
+namespace doris {
+#include "common/compile_check_begin.h"
+
+class WriterAssigner {
+public:
+ virtual ~WriterAssigner() = default;
+
+ virtual void assign(const std::vector<uint32_t>& partition_ids,
+ const std::vector<uint8_t>* mask, size_t rows, size_t block_bytes,
+ std::vector<uint32_t>& writer_ids) = 0;
+};
+
+class IdentityWriterAssigner final : public WriterAssigner {
+public:
+ void assign(const std::vector<uint32_t>& partition_ids, const std::vector<uint8_t>* mask,
+ size_t rows, size_t /*block_bytes*/, std::vector<uint32_t>& writer_ids) override {
+ if (rows == 0) {
+ return;
+ }
+ if (writer_ids.size() != rows && &writer_ids != &partition_ids) {
+ writer_ids.resize(rows);
+ }
+ if (mask == nullptr) {
+ for (size_t i = 0; i < rows; ++i) {
+ writer_ids[i] = partition_ids[i];
+ }
+ return;
+ }
+ for (size_t i = 0; i < rows; ++i) {
+ if ((*mask)[i] == 0) {
+ continue;
+ }
+ writer_ids[i] = partition_ids[i];
+ }
+ }
+};
+
+class SkewedWriterAssigner final : public WriterAssigner {
+public:
+ SkewedWriterAssigner(int partition_count, int task_count, int task_bucket_count,
+ long min_partition_data_processed_rebalance_threshold,
+ long min_data_processed_rebalance_threshold)
+ : _rebalancer(partition_count, task_count, task_bucket_count,
+ min_partition_data_processed_rebalance_threshold,
+ min_data_processed_rebalance_threshold),
+ _partition_row_counts(partition_count, 0),
+ _partition_writer_ids(partition_count, -1),
+ _partition_writer_indexes(partition_count, 0) {}
+
+ void assign(const std::vector<uint32_t>& partition_ids, const std::vector<uint8_t>* mask,
+ size_t rows, size_t block_bytes, std::vector<uint32_t>& writer_ids) override {
+ if (rows == 0 || _partition_row_counts.empty()) {
+ return;
+ }
+ if (writer_ids.size() != rows && &writer_ids != &partition_ids) {
+ writer_ids.resize(rows);
+ }
+
+ std::fill(_partition_row_counts.begin(), _partition_row_counts.end(), 0);
+ std::fill(_partition_writer_ids.begin(), _partition_writer_ids.end(), -1);
+ _rebalancer.rebalance();
+
+ const size_t partition_count = _partition_row_counts.size();
+ for (size_t i = 0; i < rows; ++i) {
+ if (mask != nullptr && (*mask)[i] == 0) {
+ continue;
+ }
+ const uint32_t partition_id = partition_ids[i];
+ if (partition_id >= partition_count) {
+ continue;
+ }
+ _partition_row_counts[partition_id] += 1;
+ int writer_id = _partition_writer_ids[partition_id];
+ if (writer_id == -1) {
+ writer_id = _get_next_writer_id(partition_id);
+ _partition_writer_ids[partition_id] = writer_id;
+ }
+ writer_ids[i] = static_cast<uint32_t>(writer_id);
+ }
+
+ for (size_t i = 0; i < partition_count; ++i) {
+ if (_partition_row_counts[i] > 0) {
+ _rebalancer.add_partition_row_count(static_cast<int>(i), _partition_row_counts[i]);
+ }
+ }
+ _rebalancer.add_data_processed(static_cast<long>(block_bytes));
+ }
+
+private:
+ int _get_next_writer_id(uint32_t partition_id) {
+ return _rebalancer.get_task_id(partition_id, _partition_writer_indexes[partition_id]++);
+ }
+
+ SkewedPartitionRebalancer _rebalancer;
+ std::vector<int> _partition_row_counts;
+ std::vector<int> _partition_writer_ids;
+ std::vector<int> _partition_writer_indexes;
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris
diff --git a/be/test/core/value/merge_partitioner_test.cpp b/be/test/core/value/merge_partitioner_test.cpp
new file mode 100644
index 00000000000..aeb18053645
--- /dev/null
+++ b/be/test/core/value/merge_partitioner_test.cpp
@@ -0,0 +1,351 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format/transformer/merge_partitioner.h"
+
+#include <gen_cpp/Exprs_types.h>
+#include <gen_cpp/Partitions_types.h>
+#include <gtest/gtest.h>
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "common/config.h"
+#include "common/object_pool.h"
+#include "core/block/block.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_string.h"
+#include "core/column/column_struct.h"
+#include "core/column/column_vector.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_struct.h"
+#include "core/types.h"
+#include "runtime/descriptor_helper.h"
+#include "runtime/descriptors.h"
+#include "testutil/mock/mock_runtime_state.h"
+
+namespace doris {
+
+namespace {
+
+template <typename T>
+class ScopedConfigValue {
+public:
+ ScopedConfigValue(T& ref, T value) : _ref(ref), _old(ref) { _ref = value; }
+ ~ScopedConfigValue() { _ref = _old; }
+
+private:
+ T& _ref;
+ T _old;
+};
+
+} // namespace
+
+class MergePartitionerTest : public ::testing::Test {
+protected:
+ void SetUp() override { _build_descriptors(); }
+
+ TExpr _make_slot_ref_expr(TSlotId slot_id, TTupleId tuple_id, PrimitiveType type) {
+ TExprNode node;
+ node.__set_node_type(TExprNodeType::SLOT_REF);
+ node.__set_num_children(0);
+
+ TSlotRef slot_ref;
+ slot_ref.__set_slot_id(slot_id);
+ slot_ref.__set_tuple_id(tuple_id);
+ node.__set_slot_ref(slot_ref);
+
+ TTypeDesc type_desc = create_type_desc(type);
+ type_desc.__set_is_nullable(false);
+ node.__set_type(type_desc);
+ node.__set_is_nullable(false);
+
+ TExpr expr;
+ expr.nodes.emplace_back(std::move(node));
+ return expr;
+ }
+
+ TMergePartitionInfo _make_base_merge_info(bool insert_random) {
+ TMergePartitionInfo merge_info;
+ merge_info.__set_operation_expr(
+ _make_slot_ref_expr(_operation_slot_id, _tuple_id, TYPE_TINYINT));
+ merge_info.__set_insert_random(insert_random);
+ return merge_info;
+ }
+
+ Block _build_block(const std::vector<int8_t>& ops, const std::vector<std::string>& file_paths,
+ const std::vector<int64_t>& row_positions,
+ const std::vector<int32_t>& insert_keys,
+ const std::vector<std::string>& delete_keys) {
+ const size_t rows = ops.size();
+ EXPECT_EQ(rows, file_paths.size());
+ EXPECT_EQ(rows, row_positions.size());
+ EXPECT_EQ(rows, insert_keys.size());
+ EXPECT_EQ(rows, delete_keys.size());
+
+ Block block;
+
+ auto op_col = ColumnInt8::create();
+ for (auto op : ops) {
+ op_col->insert_value(op);
+ }
+ block.insert(ColumnWithTypeAndName(std::move(op_col), std::make_shared<DataTypeInt8>(),
+ "operation"));
+
+ auto file_path_col = ColumnString::create();
+ auto row_pos_col = ColumnInt64::create();
+ for (size_t i = 0; i < rows; ++i) {
+ file_path_col->insert_data(file_paths[i].data(), file_paths[i].size());
+ row_pos_col->insert_value(row_positions[i]);
+ }
+ Columns struct_cols;
+ struct_cols.push_back(std::move(file_path_col));
+ struct_cols.push_back(std::move(row_pos_col));
+
+ auto struct_col = ColumnStruct::create(std::move(struct_cols));
+ DataTypes struct_types {std::make_shared<DataTypeString>(),
+ std::make_shared<DataTypeInt64>()};
+ Strings field_names {"file_path", "row_position"};
+ auto struct_type = std::make_shared<DataTypeStruct>(struct_types, field_names);
+
+ block.insert(ColumnWithTypeAndName(std::move(struct_col), struct_type, "row_id"));
+
+ auto insert_col = ColumnInt32::create();
+ for (auto key : insert_keys) {
+ insert_col->insert_value(key);
+ }
+ block.insert(ColumnWithTypeAndName(std::move(insert_col), std::make_shared<DataTypeInt32>(),
+ "insert_key"));
+
+ auto delete_col = ColumnString::create();
+ for (const auto& key : delete_keys) {
+ delete_col->insert_data(key.data(), key.size());
+ }
+ block.insert(ColumnWithTypeAndName(std::move(delete_col),
+ std::make_shared<DataTypeString>(), "delete_key"));
+
+ return block;
+ }
+
+ const ColumnInt8& _get_op_column(const Block& block) const {
+ const auto& column = block.get_by_position(0).column;
+ if (auto* nullable_col = check_and_get_column<ColumnNullable>(column.get())) {
+ return *check_and_get_column<ColumnInt8>(nullable_col->get_nested_column_ptr().get());
+ }
+ return *check_and_get_column<ColumnInt8>(column.get());
+ }
+
+ void _build_descriptors() {
+ TDescriptorTableBuilder dtb;
+ TTupleDescriptorBuilder tuple_builder;
+ tuple_builder.add_slot(TSlotDescriptorBuilder()
+ .type(TYPE_TINYINT)
+ .nullable(false)
+ .column_name("operation")
+ .column_pos(1)
+ .build());
+ tuple_builder.add_slot(TSlotDescriptorBuilder()
+ .type(TYPE_STRING)
+ .nullable(false)
+ .column_name("row_id")
+ .column_pos(2)
+ .build());
+ tuple_builder.add_slot(TSlotDescriptorBuilder()
+ .type(TYPE_INT)
+ .nullable(false)
+ .column_name("insert_key")
+ .column_pos(3)
+ .build());
+ tuple_builder.add_slot(TSlotDescriptorBuilder()
+ .type(TYPE_STRING)
+ .nullable(false)
+ .column_name("delete_key")
+ .column_pos(4)
+ .build());
+ tuple_builder.build(&dtb);
+ TDescriptorTable thrift_tbl = dtb.desc_tbl();
+
+ DescriptorTbl* desc_tbl = nullptr;
+ auto st = DescriptorTbl::create(&_pool, thrift_tbl, &desc_tbl);
+ ASSERT_TRUE(st.ok()) << st.to_string();
+ _state.set_desc_tbl(desc_tbl);
+
+ ASSERT_FALSE(thrift_tbl.tupleDescriptors.empty());
+ _tuple_id = thrift_tbl.tupleDescriptors[0].id;
+ _row_desc = std::make_unique<RowDescriptor>(*desc_tbl, std::vector<TTupleId> {_tuple_id});
+
+ auto find_slot_id = [&](const std::string& name) -> TSlotId {
+ for (const auto& slot : thrift_tbl.slotDescriptors) {
+ if (slot.colName == name) {
+ return slot.id;
+ }
+ }
+ return -1;
+ };
+ _operation_slot_id = find_slot_id("operation");
+ _row_id_slot_id = find_slot_id("row_id");
+ _insert_key_slot_id = find_slot_id("insert_key");
+ _delete_key_slot_id = find_slot_id("delete_key");
+
+ ASSERT_GE(_operation_slot_id, 0);
+ ASSERT_GE(_row_id_slot_id, 0);
+ ASSERT_GE(_insert_key_slot_id, 0);
+ ASSERT_GE(_delete_key_slot_id, 0);
+ }
+
+ ObjectPool _pool;
+ MockRuntimeState _state;
+ std::unique_ptr<RowDescriptor> _row_desc;
+ TTupleId _tuple_id = 0;
+ TSlotId _operation_slot_id = -1;
+ TSlotId _row_id_slot_id = -1;
+ TSlotId _insert_key_slot_id = -1;
+ TSlotId _delete_key_slot_id = -1;
+};
+
+TEST_F(MergePartitionerTest, TestInsertDeleteUpdatePartitioning) {
+ TMergePartitionInfo merge_info = _make_base_merge_info(false);
+ merge_info.__set_insert_partition_exprs(
+ {_make_slot_ref_expr(_insert_key_slot_id, _tuple_id, TYPE_INT)});
+ merge_info.__set_delete_partition_exprs(
+ {_make_slot_ref_expr(_row_id_slot_id, _tuple_id, TYPE_STRING)});
+
+ MergePartitioner partitioner(4, merge_info, false);
+ ASSERT_TRUE(partitioner.init({}).ok());
+ ASSERT_TRUE(partitioner.prepare(&_state, *_row_desc).ok());
+ ASSERT_TRUE(partitioner.open(&_state).ok());
+
+ Block block =
+ _build_block({3, 1, 1, 2, 2}, {"path_u", "path_i1", "path_i2", "path_d", "path_d"},
+ {10, 20, 21, 30, 31}, {100, 42, 42, 7, 8}, {"d1", "d2", "d3", "d4", "d4"});
+
+ ASSERT_TRUE(partitioner.do_partitioning(&_state, &block).ok());
+
+ const auto& channel_ids = partitioner.get_channel_ids();
+ ASSERT_EQ(block.rows(), channel_ids.size());
+ EXPECT_EQ(6, block.rows());
+
+ const auto& op_values = _get_op_column(block).get_data();
+ ASSERT_EQ(6, op_values.size());
+ EXPECT_EQ(5, op_values[0]);
+ EXPECT_EQ(1, op_values[1]);
+ EXPECT_EQ(1, op_values[2]);
+ EXPECT_EQ(2, op_values[3]);
+ EXPECT_EQ(2, op_values[4]);
+ EXPECT_EQ(4, op_values[5]);
+
+ EXPECT_EQ(channel_ids[1], channel_ids[2]);
+ EXPECT_EQ(channel_ids[3], channel_ids[4]);
+
+ ASSERT_TRUE(partitioner.close(&_state).ok());
+}
+
+TEST_F(MergePartitionerTest, TestMissingInsertPartitionExprs) {
+ TMergePartitionInfo merge_info = _make_base_merge_info(false);
+
+ MergePartitioner partitioner(4, merge_info, false);
+ ASSERT_TRUE(partitioner.init({}).ok());
+ ASSERT_TRUE(partitioner.prepare(&_state, *_row_desc).ok());
+ ASSERT_TRUE(partitioner.open(&_state).ok());
+
+ Block block = _build_block({1}, {"path_i"}, {1}, {100}, {"d1"});
+ Status status = partitioner.do_partitioning(&_state, &block);
+ ASSERT_FALSE(status.ok());
+ EXPECT_NE(std::string::npos, status.to_string().find("insert exprs are empty"));
+
+ ASSERT_TRUE(partitioner.close(&_state).ok());
+}
+
+TEST_F(MergePartitionerTest, TestMissingDeletePartitionExprs) {
+ TMergePartitionInfo merge_info = _make_base_merge_info(true);
+
+ MergePartitioner partitioner(4, merge_info, false);
+ ASSERT_TRUE(partitioner.init({}).ok());
+ ASSERT_TRUE(partitioner.prepare(&_state, *_row_desc).ok());
+ ASSERT_TRUE(partitioner.open(&_state).ok());
+
+ Block block = _build_block({2}, {"path_d"}, {10}, {1}, {"d1"});
+ Status status = partitioner.do_partitioning(&_state, &block);
+ ASSERT_FALSE(status.ok());
+ EXPECT_NE(std::string::npos, status.to_string().find("delete exprs are empty"));
+
+ ASSERT_TRUE(partitioner.close(&_state).ok());
+}
+
+TEST_F(MergePartitionerTest, TestInsertPartitionFieldsIdentity) {
+ ScopedConfigValue<int32_t> max_partition_guard(
+ config::table_sink_partition_write_max_partition_nums_per_writer, 0);
+
+ TMergePartitionInfo merge_info = _make_base_merge_info(false);
+ TIcebergPartitionField field;
+ field.__set_transform("identity");
+ field.__set_source_expr(_make_slot_ref_expr(_insert_key_slot_id, _tuple_id, TYPE_INT));
+ field.__set_name("id");
+ field.__set_source_id(1);
+ merge_info.__set_insert_partition_fields({field});
+
+ MergePartitioner partitioner(4, merge_info, false);
+ ASSERT_TRUE(partitioner.init({}).ok());
+ ASSERT_TRUE(partitioner.prepare(&_state, *_row_desc).ok());
+ ASSERT_TRUE(partitioner.open(&_state).ok());
+
+ Block block = _build_block({1, 1}, {"path_i1", "path_i2"}, {1, 2}, {9, 9}, {"d1", "d2"});
+ ASSERT_TRUE(partitioner.do_partitioning(&_state, &block).ok());
+
+ const auto& channel_ids = partitioner.get_channel_ids();
+ ASSERT_EQ(2, channel_ids.size());
+ EXPECT_EQ(channel_ids[0], channel_ids[1]);
+
+ ASSERT_TRUE(partitioner.close(&_state).ok());
+}
+
+TEST_F(MergePartitionerTest, TestInvalidTransformFallbacksToRandom) {
+ ScopedConfigValue<int64_t> threshold_guard(
+ config::table_sink_non_partition_write_scaling_data_processed_threshold, 0);
+
+ TMergePartitionInfo merge_info = _make_base_merge_info(false);
+ TIcebergPartitionField field;
+ field.__set_transform("invalid");
+ field.__set_source_expr(_make_slot_ref_expr(_insert_key_slot_id, _tuple_id, TYPE_INT));
+ field.__set_name("id");
+ field.__set_source_id(1);
+ merge_info.__set_insert_partition_fields({field});
+
+ MergePartitioner partitioner(3, merge_info, false);
+ ASSERT_TRUE(partitioner.init({}).ok());
+ ASSERT_TRUE(partitioner.prepare(&_state, *_row_desc).ok());
+ ASSERT_TRUE(partitioner.open(&_state).ok());
+
+ Block block = _build_block({1, 1, 1, 1}, {"path_1", "path_2", "path_3", "path_4"}, {1, 2, 3, 4},
+ {10, 11, 12, 13}, {"d1", "d2", "d3", "d4"});
+ ASSERT_TRUE(partitioner.do_partitioning(&_state, &block).ok());
+
+ const auto& channel_ids = partitioner.get_channel_ids();
+ ASSERT_EQ(4, channel_ids.size());
+ EXPECT_EQ(0U, channel_ids[0]);
+ EXPECT_EQ(1U, channel_ids[1]);
+ EXPECT_EQ(2U, channel_ids[2]);
+ EXPECT_EQ(0U, channel_ids[3]);
+
+ ASSERT_TRUE(partitioner.close(&_state).ok());
+}
+
+} // namespace doris
diff --git a/be/test/exec/exchange/exchange_writer_test.cpp b/be/test/exec/exchange/exchange_writer_test.cpp
index 5624dc1ccda..28481d08eb3 100644
--- a/be/test/exec/exchange/exchange_writer_test.cpp
+++ b/be/test/exec/exchange/exchange_writer_test.cpp
@@ -60,6 +60,49 @@ static std::vector<std::shared_ptr<Channel>> make_disabled_channels(
return channels;
}
+class RowExpandingPartitioner final : public PartitionerBase {
+public:
+ RowExpandingPartitioner() : PartitionerBase(2) {}
+
+ Status init(const std::vector<TExpr>& /*texprs*/) override { return Status::OK(); }
+
+ Status prepare(RuntimeState* /*state*/, const RowDescriptor& /*row_desc*/) override {
+ return Status::OK();
+ }
+
+ Status open(RuntimeState* /*state*/) override { return Status::OK(); }
+
+ Status close(RuntimeState* /*state*/) override { return Status::OK(); }
+
+ Status do_partitioning(RuntimeState* /*state*/, Block* block) const override {
+ if (block->rows() == 0) {
+ _channel_ids.clear();
+ return Status::OK();
+ }
+
+ _channel_ids.assign(block->rows(), 0);
+
+ auto mutable_columns = block->mutate_columns();
+ for (size_t col_idx = 0; col_idx < mutable_columns.size(); ++col_idx) {
+ mutable_columns[col_idx]->insert_from(*mutable_columns[col_idx], 0);
+ }
+ block->set_columns(std::move(mutable_columns));
+
+ _channel_ids.push_back(1);
+ return Status::OK();
+ }
+
+ const std::vector<HashValType>& get_channel_ids() const override { return _channel_ids; }
+
+ Status clone(RuntimeState* /*state*/, std::unique_ptr<PartitionerBase>& partitioner) override {
+ partitioner = std::make_unique<RowExpandingPartitioner>();
+ return Status::OK();
+ }
+
+private:
+ mutable std::vector<HashValType> _channel_ids;
+};
+
TEST(TrivialExchangeWriterTest, BasicDistribution) {
MockRuntimeState state;
ExchangeSinkLocalState local_state(&state);
@@ -145,6 +188,29 @@ TEST(TrivialExchangeWriterTest, EmptyInput) {
EXPECT_EQ(writer._origin_row_idx.size(), 0U);
}
+TEST(TrivialExchangeWriterTest, WriteUsesRowsAfterPartitioning) {
+ MockRuntimeState state;
+ ExchangeSinkLocalState local_state(&state);
+ local_state._partitioner = std::make_unique<RowExpandingPartitioner>();
+ ExchangeTrivialWriter writer {local_state};
+
+ const size_t channel_count = 2;
+ local_state.channels = make_disabled_channels(&local_state, channel_count);
+
+ Block block = ColumnHelper::create_block<DataTypeInt32>({7});
+ Status st = writer.write(&state, &block, /*eos=*/false);
+ ASSERT_TRUE(st.ok()) << st.to_string();
+
+ ASSERT_EQ(block.rows(), 2U);
+ ASSERT_EQ(writer._origin_row_idx.size(), 2U);
+ EXPECT_EQ(writer._origin_row_idx[0], 0U);
+ EXPECT_EQ(writer._origin_row_idx[1], 1U);
+
+ ASSERT_EQ(writer._channel_rows_histogram.size(), channel_count);
+ EXPECT_EQ(writer._channel_rows_histogram[0], 1U);
+ EXPECT_EQ(writer._channel_rows_histogram[1], 1U);
+}
+
TEST(OlapExchangeWriterTest, NeedCheckSkipsInvalidChannelIds) {
MockRuntimeState state;
ExchangeSinkLocalState local_state(&state);
diff --git a/be/test/exec/sink/viceberg_delete_sink_test.cpp b/be/test/exec/sink/viceberg_delete_sink_test.cpp
new file mode 100644
index 00000000000..fd5bec84442
--- /dev/null
+++ b/be/test/exec/sink/viceberg_delete_sink_test.cpp
@@ -0,0 +1,485 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exec/sink/viceberg_delete_sink.h"
+
+#include <gtest/gtest.h>
+
+#include "common/consts.h"
+#include "common/object_pool.h"
+#include "core/block/block.h"
+#include "core/column/column_string.h"
+#include "core/column/column_struct.h"
+#include "core/column/column_vector.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_struct.h"
+#include "gen_cpp/DataSinks_types.h"
+#include "gen_cpp/Types_types.h"
+#include "runtime/runtime_state.h"
+
+namespace doris {
+
+class VIcebergDeleteSinkTest : public testing::Test {
+protected:
+ void SetUp() override {
+ // Create a basic TDataSink for testing
+ _t_data_sink.__set_type(TDataSinkType::ICEBERG_DELETE_SINK);
+
+ TIcebergDeleteSink delete_sink;
+ delete_sink.__set_db_name("test_db");
+ delete_sink.__set_tb_name("test_table");
+ delete_sink.__set_delete_type(TFileContent::POSITION_DELETES);
+ delete_sink.__set_file_format(TFileFormatType::FORMAT_PARQUET);
+ delete_sink.__set_compress_type(TFileCompressType::SNAPPYBLOCK);
+ delete_sink.__set_output_path("/tmp/iceberg/test");
+ delete_sink.__set_table_location("/tmp/iceberg/test_table");
+
+ std::map<std::string, std::string> hadoop_conf;
+ hadoop_conf["fs.defaultFS"] = "hdfs://localhost:9000";
+ delete_sink.__set_hadoop_config(hadoop_conf);
+
+ _t_data_sink.__set_iceberg_delete_sink(delete_sink);
+ }
+
+ TDataSink _t_data_sink;
+};
+
+TEST_F(VIcebergDeleteSinkTest, TestInitProperties) {
+ VExprContextSPtrs output_exprs;
+ auto sink = std::make_shared<VIcebergDeleteSink>(_t_data_sink, output_exprs, nullptr, nullptr);
+
+ ObjectPool pool;
+ Status status = sink->init_properties(&pool);
+ ASSERT_TRUE(status.ok());
+}
+
+TEST_F(VIcebergDeleteSinkTest, TestGetRowIdColumnIndex) {
+ VExprContextSPtrs output_exprs;
+ auto sink = std::make_shared<VIcebergDeleteSink>(_t_data_sink, output_exprs, nullptr, nullptr);
+
+ // Create a block with $row_id column
+ Block block;
+
+ // Add a regular column
+ auto col1 = ColumnInt32::create();
+ col1->insert_value(1);
+ block.insert(ColumnWithTypeAndName(std::move(col1), std::make_shared<DataTypeInt32>(), "id"));
+
+ // Add __DORIS_ICEBERG_ROWID_COL__ column (as struct)
+ auto file_path_col = ColumnString::create();
+ file_path_col->insert_data("file1.parquet", 13);
+
+ auto row_pos_col = ColumnInt64::create();
+ row_pos_col->insert_value(100);
+
+ Columns struct_cols;
+ struct_cols.push_back(std::move(file_path_col));
+ struct_cols.push_back(std::move(row_pos_col));
+
+ auto struct_col = ColumnStruct::create(std::move(struct_cols));
+
+ DataTypes struct_types;
+ struct_types.push_back(std::make_shared<DataTypeString>());
+ struct_types.push_back(std::make_shared<DataTypeInt64>());
+
+ Strings field_names = {"file_path", "row_position"};
+ auto struct_type = std::make_shared<DataTypeStruct>(struct_types, field_names);
+
+ block.insert(ColumnWithTypeAndName(std::move(struct_col), struct_type,
+ doris::BeConsts::ICEBERG_ROWID_COL));
+
+ // Test finding the column
+ int idx = sink->_get_row_id_column_index(block);
+ ASSERT_EQ(1, idx);
+ ASSERT_EQ(doris::BeConsts::ICEBERG_ROWID_COL, block.get_by_position(idx).name);
+}
+
+TEST_F(VIcebergDeleteSinkTest, TestGetRowIdColumnIndexWithIcebergRowId) {
+ VExprContextSPtrs output_exprs;
+ auto sink = std::make_shared<VIcebergDeleteSink>(_t_data_sink, output_exprs, nullptr, nullptr);
+
+ Block block;
+
+ auto file_path_col = ColumnString::create();
+ file_path_col->insert_data("file1.parquet", 13);
+
+ auto row_pos_col = ColumnInt64::create();
+ row_pos_col->insert_value(100);
+
+ Columns struct_cols;
+ struct_cols.push_back(std::move(file_path_col));
+ struct_cols.push_back(std::move(row_pos_col));
+
+ auto struct_col = ColumnStruct::create(std::move(struct_cols));
+
+ DataTypes struct_types;
+ struct_types.push_back(std::make_shared<DataTypeString>());
+ struct_types.push_back(std::make_shared<DataTypeInt64>());
+
+ Strings field_names = {"file_path", "row_position"};
+ auto struct_type = std::make_shared<DataTypeStruct>(struct_types, field_names);
+
+ block.insert(ColumnWithTypeAndName(std::move(struct_col), struct_type,
+ doris::BeConsts::ICEBERG_ROWID_COL));
+
+ int idx = sink->_get_row_id_column_index(block);
+ ASSERT_EQ(0, idx);
+ ASSERT_EQ(doris::BeConsts::ICEBERG_ROWID_COL, block.get_by_position(idx).name);
+}
+
+TEST_F(VIcebergDeleteSinkTest, TestCollectPositionDeletes) {
+ VExprContextSPtrs output_exprs;
+ auto sink = std::make_shared<VIcebergDeleteSink>(_t_data_sink, output_exprs, nullptr, nullptr);
+
+ // Create a block with multiple $row_id entries
+ Block block;
+
+ auto file_path_col = ColumnString::create();
+ file_path_col->insert_data("file1.parquet", 13);
+ file_path_col->insert_data("file2.parquet", 13);
+ file_path_col->insert_data("file1.parquet", 13);
+
+ auto row_pos_col = ColumnInt64::create();
+ row_pos_col->insert_value(100);
+ row_pos_col->insert_value(200);
+ row_pos_col->insert_value(150);
+
+ auto spec_id_col = ColumnInt32::create();
+ spec_id_col->insert_value(1);
+ spec_id_col->insert_value(2);
+ spec_id_col->insert_value(1);
+
+ auto partition_data_col = ColumnString::create();
+ partition_data_col->insert_data("p=1", 3);
+ partition_data_col->insert_data("p=2", 3);
+ partition_data_col->insert_data("p=1", 3);
+
+ Columns struct_cols;
+ struct_cols.push_back(std::move(file_path_col));
+ struct_cols.push_back(std::move(row_pos_col));
+ struct_cols.push_back(std::move(spec_id_col));
+ struct_cols.push_back(std::move(partition_data_col));
+
+ auto struct_col = ColumnStruct::create(std::move(struct_cols));
+
+ DataTypes struct_types;
+ struct_types.push_back(std::make_shared<DataTypeString>());
+ struct_types.push_back(std::make_shared<DataTypeInt64>());
+ struct_types.push_back(std::make_shared<DataTypeInt32>());
+ struct_types.push_back(std::make_shared<DataTypeString>());
+
+ Strings field_names = {"file_path", "row_position", "partition_spec_id", "partition_data"};
+ auto struct_type = std::make_shared<DataTypeStruct>(struct_types, field_names);
+
+ block.insert(ColumnWithTypeAndName(std::move(struct_col), struct_type,
+ doris::BeConsts::ICEBERG_ROWID_COL));
+
+ // Extract and group
+ std::map<std::string, IcebergFileDeletion> file_deletions;
+ Status status = sink->_collect_position_deletes(block, file_deletions);
+ ASSERT_TRUE(status.ok());
+
+ // Verify grouping
+ ASSERT_EQ(2, file_deletions.size());
+
+ ASSERT_TRUE(file_deletions.count("file1.parquet") > 0);
+ const auto& file1 = file_deletions.at("file1.parquet");
+ ASSERT_EQ(1, file1.partition_spec_id);
+ ASSERT_EQ("p=1", file1.partition_data_json);
+ ASSERT_EQ(2, file1.rows_to_delete.cardinality());
+ ASSERT_TRUE(file1.rows_to_delete.contains(static_cast<uint64_t>(100)));
+ ASSERT_TRUE(file1.rows_to_delete.contains(static_cast<uint64_t>(150)));
+
+ ASSERT_TRUE(file_deletions.count("file2.parquet") > 0);
+ const auto& file2 = file_deletions.at("file2.parquet");
+ ASSERT_EQ(2, file2.partition_spec_id);
+ ASSERT_EQ("p=2", file2.partition_data_json);
+ ASSERT_EQ(1, file2.rows_to_delete.cardinality());
+ ASSERT_TRUE(file2.rows_to_delete.contains(static_cast<uint64_t>(200)));
+}
+
+TEST_F(VIcebergDeleteSinkTest, TestCollectPositionDeletesByFieldNames) {
+ VExprContextSPtrs output_exprs;
+ auto sink = std::make_shared<VIcebergDeleteSink>(_t_data_sink, output_exprs, nullptr, nullptr);
+
+ Block block;
+
+ auto row_pos_col = ColumnInt64::create();
+ row_pos_col->insert_value(100);
+
+ auto file_path_col = ColumnString::create();
+ file_path_col->insert_data("file1.parquet", 13);
+
+ auto partition_data_col = ColumnString::create();
+ partition_data_col->insert_data("[\"p=1\"]", 7);
+
+ auto spec_id_col = ColumnInt32::create();
+ spec_id_col->insert_value(3);
+
+ Columns struct_cols;
+ struct_cols.push_back(std::move(row_pos_col));
+ struct_cols.push_back(std::move(file_path_col));
+ struct_cols.push_back(std::move(partition_data_col));
+ struct_cols.push_back(std::move(spec_id_col));
+
+ DataTypes struct_types;
+ struct_types.push_back(std::make_shared<DataTypeInt64>());
+ struct_types.push_back(std::make_shared<DataTypeString>());
+ struct_types.push_back(std::make_shared<DataTypeString>());
+ struct_types.push_back(std::make_shared<DataTypeInt32>());
+
+ // Standard field names are accepted even if order changes.
+ Strings field_names = {"row_position", "file_path", "partition_data", "partition_spec_id"};
+ auto struct_type = std::make_shared<DataTypeStruct>(struct_types, field_names);
+
+ auto struct_col = ColumnStruct::create(std::move(struct_cols));
+ block.insert(ColumnWithTypeAndName(std::move(struct_col), struct_type,
+ doris::BeConsts::ICEBERG_ROWID_COL));
+
+ std::map<std::string, IcebergFileDeletion> file_deletions;
+ Status status = sink->_collect_position_deletes(block, file_deletions);
+ ASSERT_TRUE(status.ok());
+
+ ASSERT_EQ(1, file_deletions.size());
+ const auto& file = file_deletions.at("file1.parquet");
+ ASSERT_EQ(3, file.partition_spec_id);
+ ASSERT_EQ("[\"p=1\"]", file.partition_data_json);
+ ASSERT_EQ(1, file.rows_to_delete.cardinality());
+ ASSERT_TRUE(file.rows_to_delete.contains(static_cast<uint64_t>(100)));
+}
+
+TEST_F(VIcebergDeleteSinkTest, TestCollectPositionDeletesRejectNonStandardFieldNames) {
+ VExprContextSPtrs output_exprs;
+ auto sink = std::make_shared<VIcebergDeleteSink>(_t_data_sink, output_exprs, nullptr, nullptr);
+
+ Block block;
+
+ auto row_pos_col = ColumnInt64::create();
+ row_pos_col->insert_value(100);
+
+ auto file_path_col = ColumnString::create();
+ file_path_col->insert_data("file1.parquet", 13);
+
+ auto partition_data_col = ColumnString::create();
+ partition_data_col->insert_data("[\"p=1\"]", 7);
+
+ auto spec_id_col = ColumnInt32::create();
+ spec_id_col->insert_value(3);
+
+ Columns struct_cols;
+ struct_cols.push_back(std::move(row_pos_col));
+ struct_cols.push_back(std::move(file_path_col));
+ struct_cols.push_back(std::move(partition_data_col));
+ struct_cols.push_back(std::move(spec_id_col));
+
+ DataTypes struct_types;
+ struct_types.push_back(std::make_shared<DataTypeInt64>());
+ struct_types.push_back(std::make_shared<DataTypeString>());
+ struct_types.push_back(std::make_shared<DataTypeString>());
+ struct_types.push_back(std::make_shared<DataTypeInt32>());
+
+ Strings field_names = {"pos", "file_path", "partition_data_json", "spec_id"};
+ auto struct_type = std::make_shared<DataTypeStruct>(struct_types, field_names);
+
+ auto struct_col = ColumnStruct::create(std::move(struct_cols));
+ block.insert(ColumnWithTypeAndName(std::move(struct_col), struct_type,
+ doris::BeConsts::ICEBERG_ROWID_COL));
+
+ std::map<std::string, IcebergFileDeletion> file_deletions;
+ Status status = sink->_collect_position_deletes(block, file_deletions);
+ ASSERT_FALSE(status.ok());
+}
+
+TEST_F(VIcebergDeleteSinkTest, TestCollectPositionDeletesFallbackPartitionInfo) {
+ TDataSink t_data_sink;
+ t_data_sink.__set_type(TDataSinkType::ICEBERG_DELETE_SINK);
+
+ TIcebergDeleteSink delete_sink;
+ delete_sink.__set_db_name("test_db");
+ delete_sink.__set_tb_name("test_table");
+ delete_sink.__set_delete_type(TFileContent::POSITION_DELETES);
+ delete_sink.__set_file_format(TFileFormatType::FORMAT_PARQUET);
+ delete_sink.__set_compress_type(TFileCompressType::SNAPPYBLOCK);
+ delete_sink.__set_output_path("/tmp/iceberg/test");
+ delete_sink.__set_table_location("/tmp/iceberg/test_table");
+ delete_sink.__set_partition_spec_id(11);
+ delete_sink.__set_partition_data_json("[\"11\"]");
+
+ t_data_sink.__set_iceberg_delete_sink(delete_sink);
+
+ VExprContextSPtrs output_exprs;
+ auto sink = std::make_shared<VIcebergDeleteSink>(t_data_sink, output_exprs, nullptr, nullptr);
+ ObjectPool pool;
+ ASSERT_TRUE(sink->init_properties(&pool).ok());
+
+ Block block;
+
+ auto file_path_col = ColumnString::create();
+ file_path_col->insert_data("file1.parquet", 13);
+
+ auto row_pos_col = ColumnInt64::create();
+ row_pos_col->insert_value(100);
+
+ Columns struct_cols;
+ struct_cols.push_back(std::move(file_path_col));
+ struct_cols.push_back(std::move(row_pos_col));
+
+ auto struct_col = ColumnStruct::create(std::move(struct_cols));
+
+ DataTypes struct_types;
+ struct_types.push_back(std::make_shared<DataTypeString>());
+ struct_types.push_back(std::make_shared<DataTypeInt64>());
+
+ Strings field_names = {"file_path", "row_position"};
+ auto struct_type = std::make_shared<DataTypeStruct>(struct_types, field_names);
+
+ block.insert(ColumnWithTypeAndName(std::move(struct_col), struct_type,
+ doris::BeConsts::ICEBERG_ROWID_COL));
+
+ std::map<std::string, IcebergFileDeletion> file_deletions;
+ Status status = sink->_collect_position_deletes(block, file_deletions);
+ ASSERT_TRUE(status.ok());
+
+ ASSERT_EQ(1, file_deletions.size());
+ const auto& file = file_deletions.at("file1.parquet");
+ ASSERT_EQ(11, file.partition_spec_id);
+ ASSERT_EQ("[\"11\"]", file.partition_data_json);
+ ASSERT_EQ(1, file.rows_to_delete.cardinality());
+ ASSERT_TRUE(file.rows_to_delete.contains(static_cast<uint64_t>(100)));
+}
+
+TEST_F(VIcebergDeleteSinkTest, TestCollectPositionDeletesInvalidRowPosition) {
+ VExprContextSPtrs output_exprs;
+ auto sink = std::make_shared<VIcebergDeleteSink>(_t_data_sink, output_exprs, nullptr, nullptr);
+
+ Block block;
+
+ auto file_path_col = ColumnString::create();
+ file_path_col->insert_data("file1.parquet", 13);
+
+ auto row_pos_col = ColumnInt64::create();
+ row_pos_col->insert_value(-1);
+
+ Columns struct_cols;
+ struct_cols.push_back(std::move(file_path_col));
+ struct_cols.push_back(std::move(row_pos_col));
+
+ auto struct_col = ColumnStruct::create(std::move(struct_cols));
+
+ DataTypes struct_types;
+ struct_types.push_back(std::make_shared<DataTypeString>());
+ struct_types.push_back(std::make_shared<DataTypeInt64>());
+
+ Strings field_names = {"file_path", "row_position"};
+ auto struct_type = std::make_shared<DataTypeStruct>(struct_types, field_names);
+
+ block.insert(ColumnWithTypeAndName(std::move(struct_col), struct_type,
+ doris::BeConsts::ICEBERG_ROWID_COL));
+
+ std::map<std::string, IcebergFileDeletion> file_deletions;
+ Status status = sink->_collect_position_deletes(block, file_deletions);
+ ASSERT_FALSE(status.ok());
+}
+
+TEST_F(VIcebergDeleteSinkTest, TestBuildPositionDeleteBlock) {
+ VExprContextSPtrs output_exprs;
+ auto sink = std::make_shared<VIcebergDeleteSink>(_t_data_sink, output_exprs, nullptr, nullptr);
+
+ std::string file_path = "test_file.parquet";
+ std::vector<int64_t> positions = {10, 20, 30, 40};
+
+ Block output_block;
+ Status status = sink->_build_position_delete_block(file_path, positions, output_block);
+ ASSERT_TRUE(status.ok());
+
+ // Verify block structure
+ ASSERT_EQ(2, output_block.columns());
+ ASSERT_EQ(4, output_block.rows());
+
+ // Verify column names
+ ASSERT_EQ("file_path", output_block.get_by_position(0).name);
+ ASSERT_EQ("pos", output_block.get_by_position(1).name);
+
+ // Verify file_path column
+ auto file_path_column =
+ check_and_get_column<ColumnString>(output_block.get_by_position(0).column.get());
+ ASSERT_NE(nullptr, file_path_column);
+ for (size_t i = 0; i < 4; i++) {
+ ASSERT_EQ(file_path, file_path_column->get_data_at(i).to_string());
+ }
+
+ // Verify pos column
+ auto pos_column =
+ check_and_get_column<ColumnInt64>(output_block.get_by_position(1).column.get());
+ ASSERT_NE(nullptr, pos_column);
+ ASSERT_EQ(10, pos_column->get_element(0));
+ ASSERT_EQ(20, pos_column->get_element(1));
+ ASSERT_EQ(30, pos_column->get_element(2));
+ ASSERT_EQ(40, pos_column->get_element(3));
+}
+
+TEST_F(VIcebergDeleteSinkTest, TestGenerateDeleteFilePath) {
+ VExprContextSPtrs output_exprs;
+ auto sink = std::make_shared<VIcebergDeleteSink>(_t_data_sink, output_exprs, nullptr, nullptr);
+ ObjectPool pool;
+ Status status = sink->init_properties(&pool);
+ ASSERT_TRUE(status.ok());
+
+ std::string data_file_path = "data/file1.parquet";
+ std::string delete_file_path = sink->_generate_delete_file_path(data_file_path);
+
+ // Verify the path format
+ ASSERT_FALSE(delete_file_path.empty());
+ const auto& delete_sink = _t_data_sink.iceberg_delete_sink;
+ std::string expected_base =
+ delete_sink.__isset.output_path ? delete_sink.output_path : delete_sink.table_location;
+ if (!expected_base.empty() && expected_base.back() != '/') {
+ expected_base += '/';
+ }
+ ASSERT_TRUE(delete_file_path.rfind(expected_base, 0) == 0);
+ ASSERT_NE(std::string::npos, delete_file_path.find("delete_pos_"));
+}
+
+TEST_F(VIcebergDeleteSinkTest, TestUnsupportedDeleteType) {
+ // Create a TDataSink for an unsupported delete type
+ TDataSink t_eq_delete_sink;
+ t_eq_delete_sink.__set_type(TDataSinkType::ICEBERG_DELETE_SINK);
+
+ TIcebergDeleteSink delete_sink;
+ delete_sink.__set_db_name("test_db");
+ delete_sink.__set_tb_name("test_table");
+ delete_sink.__set_delete_type(TFileContent::EQUALITY_DELETES);
+ delete_sink.__set_file_format(TFileFormatType::FORMAT_PARQUET);
+ delete_sink.__set_compress_type(TFileCompressType::SNAPPYBLOCK);
+ delete_sink.__set_output_path("/tmp/iceberg/test");
+
+ std::map<std::string, std::string> hadoop_conf;
+ delete_sink.__set_hadoop_config(hadoop_conf);
+
+ t_eq_delete_sink.__set_iceberg_delete_sink(delete_sink);
+
+ VExprContextSPtrs output_exprs;
+ auto sink =
+ std::make_shared<VIcebergDeleteSink>(t_eq_delete_sink, output_exprs, nullptr, nullptr);
+
+ ObjectPool pool;
+ Status status = sink->init_properties(&pool);
+ ASSERT_FALSE(status.ok());
+}
+
+} // namespace doris
diff --git a/be/test/exec/sink/viceberg_merge_sink_test.cpp b/be/test/exec/sink/viceberg_merge_sink_test.cpp
new file mode 100644
index 00000000000..d5c9ac1b891
--- /dev/null
+++ b/be/test/exec/sink/viceberg_merge_sink_test.cpp
@@ -0,0 +1,318 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exec/sink/viceberg_merge_sink.h"
+
+#include <gtest/gtest.h>
+
+#include "common/consts.h"
+#include "common/object_pool.h"
+#include "core/block/block.h"
+#include "core/column/column_string.h"
+#include "core/column/column_struct.h"
+#include "core/column/column_vector.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_struct.h"
+#include "exprs/vexpr_context.h"
+#include "gen_cpp/DataSinks_types.h"
+#include "gen_cpp/Types_types.h"
+#include "runtime/runtime_profile.h"
+#include "testutil/mock/mock_descriptors.h"
+#include "testutil/mock/mock_runtime_state.h"
+#include "testutil/mock/mock_slot_ref.h"
+
+namespace doris {
+
+class VIcebergMergeSinkTest : public testing::Test {
+protected:
+ static std::string test_schema_json() {
+ return "{\"type\":\"struct\",\"schema-id\":0,\"fields\":["
+ "{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},"
+ "{\"id\":2,\"name\":\"name\",\"required\":false,\"type\":\"string\"}"
+ "]}";
+ }
+
+ TDataSink build_sink() {
+ TDataSink t_sink;
+ t_sink.__set_type(TDataSinkType::ICEBERG_MERGE_SINK);
+
+ TIcebergMergeSink merge_sink;
+ merge_sink.__set_db_name("test_db");
+ merge_sink.__set_tb_name("test_table");
+ merge_sink.__set_schema_json(test_schema_json());
+ merge_sink.__set_partition_spec_id(0);
+ merge_sink.__set_file_format(TFileFormatType::FORMAT_PARQUET);
+ merge_sink.__set_compression_type(TFileCompressType::SNAPPYBLOCK);
+ merge_sink.__set_output_path("file:///tmp/iceberg_merge_sink");
+ merge_sink.__set_original_output_path("file:///tmp/iceberg_merge_sink");
+ merge_sink.__set_table_location("file:///tmp/iceberg_merge_sink");
+ merge_sink.__set_file_type(TFileType::FILE_LOCAL);
+ merge_sink.__set_delete_type(TFileContent::POSITION_DELETES);
+ merge_sink.__set_partition_spec_id_for_delete(0);
+
+ t_sink.__set_iceberg_merge_sink(merge_sink);
+ return t_sink;
+ }
+
+ VExprContextSPtrs build_output_exprs(ObjectPool* pool, doris::RuntimeState* state,
+ const doris::RowDescriptor& row_desc,
+ bool include_operation = true,
+ bool include_row_id = true) {
+ VExprContextSPtrs output_exprs;
+
+ if (include_operation) {
+ auto op_expr = std::make_shared<MockSlotRef>(0, std::make_shared<DataTypeInt8>());
+ op_expr->set_expr_name("operation");
+ output_exprs.emplace_back(VExprContext::create_shared(op_expr));
+ }
+
+ if (include_row_id) {
+ auto row_id_expr = std::make_shared<MockSlotRef>(
+ 1,
+ std::make_shared<DataTypeStruct>(DataTypes {std::make_shared<DataTypeString>(),
+ std::make_shared<DataTypeInt64>()},
+ Strings {"file_path", "row_position"}));
+ row_id_expr->set_expr_name(doris::BeConsts::ICEBERG_ROWID_COL);
+ output_exprs.emplace_back(VExprContext::create_shared(row_id_expr));
+ }
+
+ auto id_expr = std::make_shared<MockSlotRef>(2, std::make_shared<DataTypeInt32>());
+ id_expr->set_expr_name("id");
+ output_exprs.emplace_back(VExprContext::create_shared(id_expr));
+
+ auto name_expr = std::make_shared<MockSlotRef>(3, std::make_shared<DataTypeString>());
+ name_expr->set_expr_name("name");
+ output_exprs.emplace_back(VExprContext::create_shared(name_expr));
+
+ for (auto& ctx : output_exprs) {
+ EXPECT_TRUE(ctx->prepare(state, row_desc).ok());
+ EXPECT_TRUE(ctx->open(state).ok());
+ }
+ return output_exprs;
+ }
+
+ Block build_block_with_ops(const std::vector<int8_t>& ops) {
+ Block block;
+
+ auto op_col = ColumnInt8::create();
+ for (auto op : ops) {
+ op_col->insert_value(op);
+ }
+ block.insert(ColumnWithTypeAndName(std::move(op_col), std::make_shared<DataTypeInt8>(),
+ "operation"));
+
+ auto file_path_col = ColumnString::create();
+ auto row_pos_col = ColumnInt64::create();
+ auto id_col = ColumnInt32::create();
+ auto name_col = ColumnString::create();
+ for (size_t i = 0; i < ops.size(); ++i) {
+ std::string file_path = "file" + std::to_string(i + 1) + ".parquet";
+ file_path_col->insert_data(file_path.data(), file_path.size());
+ row_pos_col->insert_value(static_cast<int64_t>((i + 1) * 10));
+ id_col->insert_value(static_cast<int32_t>(i + 1));
+ char name_value = static_cast<char>('a' + i);
+ name_col->insert_data(&name_value, 1);
+ }
+
+ Columns struct_cols;
+ struct_cols.push_back(std::move(file_path_col));
+ struct_cols.push_back(std::move(row_pos_col));
+
+ auto struct_col = ColumnStruct::create(std::move(struct_cols));
+ DataTypes struct_types;
+ struct_types.push_back(std::make_shared<DataTypeString>());
+ struct_types.push_back(std::make_shared<DataTypeInt64>());
+ Strings field_names = {"file_path", "row_position"};
+ auto struct_type = std::make_shared<DataTypeStruct>(struct_types, field_names);
+
+ block.insert(ColumnWithTypeAndName(std::move(struct_col), struct_type,
+ doris::BeConsts::ICEBERG_ROWID_COL));
+
+ block.insert(
+ ColumnWithTypeAndName(std::move(id_col), std::make_shared<DataTypeInt32>(), "id"));
+
+ block.insert(ColumnWithTypeAndName(std::move(name_col), std::make_shared<DataTypeString>(),
+ "name"));
+
+ return block;
+ }
+
+ Block build_block() { return build_block_with_ops({3, 2, 1}); }
+};
+
+TEST_F(VIcebergMergeSinkTest, TestUpdateProducesDeleteAndInsert) {
+ ObjectPool pool;
+ MockRuntimeState state;
+
+ DataTypes types {std::make_shared<DataTypeInt8>(),
+ std::make_shared<DataTypeStruct>(DataTypes {std::make_shared<DataTypeString>(),
+ std::make_shared<DataTypeInt64>()},
+ Strings {"file_path", "row_position"}),
+ std::make_shared<DataTypeInt32>(), std::make_shared<DataTypeString>()};
+ MockRowDescriptor row_desc(types, &pool);
+
+ auto output_exprs = build_output_exprs(&pool, &state, row_desc);
+ TDataSink t_sink = build_sink();
+
+ auto sink = std::make_shared<VIcebergMergeSink>(t_sink, output_exprs, nullptr, nullptr);
+ sink->set_skip_io(true);
+
+ ASSERT_TRUE(sink->init_properties(&pool, row_desc).ok());
+ RuntimeProfile profile("iceberg_merge_sink");
+ ASSERT_TRUE(sink->open(&state, &profile).ok());
+
+ Block block = build_block();
+ ASSERT_TRUE(sink->write(&state, block).ok());
+
+ EXPECT_EQ(2, sink->_delete_row_count);
+ EXPECT_EQ(2, sink->_insert_row_count);
+
+ ASSERT_TRUE(sink->close(Status::OK()).ok());
+}
+
+TEST_F(VIcebergMergeSinkTest, TestMissingOperationColumn) {
+ ObjectPool pool;
+ MockRuntimeState state;
+
+ DataTypes types {std::make_shared<DataTypeInt8>(),
+ std::make_shared<DataTypeStruct>(DataTypes {std::make_shared<DataTypeString>(),
+ std::make_shared<DataTypeInt64>()},
+ Strings {"file_path", "row_position"}),
+ std::make_shared<DataTypeInt32>(), std::make_shared<DataTypeString>()};
+ MockRowDescriptor row_desc(types, &pool);
+
+ auto output_exprs = build_output_exprs(&pool, &state, row_desc, false, true);
+ TDataSink t_sink = build_sink();
+
+ auto sink = std::make_shared<VIcebergMergeSink>(t_sink, output_exprs, nullptr, nullptr);
+
+ ASSERT_TRUE(sink->init_properties(&pool, row_desc).ok());
+ RuntimeProfile profile("iceberg_merge_sink");
+ Status status = sink->open(&state, &profile);
+ ASSERT_FALSE(status.ok());
+ ASSERT_NE(std::string::npos, status.to_string().find("missing operation column"));
+}
+
+TEST_F(VIcebergMergeSinkTest, TestMissingRowIdColumn) {
+ ObjectPool pool;
+ MockRuntimeState state;
+
+ DataTypes types {std::make_shared<DataTypeInt8>(),
+ std::make_shared<DataTypeStruct>(DataTypes {std::make_shared<DataTypeString>(),
+ std::make_shared<DataTypeInt64>()},
+ Strings {"file_path", "row_position"}),
+ std::make_shared<DataTypeInt32>(), std::make_shared<DataTypeString>()};
+ MockRowDescriptor row_desc(types, &pool);
+
+ auto output_exprs = build_output_exprs(&pool, &state, row_desc, true, false);
+ TDataSink t_sink = build_sink();
+
+ auto sink = std::make_shared<VIcebergMergeSink>(t_sink, output_exprs, nullptr, nullptr);
+
+ ASSERT_TRUE(sink->init_properties(&pool, row_desc).ok());
+ RuntimeProfile profile("iceberg_merge_sink");
+ Status status = sink->open(&state, &profile);
+ ASSERT_FALSE(status.ok());
+ ASSERT_NE(std::string::npos, status.to_string().find("missing row_id column"));
+}
+
+TEST_F(VIcebergMergeSinkTest, TestUnknownOperation) {
+ ObjectPool pool;
+ MockRuntimeState state;
+
+ DataTypes types {std::make_shared<DataTypeInt8>(),
+ std::make_shared<DataTypeStruct>(DataTypes {std::make_shared<DataTypeString>(),
+ std::make_shared<DataTypeInt64>()},
+ Strings {"file_path", "row_position"}),
+ std::make_shared<DataTypeInt32>(), std::make_shared<DataTypeString>()};
+ MockRowDescriptor row_desc(types, &pool);
+
+ auto output_exprs = build_output_exprs(&pool, &state, row_desc);
+ TDataSink t_sink = build_sink();
+
+ auto sink = std::make_shared<VIcebergMergeSink>(t_sink, output_exprs, nullptr, nullptr);
+ sink->set_skip_io(true);
+
+ ASSERT_TRUE(sink->init_properties(&pool, row_desc).ok());
+ RuntimeProfile profile("iceberg_merge_sink");
+ ASSERT_TRUE(sink->open(&state, &profile).ok());
+
+ Block block = build_block_with_ops({9});
+ Status status = sink->write(&state, block);
+ ASSERT_FALSE(status.ok());
+ ASSERT_NE(std::string::npos, status.to_string().find("Unknown Iceberg merge operation"));
+}
+
+TEST_F(VIcebergMergeSinkTest, TestUpdateInsertAndDeleteOperations) {
+ ObjectPool pool;
+ MockRuntimeState state;
+
+ DataTypes types {std::make_shared<DataTypeInt8>(),
+ std::make_shared<DataTypeStruct>(DataTypes {std::make_shared<DataTypeString>(),
+ std::make_shared<DataTypeInt64>()},
+ Strings {"file_path", "row_position"}),
+ std::make_shared<DataTypeInt32>(), std::make_shared<DataTypeString>()};
+ MockRowDescriptor row_desc(types, &pool);
+
+ auto output_exprs = build_output_exprs(&pool, &state, row_desc);
+ TDataSink t_sink = build_sink();
+
+ auto sink = std::make_shared<VIcebergMergeSink>(t_sink, output_exprs, nullptr, nullptr);
+ sink->set_skip_io(true);
+
+ ASSERT_TRUE(sink->init_properties(&pool, row_desc).ok());
+ RuntimeProfile profile("iceberg_merge_sink");
+ ASSERT_TRUE(sink->open(&state, &profile).ok());
+
+ Block block = build_block_with_ops({4, 5});
+ ASSERT_TRUE(sink->write(&state, block).ok());
+
+ EXPECT_EQ(1, sink->_delete_row_count);
+ EXPECT_EQ(1, sink->_insert_row_count);
+
+ ASSERT_TRUE(sink->close(Status::OK()).ok());
+}
+
+TEST_F(VIcebergMergeSinkTest, TestSchemaMismatch) {
+ ObjectPool pool;
+ MockRuntimeState state;
+
+ DataTypes types {std::make_shared<DataTypeInt8>(),
+ std::make_shared<DataTypeStruct>(DataTypes {std::make_shared<DataTypeString>(),
+ std::make_shared<DataTypeInt64>()},
+ Strings {"file_path", "row_position"}),
+ std::make_shared<DataTypeInt32>(), std::make_shared<DataTypeString>()};
+ MockRowDescriptor row_desc(types, &pool);
+
+ auto output_exprs = build_output_exprs(&pool, &state, row_desc);
+ TDataSink t_sink = build_sink();
+ t_sink.iceberg_merge_sink.__set_schema_json(
+ "{\"type\":\"struct\",\"schema-id\":0,\"fields\":["
+ "{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"}"
+ "]}");
+
+ auto sink = std::make_shared<VIcebergMergeSink>(t_sink, output_exprs, nullptr, nullptr);
+
+ ASSERT_TRUE(sink->init_properties(&pool, row_desc).ok());
+ RuntimeProfile profile("iceberg_merge_sink");
+ Status status = sink->open(&state, &profile);
+ ASSERT_FALSE(status.ok());
+ ASSERT_NE(std::string::npos, status.to_string().find("do not match schema columns"));
+}
+
+} // namespace doris
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index c2c6402d5e3..3ea2b7d6879 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -67,6 +67,7 @@ public class Column implements GsonPostProcessable {
public static final String ROW_STORE_COL = "__DORIS_ROW_STORE_COL__";
public static final String VERSION_COL = "__DORIS_VERSION_COL__";
public static final String SKIP_BITMAP_COL = "__DORIS_SKIP_BITMAP_COL__";
+ public static final String ICEBERG_ROWID_COL = "__DORIS_ICEBERG_ROWID_COL__";
// NOTE: you should name hidden column start with '__DORIS_' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
private static final String COLUMN_ARRAY_CHILDREN = "item";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
index 691f91bb08d..800f98cea77 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
@@ -60,6 +60,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.stream.Collectors;
/**
* External table represent tables that are not self-managed by Doris.
@@ -177,14 +178,29 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable {
return schemaCacheValue.map(SchemaCacheValue::getSchema).orElse(null);
}
+ protected boolean needInternalHiddenColumns() {
+ return false;
+ }
+
@Override
public List<Column> getBaseSchema() {
- return getFullSchema();
+ boolean showHidden = Util.showHiddenColumns();
+ if (!showHidden && needInternalHiddenColumns()) {
+ showHidden = true;
+ }
+ return getBaseSchema(showHidden);
}
@Override
public List<Column> getBaseSchema(boolean full) {
- return getFullSchema();
+ List<Column> schema = getFullSchema();
+ if (schema == null) {
+ return null;
+ }
+ if (full) {
+ return schema;
+ }
+ return schema.stream().filter(Column::isVisible).collect(Collectors.toList());
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
index a38305b7ea5..fd0ff62a804 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
@@ -175,7 +175,11 @@ public abstract class FileQueryScanNode extends FileScanNode {
for (SlotDescriptor slot : desc.getSlots()) {
TFileScanSlotInfo slotInfo = new TFileScanSlotInfo();
slotInfo.setSlotId(slot.getId().asInt());
- slotInfo.setIsFileSlot(!partitionKeys.contains(slot.getColumn().getName()));
+ boolean isFileSlot = !partitionKeys.contains(slot.getColumn().getName());
+ if (isIcebergRowIdColumn(slot)) {
+ isFileSlot = false;
+ }
+ slotInfo.setIsFileSlot(isFileSlot);
params.addToRequiredSlots(slotInfo);
}
setDefaultValueExprs(getTargetTable(), destSlotDescByName, null, params, false);
@@ -192,13 +196,21 @@ public abstract class FileQueryScanNode extends FileScanNode {
for (SlotDescriptor slot : desc.getSlots()) {
TFileScanSlotInfo slotInfo = new TFileScanSlotInfo();
slotInfo.setSlotId(slot.getId().asInt());
- slotInfo.setIsFileSlot(!getPathPartitionKeys().contains(slot.getColumn().getName()));
+ boolean isFileSlot = !getPathPartitionKeys().contains(slot.getColumn().getName());
+ if (isIcebergRowIdColumn(slot)) {
+ isFileSlot = false;
+ }
+ slotInfo.setIsFileSlot(isFileSlot);
params.addToRequiredSlots(slotInfo);
}
// Update required slots and column_idxs in scanRangeLocations.
setColumnPositionMapping();
}
+ private boolean isIcebergRowIdColumn(SlotDescriptor slot) {
+ return Column.ICEBERG_ROWID_COL.equalsIgnoreCase(slot.getColumn().getName());
+ }
+
public void setTableSample(TableSample tSample) {
this.tableSample = tSample;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergConflictDetectionFilterUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergConflictDetectionFilterUtils.java
new file mode 100644
index 00000000000..5423e21895d
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergConflictDetectionFilterUtils.java
@@ -0,0 +1,336 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.UserException;
+import org.apache.doris.nereids.trees.expressions.And;
+import org.apache.doris.nereids.trees.expressions.Between;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.GreaterThan;
+import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
+import org.apache.doris.nereids.trees.expressions.InPredicate;
+import org.apache.doris.nereids.trees.expressions.IsNull;
+import org.apache.doris.nereids.trees.expressions.LessThan;
+import org.apache.doris.nereids.trees.expressions.LessThanEqual;
+import org.apache.doris.nereids.trees.expressions.Not;
+import org.apache.doris.nereids.trees.expressions.Or;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types.NestedField;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Utilities for building Iceberg RowDelta conflict detection filters from Nereids plans.
+ */
+public final class IcebergConflictDetectionFilterUtils {
+
+ private IcebergConflictDetectionFilterUtils() {
+ }
+
+ public static Optional<org.apache.iceberg.expressions.Expression> buildConflictDetectionFilter(
+ Plan analyzedPlan, IcebergExternalTable targetTable) {
+ if (analyzedPlan == null || targetTable == null) {
+ return Optional.empty();
+ }
+ List<Expression> targetConjuncts = new ArrayList<>();
+ collectTargetConjuncts(analyzedPlan, targetTable, targetConjuncts);
+ if (targetConjuncts.isEmpty()) {
+ return Optional.empty();
+ }
+ Schema schema = targetTable.getIcebergTable().schema();
+ org.apache.iceberg.expressions.Expression combined = null;
+ for (Expression predicate : targetConjuncts) {
+ Optional<org.apache.iceberg.expressions.Expression> icebergExpr =
+ convertPredicateToIcebergExpression(predicate, schema);
+ if (!icebergExpr.isPresent()) {
+ continue;
+ }
+ combined = combined == null ? icebergExpr.get() : Expressions.and(combined, icebergExpr.get());
+ }
+ return combined == null ? Optional.empty() : Optional.of(combined);
+ }
+
+ private static void collectTargetConjuncts(Plan plan, IcebergExternalTable targetTable,
+ List<Expression> output) {
+ if (plan instanceof LogicalFilter) {
+ LogicalFilter<?> filter = (LogicalFilter<?>) plan;
+ for (Expression conjunct : filter.getConjuncts()) {
+ if (isTargetOnlyPredicate(conjunct, targetTable)) {
+ output.add(conjunct);
+ }
+ }
+ }
+ for (Plan child : plan.children()) {
+ collectTargetConjuncts(child, targetTable, output);
+ }
+ }
+
+ private static boolean isTargetOnlyPredicate(Expression predicate, IcebergExternalTable targetTable) {
+ if (predicate == null) {
+ return false;
+ }
+ Set<Slot> slots = predicate.getInputSlots();
+ if (slots.isEmpty()) {
+ return false;
+ }
+ for (Slot slot : slots) {
+ if (!(slot instanceof SlotReference)) {
+ return false;
+ }
+ SlotReference slotReference = (SlotReference) slot;
+ if (Column.ICEBERG_ROWID_COL.equalsIgnoreCase(slotReference.getName())) {
+ return false;
+ }
+ if (IcebergMetadataColumn.isMetadataColumn(slotReference.getName())) {
+ return false;
+ }
+ Optional<TableIf> table = slotReference.getOriginalTable();
+ if (!table.isPresent() || table.get().getId() != targetTable.getId()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static Optional<org.apache.iceberg.expressions.Expression> convertPredicateToIcebergExpression(
+ Expression predicate, Schema schema) {
+ if (predicate == null) {
+ return Optional.empty();
+ }
+ if (predicate instanceof And) {
+ And andExpr = (And) predicate;
+ Optional<org.apache.iceberg.expressions.Expression> left =
+ convertPredicateToIcebergExpression(andExpr.child(0), schema);
+ Optional<org.apache.iceberg.expressions.Expression> right =
+ convertPredicateToIcebergExpression(andExpr.child(1), schema);
+ return combineAnd(left, right);
+ }
+ if (predicate instanceof Or) {
+ Or orExpr = (Or) predicate;
+ Optional<org.apache.iceberg.expressions.Expression> left =
+ convertPredicateToIcebergExpression(orExpr.child(0), schema);
+ Optional<org.apache.iceberg.expressions.Expression> right =
+ convertPredicateToIcebergExpression(orExpr.child(1), schema);
+ if (!left.isPresent() || !right.isPresent()) {
+ return Optional.empty();
+ }
+ if (!isSameColumnPredicate(orExpr.child(0), orExpr.child(1), schema)) {
+ return Optional.empty();
+ }
+ return Optional.of(Expressions.or(left.get(), right.get()));
+ }
+ if (predicate instanceof Not) {
+ Not notExpr = (Not) predicate;
+ if (!(notExpr.child() instanceof IsNull)) {
+ return Optional.empty();
+ }
+ return convertIsNullPredicate((IsNull) notExpr.child(), schema, true);
+ }
+ if (predicate instanceof IsNull) {
+ return convertIsNullPredicate((IsNull) predicate, schema, false);
+ }
+ if (predicate instanceof InPredicate) {
+ return convertInPredicate((InPredicate) predicate, schema);
+ }
+ if (predicate instanceof Between) {
+ return convertComparablePredicate((Between) predicate, schema);
+ }
+ if (predicate instanceof EqualTo
+ || predicate instanceof GreaterThan
+ || predicate instanceof GreaterThanEqual
+ || predicate instanceof LessThan
+ || predicate instanceof LessThanEqual) {
+ return convertComparablePredicate(predicate, schema);
+ }
+ return Optional.empty();
+ }
+
+ private static Optional<org.apache.iceberg.expressions.Expression> convertIsNullPredicate(
+ IsNull predicate, Schema schema, boolean negated) {
+ Optional<NestedField> nestedField = resolveSingleField(predicate, schema);
+ if (!nestedField.isPresent()) {
+ return Optional.empty();
+ }
+ if (isStructuralType(nestedField.get().type())) {
+ return Optional.empty();
+ }
+ org.apache.iceberg.expressions.Expression isNullExpr = Expressions.isNull(nestedField.get().name());
+ return Optional.of(negated ? Expressions.not(isNullExpr) : isNullExpr);
+ }
+
+ private static Optional<org.apache.iceberg.expressions.Expression> convertInPredicate(
+ InPredicate predicate, Schema schema) {
+ if (!(predicate.child(0) instanceof Slot)) {
+ return Optional.empty();
+ }
+ Optional<NestedField> nestedField = resolveSingleField(predicate, schema);
+ if (!nestedField.isPresent()) {
+ return Optional.empty();
+ }
+ Type type = nestedField.get().type();
+ if (isStructuralType(type)) {
+ return Optional.empty();
+ }
+
+ boolean hasNull = false;
+ List<Object> values = new ArrayList<>();
+ for (int i = 1; i < predicate.children().size(); i++) {
+ Expression child = predicate.child(i);
+ if (!(child instanceof Literal)) {
+ return Optional.empty();
+ }
+ Literal literal = (Literal) child;
+ if (literal instanceof NullLiteral) {
+ hasNull = true;
+ continue;
+ }
+ try {
+ Object value = IcebergNereidsUtils.extractNereidsLiteralValue(literal, type);
+ if (value == null) {
+ return Optional.empty();
+ }
+ values.add(value);
+ } catch (UserException ignored) {
+ return Optional.empty();
+ }
+ }
+
+ if (isUuidType(type) && !values.isEmpty()) {
+ return Optional.empty();
+ }
+
+ org.apache.iceberg.expressions.Expression valuesExpr = values.isEmpty()
+ ? null
+ : Expressions.in(nestedField.get().name(), values);
+ org.apache.iceberg.expressions.Expression nullExpr = hasNull
+ ? Expressions.isNull(nestedField.get().name())
+ : null;
+ return combineOr(nullExpr, valuesExpr);
+ }
+
+ private static Optional<org.apache.iceberg.expressions.Expression> convertComparablePredicate(
+ Expression predicate, Schema schema) {
+ Optional<NestedField> nestedField = resolveSingleField(predicate, schema);
+ if (!nestedField.isPresent()) {
+ return Optional.empty();
+ }
+ Type type = nestedField.get().type();
+ if (isStructuralType(type)) {
+ return Optional.empty();
+ }
+ if (isUuidType(type)) {
+ return isNullComparison(predicate)
+ ? Optional.of(Expressions.isNull(nestedField.get().name()))
+ : Optional.empty();
+ }
+ try {
+ return Optional.of(IcebergNereidsUtils.convertNereidsToIcebergExpression(predicate, schema));
+ } catch (UserException ignored) {
+ return Optional.empty();
+ }
+ }
+
+ private static boolean isNullComparison(Expression predicate) {
+ if (!(predicate instanceof EqualTo)) {
+ return false;
+ }
+ Expression left = predicate.child(0);
+ Expression right = predicate.child(1);
+ return (left instanceof Slot && right instanceof NullLiteral)
+ || (right instanceof Slot && left instanceof NullLiteral);
+ }
+
+ private static Optional<NestedField> resolveSingleField(Expression predicate, Schema schema) {
+ Set<Slot> slots = predicate.getInputSlots();
+ if (slots.size() != 1) {
+ return Optional.empty();
+ }
+ Slot slot = slots.iterator().next();
+ if (!(slot instanceof SlotReference)) {
+ return Optional.empty();
+ }
+ String columnName = ((SlotReference) slot).getName();
+ if (Column.ICEBERG_ROWID_COL.equalsIgnoreCase(columnName)) {
+ return Optional.empty();
+ }
+ if (IcebergMetadataColumn.isMetadataColumn(columnName)) {
+ return Optional.empty();
+ }
+ NestedField nestedField = schema.caseInsensitiveFindField(columnName);
+ return Optional.ofNullable(nestedField);
+ }
+
+ private static boolean isSameColumnPredicate(Expression left, Expression right, Schema schema) {
+ Optional<NestedField> leftField = resolveSingleField(left, schema);
+ if (!leftField.isPresent()) {
+ return false;
+ }
+ Optional<NestedField> rightField = resolveSingleField(right, schema);
+ if (!rightField.isPresent()) {
+ return false;
+ }
+ return leftField.get().fieldId() == rightField.get().fieldId();
+ }
+
+ private static Optional<org.apache.iceberg.expressions.Expression> combineAnd(
+ Optional<org.apache.iceberg.expressions.Expression> left,
+ Optional<org.apache.iceberg.expressions.Expression> right) {
+ if (!left.isPresent()) {
+ return right;
+ }
+ if (!right.isPresent()) {
+ return left;
+ }
+ return Optional.of(Expressions.and(left.get(), right.get()));
+ }
+
+ private static Optional<org.apache.iceberg.expressions.Expression> combineOr(
+ org.apache.iceberg.expressions.Expression left,
+ org.apache.iceberg.expressions.Expression right) {
+ if (left == null) {
+ return Optional.ofNullable(right);
+ }
+ if (right == null) {
+ return Optional.of(left);
+ }
+ return Optional.of(Expressions.or(left, right));
+ }
+
+ private static boolean isStructuralType(Type type) {
+ return type.isStructType() || type.isListType() || type.isMapType();
+ }
+
+ private static boolean isUuidType(Type type) {
+ return type.typeId() == Type.TypeID.UUID;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
index aceb2a2408a..eb00d82adc7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
@@ -25,6 +25,7 @@ import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
+import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.SchemaCacheKey;
import org.apache.doris.datasource.SchemaCacheValue;
@@ -40,6 +41,7 @@ import org.apache.doris.mtmv.MTMVRelatedTableIf;
import org.apache.doris.mtmv.MTMVSnapshotIdSnapshot;
import org.apache.doris.mtmv.MTMVSnapshotIf;
import org.apache.doris.nereids.trees.plans.commands.info.SortFieldInfo;
+import org.apache.doris.qe.ConnectContext;
import org.apache.doris.statistics.AnalysisInfo;
import org.apache.doris.statistics.BaseAnalysisTask;
import org.apache.doris.statistics.ExternalAnalysisTask;
@@ -59,6 +61,7 @@ import org.apache.iceberg.view.SQLViewRepresentation;
import org.apache.iceberg.view.View;
import org.apache.iceberg.view.ViewVersion;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -267,9 +270,28 @@ public class IcebergExternalTable extends ExternalTable implements MTMVRelatedTa
}
}
+ @Override
+ protected boolean needInternalHiddenColumns() {
+ ConnectContext ctx = ConnectContext.get();
+ return ctx != null && ctx.needIcebergRowIdForTable(this.getId());
+ }
+
@Override
public List<Column> getFullSchema() {
- return IcebergUtils.getIcebergSchema(this);
+ List<Column> schema = IcebergUtils.getIcebergSchema(this);
+
+ // 添加隐藏列: __DORIS_ICEBERG_ROWID_COL__
+ // 只有在 Util.showHiddenColumns() 或内部需要时返回
+ if (Util.showHiddenColumns() || needInternalHiddenColumns()) {
+ schema = new ArrayList<>(schema);
+ schema.add(createIcebergRowIdColumn());
+ }
+
+ return schema;
+ }
+
+ private Column createIcebergRowIdColumn() {
+ return IcebergRowId.createHiddenColumn();
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMergeOperation.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMergeOperation.java
new file mode 100644
index 00000000000..94073c70007
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMergeOperation.java
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg;
+
+/**
+ * Operation codes used for merge-style DML routing.
+ */
+public final class IcebergMergeOperation {
+ public static final String OPERATION_COLUMN = "operation";
+
+ // Merge sink routing:
+ // 1 (INSERT): only insert writer
+ // 2 (DELETE): only delete writer
+ // 3 (UPDATE): update rows (merge sink writes delete + insert)
+ // 4 (UPDATE_INSERT): pre-split update insert rows
+ // 5 (UPDATE_DELETE): pre-split update delete rows
+ public static final byte INSERT_OPERATION_NUMBER = 1;
+ public static final byte DELETE_OPERATION_NUMBER = 2;
+ public static final byte UPDATE_OPERATION_NUMBER = 3;
+ public static final byte UPDATE_INSERT_OPERATION_NUMBER = 4;
+ public static final byte UPDATE_DELETE_OPERATION_NUMBER = 5;
+
+ private IcebergMergeOperation() {}
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataColumn.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataColumn.java
new file mode 100644
index 00000000000..bad7e37b25b
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataColumn.java
@@ -0,0 +1,122 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg;
+
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.Type;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * Iceberg metadata columns that can be projected in queries.
+ *
+ * These columns are not stored in data files but are generated during scanning.
+ * Doris uses __DORIS_ICEBERG_ROWID_COL__ for row-id and does not expose $row_id.
+ */
+public enum IcebergMetadataColumn {
+ /**
+ * File path of the data file containing the row.
+ * Type: STRING
+ * Used in Position Delete files to identify which file a row belongs to.
+ */
+ FILE_PATH("$file_path", ScalarType.createStringType()),
+
+ /**
+ * Position (row number) of the row within the data file.
+ * Type: BIGINT
+ * Used in Position Delete files to identify the exact row to delete.
+ */
+ ROW_POSITION("$row_position", ScalarType.createType(org.apache.doris.catalog.PrimitiveType.BIGINT)),
+
+ /**
+ * Partition specification ID.
+ * Type: INT
+ * Identifies which partition specification is used for this row.
+ */
+ PARTITION_SPEC_ID("$partition_spec_id", ScalarType.createType(org.apache.doris.catalog.PrimitiveType.INT)),
+
+ /**
+ * Partition data as JSON string.
+ * Type: STRING
+ * Contains the partition values for this row in JSON format.
+ */
+ PARTITION_DATA("$partition_data", ScalarType.createStringType());
+
+ private final String columnName;
+ private final Type columnType;
+
+ IcebergMetadataColumn(String columnName, Type columnType) {
+ this.columnName = columnName;
+ this.columnType = columnType;
+ }
+
+ public String getColumnName() {
+ return columnName;
+ }
+
+ public Type getColumnType() {
+ return columnType;
+ }
+
+ /**
+ * Check if a column name is a metadata column.
+ */
+ public static boolean isMetadataColumn(String columnName) {
+ return fromColumnName(columnName) != null;
+ }
+
+ /**
+ * Get metadata column by name.
+ */
+ public static IcebergMetadataColumn fromColumnName(String columnName) {
+ for (IcebergMetadataColumn column : values()) {
+ if (column.columnName.equals(columnName)) {
+ return column;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Get all metadata column names.
+ */
+ public static List<String> getAllColumnNames() {
+ ImmutableList.Builder<String> builder = ImmutableList.builder();
+ for (IcebergMetadataColumn column : values()) {
+ builder.add(column.columnName);
+ }
+ return builder.build();
+ }
+
+ /**
+ * Check if a column name is $file_path.
+ */
+ public static boolean isFilePathColumn(String columnName) {
+ return FILE_PATH.columnName.equals(columnName);
+ }
+
+ /**
+ * Check if a column name is $row_position.
+ */
+ public static boolean isRowPositionColumn(String columnName) {
+ return ROW_POSITION.columnName.equals(columnName);
+ }
+
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergNereidsUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergNereidsUtils.java
index 693f7390e90..2308ee0b86d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergNereidsUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergNereidsUtils.java
@@ -17,7 +17,9 @@
package org.apache.doris.datasource.iceberg;
+import org.apache.doris.catalog.Column;
import org.apache.doris.common.UserException;
+import org.apache.doris.nereids.analyzer.Unbound;
import org.apache.doris.nereids.analyzer.UnboundSlot;
import org.apache.doris.nereids.trees.expressions.And;
import org.apache.doris.nereids.trees.expressions.Between;
@@ -29,10 +31,12 @@ import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.LessThan;
import org.apache.doris.nereids.trees.expressions.LessThanEqual;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
@@ -40,6 +44,11 @@ import org.apache.doris.nereids.trees.expressions.literal.DecimalLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
import org.apache.doris.nereids.util.DateUtils;
import org.apache.iceberg.Schema;
@@ -53,13 +62,134 @@ import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.List;
+import java.util.Optional;
import java.util.function.BiFunction;
+import javax.annotation.Nullable;
/**
- * Utility class for converting Nereids expressions to Iceberg expressions.
+ * Utility class for Iceberg + Nereids integration.
+ * Provides shared helpers for row-id injection and expression conversion.
*/
public class IcebergNereidsUtils {
+ // ==================== Row-ID Injection Utilities ====================
+
+ /**
+ * Inject $row_id column into the plan for any Iceberg table scan.
+ * Used by DELETE and UPDATE commands (single-table, no ambiguity).
+ */
+ public static LogicalPlan injectRowIdColumn(LogicalPlan plan) {
+ if (hasUnboundPlan(plan)) {
+ return plan;
+ }
+ return (LogicalPlan) plan.accept(new IcebergRowIdInjector(null), null);
+ }
+
+ /**
+ * Inject $row_id column only for the specified target table.
+ * Used by MERGE INTO where source may also be an Iceberg table.
+ */
+ public static LogicalPlan injectRowIdColumn(LogicalPlan plan, IcebergExternalTable targetTable) {
+ if (hasUnboundPlan(plan)) {
+ return plan;
+ }
+ return (LogicalPlan) plan.accept(new IcebergRowIdInjector(targetTable), null);
+ }
+
+ /** Check if any slot in the list is the row-id column. */
+ public static boolean hasRowIdSlot(List<Slot> slots) {
+ return findRowIdSlot(slots).isPresent();
+ }
+
+ /** Find the row-id slot in the list, if present. */
+ public static Optional<Slot> findRowIdSlot(List<Slot> slots) {
+ for (Slot slot : slots) {
+ if (Column.ICEBERG_ROWID_COL.equalsIgnoreCase(slot.getName())) {
+ return Optional.of(slot);
+ }
+ }
+ return Optional.empty();
+ }
+
+ /** Check if any project expression is the row-id column. */
+ public static boolean hasRowIdProject(List<NamedExpression> projects) {
+ for (NamedExpression project : projects) {
+ if (project instanceof Slot
+ && Column.ICEBERG_ROWID_COL.equalsIgnoreCase(((Slot) project).getName())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** Resolve the row-id Column definition from the table's full schema. */
+ public static Column getRowIdColumn(IcebergExternalTable table) {
+ List<Column> fullSchema = table.getFullSchema();
+ if (fullSchema != null) {
+ for (Column column : fullSchema) {
+ if (Column.ICEBERG_ROWID_COL.equalsIgnoreCase(column.getName())) {
+ return column;
+ }
+ }
+ }
+ return IcebergRowId.createHiddenColumn();
+ }
+
+ /** Check if a plan tree contains any unbound nodes or expressions. */
+ public static boolean hasUnboundPlan(Plan plan) {
+ return plan.anyMatch(node -> node instanceof Unbound || ((Plan) node).hasUnboundExpression());
+ }
+
+ /**
+ * Plan rewriter that injects the $row_id hidden column into Iceberg scans and projects.
+ *
+ * <p>When {@code targetTable} is null, injects on ALL Iceberg scans (DELETE/UPDATE).
+ * When non-null, only injects on the scan whose table ID matches (MERGE INTO).
+ */
+ private static class IcebergRowIdInjector extends DefaultPlanRewriter<Void> {
+ @Nullable
+ private final IcebergExternalTable targetTable;
+
+ IcebergRowIdInjector(@Nullable IcebergExternalTable targetTable) {
+ this.targetTable = targetTable;
+ }
+
+ @Override
+ public Plan visitLogicalFileScan(LogicalFileScan scan, Void context) {
+ if (!(scan.getTable() instanceof IcebergExternalTable)) {
+ return scan;
+ }
+ if (targetTable != null
+ && ((IcebergExternalTable) scan.getTable()).getId() != targetTable.getId()) {
+ return scan;
+ }
+ if (hasRowIdSlot(scan.getOutput())) {
+ return scan;
+ }
+ IcebergExternalTable table = (IcebergExternalTable) scan.getTable();
+ Column rowIdColumn = getRowIdColumn(table);
+ SlotReference rowIdSlot = SlotReference.fromColumn(
+ StatementScopeIdGenerator.newExprId(), table, rowIdColumn, scan.getQualifier());
+ List<Slot> outputs = new ArrayList<>(scan.getOutput());
+ outputs.add(rowIdSlot);
+ return scan.withCachedOutput(outputs);
+ }
+
+ @Override
+ public Plan visitLogicalProject(LogicalProject<? extends Plan> project, Void context) {
+ project = (LogicalProject<? extends Plan>) visitChildren(this, project, context);
+ Optional<Slot> rowIdSlot = findRowIdSlot(project.child().getOutput());
+ if (!rowIdSlot.isPresent() || hasRowIdProject(project.getProjects())) {
+ return project;
+ }
+ List<NamedExpression> newProjects = new ArrayList<>(project.getProjects());
+ newProjects.add((NamedExpression) rowIdSlot.get());
+ return project.withProjects(newProjects);
+ }
+ }
+
+ // ==================== Expression Conversion Utilities ====================
+
/**
* Convert Nereids Expression to Iceberg Expression
*/
@@ -330,7 +460,7 @@ public class IcebergNereidsUtils {
/**
* Extract literal value from Nereids Literal expression
*/
- private static Object extractNereidsLiteralValue(
+ static Object extractNereidsLiteralValue(
Literal literal,
Type icebergType) throws UserException {
try {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRowId.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRowId.java
new file mode 100644
index 00000000000..40a51b05a4f
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRowId.java
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.StructField;
+import org.apache.doris.catalog.StructType;
+import org.apache.doris.catalog.Type;
+
+import java.util.ArrayList;
+
+/**
+ * Iceberg hidden row-id column definition for __DORIS_ICEBERG_ROWID_COL__.
+ */
+public final class IcebergRowId {
+ private static final Type ROW_ID_TYPE = createRowIdType();
+ private static final Column ROW_ID_COLUMN = createRowIdColumn();
+
+ private IcebergRowId() {}
+
+ public static Type getRowIdType() {
+ return ROW_ID_TYPE;
+ }
+
+ // Shared instance; do not mutate.
+ public static Column createHiddenColumn() {
+ return ROW_ID_COLUMN;
+ }
+
+ private static Column createRowIdColumn() {
+ Column column = new Column(
+ Column.ICEBERG_ROWID_COL,
+ ROW_ID_TYPE,
+ false,
+ null,
+ false,
+ null,
+ "Iceberg row position metadata");
+ column.setIsVisible(false);
+ return column;
+ }
+
+ private static Type createRowIdType() {
+ ArrayList<StructField> fields = new ArrayList<>();
+ fields.add(new StructField("file_path", ScalarType.createStringType()));
+ fields.add(new StructField("row_position", ScalarType.createType(PrimitiveType.BIGINT)));
+ fields.add(new StructField("partition_spec_id", ScalarType.createType(PrimitiveType.INT)));
+ fields.add(new StructField("partition_data", ScalarType.createStringType()));
+ return new StructType(fields);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java
index e6a59fd78bc..dbc7e523c40 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java
@@ -26,6 +26,7 @@ import org.apache.doris.datasource.NameMapping;
import org.apache.doris.datasource.iceberg.helper.IcebergWriterHelper;
import org.apache.doris.nereids.trees.plans.commands.insert.IcebergInsertCommandContext;
import org.apache.doris.nereids.trees.plans.commands.insert.InsertCommandContext;
+import org.apache.doris.thrift.TFileContent;
import org.apache.doris.thrift.TIcebergCommitData;
import org.apache.doris.thrift.TUpdateMode;
import org.apache.doris.transaction.Transaction;
@@ -34,12 +35,15 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.iceberg.AppendFiles;
import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.OverwriteFiles;
import org.apache.iceberg.PartitionField;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.ReplacePartitions;
import org.apache.iceberg.RewriteFiles;
+import org.apache.iceberg.RowDelta;
import org.apache.iceberg.Schema;
import org.apache.iceberg.SnapshotRef;
import org.apache.iceberg.Table;
@@ -55,6 +59,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -62,15 +67,19 @@ import java.util.Optional;
public class IcebergTransaction implements Transaction {
private static final Logger LOG = LogManager.getLogger(IcebergTransaction.class);
+ private static final String DELETE_ISOLATION_LEVEL = "delete_isolation_level";
+ private static final String DELETE_ISOLATION_LEVEL_DEFAULT = "serializable";
private final IcebergMetadataOps ops;
private Table table;
private org.apache.iceberg.Transaction transaction;
private final List<TIcebergCommitData> commitDataList = Lists.newArrayList();
+ private Optional<Expression> conflictDetectionFilter = Optional.empty();
private IcebergInsertCommandContext insertCtx;
private String branchName;
+ private Long baseSnapshotId;
// Rewrite operation support
long startingSnapshotId = -1L; // Track the starting snapshot ID for rewrite operations
@@ -88,6 +97,18 @@ public class IcebergTransaction implements Transaction {
}
}
+ public void setConflictDetectionFilter(Expression filter) {
+ conflictDetectionFilter = Optional.ofNullable(filter);
+ }
+
+ public void clearConflictDetectionFilter() {
+ conflictDetectionFilter = Optional.empty();
+ }
+
+ public List<TIcebergCommitData> getCommitDataList() {
+ return commitDataList;
+ }
+
public void updateRewriteFiles(List<DataFile> filesToDelete) {
synchronized (this) {
this.filesToDelete.addAll(filesToDelete);
@@ -100,6 +121,7 @@ public class IcebergTransaction implements Transaction {
ops.getExecutionAuthenticator().execute(() -> {
// create and start the iceberg transaction
this.table = IcebergUtils.getIcebergTable(dorisTable);
+ this.baseSnapshotId = null;
// check branch
if (insertCtx != null && insertCtx.getBranchName().isPresent()) {
this.branchName = insertCtx.getBranchName().get();
@@ -133,9 +155,11 @@ public class IcebergTransaction implements Transaction {
ops.getExecutionAuthenticator().execute(() -> {
// create and start the iceberg transaction
this.table = IcebergUtils.getIcebergTable(dorisTable);
+ this.baseSnapshotId = null;
// Capture the starting snapshot ID for validation during rewrite commit
- this.startingSnapshotId = table.currentSnapshot().snapshotId();
+ Long snapshotId = getSnapshotIdIfPresent(table);
+ this.startingSnapshotId = snapshotId != null ? snapshotId : -1L;
// For rewrite operations, we work directly on the main table
// No branch information needed
@@ -226,6 +250,219 @@ public class IcebergTransaction implements Transaction {
}
}
+ /**
+ * Begin delete operation for Iceberg table
+ */
+ public void beginDelete(ExternalTable dorisTable) throws UserException {
+ try {
+ ops.getExecutionAuthenticator().execute(() -> {
+ // create and start the iceberg transaction
+ this.table = IcebergUtils.getIcebergTable(dorisTable);
+ this.baseSnapshotId = getSnapshotIdIfPresent(table);
+ if (table instanceof org.apache.iceberg.HasTableOperations) {
+ int formatVersion = ((org.apache.iceberg.HasTableOperations) table).operations()
+ .current().formatVersion();
+ if (formatVersion < 2) {
+ throw new IllegalArgumentException("Iceberg table " + dorisTable.getName()
+ + " must have format version 2 or higher for position deletes");
+ }
+ }
+ this.transaction = table.newTransaction();
+ LOG.info("Started delete transaction for table: {}", dorisTable.getName());
+ });
+ } catch (Exception e) {
+ throw new UserException("Failed to begin delete for iceberg table " + dorisTable.getName()
+ + " because: " + e.getMessage(), e);
+ }
+ }
+
+ /**
+ * Begin merge operation for Iceberg UPDATE (single scan RowDelta).
+ */
+ public void beginMerge(ExternalTable dorisTable) throws UserException {
+ try {
+ ops.getExecutionAuthenticator().execute(() -> {
+ this.branchName = null;
+ this.table = IcebergUtils.getIcebergTable(dorisTable);
+ this.baseSnapshotId = getSnapshotIdIfPresent(table);
+ if (table instanceof org.apache.iceberg.HasTableOperations) {
+ int formatVersion = ((org.apache.iceberg.HasTableOperations) table).operations()
+ .current().formatVersion();
+ if (formatVersion < 2) {
+ throw new IllegalArgumentException("Iceberg table " + dorisTable.getName()
+ + " must have format version 2 or higher for position deletes");
+ }
+ }
+ this.transaction = table.newTransaction();
+ LOG.info("Started merge transaction for table: {}", dorisTable.getName());
+ return null;
+ });
+ } catch (Exception e) {
+ throw new UserException("Failed to begin merge for iceberg table " + dorisTable.getName()
+ + " because: " + e.getMessage(), e);
+ }
+ }
+
+ /**
+ * Finish delete operation by committing DeleteFiles using RowDelta API
+ */
+ public void finishDelete(NameMapping nameMapping) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("iceberg table {} delete operation finished!", nameMapping.getFullLocalName());
+ }
+ try {
+ ops.getExecutionAuthenticator().execute(() -> {
+ updateManifestAfterDelete();
+ });
+ } catch (Exception e) {
+ LOG.warn("Failed to finish delete for iceberg table {}.", nameMapping.getFullLocalName(), e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Finish merge operation by committing data and delete files using RowDelta.
+ */
+ public void finishMerge(NameMapping nameMapping) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("iceberg table {} merge operation finished!", nameMapping.getFullLocalName());
+ }
+ try {
+ ops.getExecutionAuthenticator().execute(() -> {
+ updateManifestAfterMerge();
+ });
+ } catch (Exception e) {
+ LOG.warn("Failed to finish merge for iceberg table {}.", nameMapping.getFullLocalName(), e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Update manifest after delete operation using RowDelta API
+ */
+ private void updateManifestAfterDelete() {
+ FileFormat fileFormat = IcebergUtils.getFileFormat(transaction.table());
+
+ if (commitDataList.isEmpty()) {
+ LOG.info("No delete files to commit");
+ return;
+ }
+ List<DeleteFile> deleteFiles = convertCommitDataToDeleteFiles(fileFormat, commitDataList);
+
+ if (deleteFiles.isEmpty()) {
+ LOG.info("No delete files generated from commit data");
+ return;
+ }
+
+ // Create RowDelta operation
+ RowDelta rowDelta = transaction.newRowDelta();
+ applyRowDeltaValidations(rowDelta, transaction.table(), commitDataList,
+ collectReferencedDataFiles(commitDataList));
+ rowDelta.scanManifestsWith(ops.getThreadPoolWithPreAuth());
+
+ // Add all delete files
+ for (DeleteFile deleteFile : deleteFiles) {
+ rowDelta.addDeletes(deleteFile);
+ }
+
+ // Commit the delete operation
+ rowDelta.commit();
+
+ LOG.info("Committed {} delete files", deleteFiles.size());
+ }
+
+ private List<DeleteFile> convertCommitDataToDeleteFiles(FileFormat fileFormat,
+ List<TIcebergCommitData> commitDataList) {
+ if (commitDataList.isEmpty()) {
+ return Collections.emptyList();
+ }
+
+ PartitionSpec currentSpec = transaction.table().spec();
+ Map<Integer, PartitionSpec> specsById = transaction.table().specs();
+ Map<Integer, List<TIcebergCommitData>> commitDataBySpecId = new HashMap<>();
+ List<TIcebergCommitData> missingSpecId = new ArrayList<>();
+
+ for (TIcebergCommitData commitData : commitDataList) {
+ if (commitData.isSetPartitionSpecId()) {
+ commitDataBySpecId.computeIfAbsent(commitData.getPartitionSpecId(), k -> new ArrayList<>())
+ .add(commitData);
+ } else {
+ missingSpecId.add(commitData);
+ }
+ }
+
+ if (!missingSpecId.isEmpty()) {
+ Preconditions.checkState(!currentSpec.isPartitioned(),
+ "Missing partition spec id for delete files in partitioned table %s",
+ transaction.table().name());
+ commitDataBySpecId.computeIfAbsent(currentSpec.specId(), k -> new ArrayList<>())
+ .addAll(missingSpecId);
+ }
+
+ List<DeleteFile> deleteFiles = new ArrayList<>();
+ for (Map.Entry<Integer, List<TIcebergCommitData>> entry : commitDataBySpecId.entrySet()) {
+ int specId = entry.getKey();
+ PartitionSpec spec = specsById.get(specId);
+ Preconditions.checkState(spec != null,
+ "Unknown partition spec id %s for delete files in table %s",
+ specId, transaction.table().name());
+ deleteFiles.addAll(IcebergWriterHelper.convertToDeleteFiles(fileFormat, spec, entry.getValue()));
+ }
+
+ return deleteFiles;
+ }
+
+ private void updateManifestAfterMerge() {
+ if (commitDataList.isEmpty()) {
+ LOG.info("No commit data for merge operation");
+ return;
+ }
+
+ FileFormat fileFormat = IcebergUtils.getFileFormat(transaction.table());
+
+ List<TIcebergCommitData> dataCommitData = new ArrayList<>();
+ List<TIcebergCommitData> deleteCommitData = new ArrayList<>();
+
+ for (TIcebergCommitData commitData : commitDataList) {
+ if (commitData.isSetFileContent()
+ && commitData.getFileContent() == TFileContent.POSITION_DELETES) {
+ deleteCommitData.add(commitData);
+ } else {
+ dataCommitData.add(commitData);
+ }
+ }
+
+ List<DataFile> dataFiles = new ArrayList<>();
+ if (!dataCommitData.isEmpty()) {
+ WriteResult writeResult = IcebergWriterHelper.convertToWriterResult(
+ transaction.table(), dataCommitData);
+ dataFiles.addAll(Arrays.asList(writeResult.dataFiles()));
+ }
+
+ List<DeleteFile> deleteFiles = convertCommitDataToDeleteFiles(fileFormat, deleteCommitData);
+
+ if (dataFiles.isEmpty() && deleteFiles.isEmpty()) {
+ LOG.info("No data or delete files generated from commit data");
+ return;
+ }
+
+ RowDelta rowDelta = transaction.newRowDelta();
+ applyRowDeltaValidations(rowDelta, transaction.table(), commitDataList,
+ collectReferencedDataFiles(deleteCommitData));
+ rowDelta.scanManifestsWith(ops.getThreadPoolWithPreAuth());
+
+ for (DataFile dataFile : dataFiles) {
+ rowDelta.addRows(dataFile);
+ }
+ for (DeleteFile deleteFile : deleteFiles) {
+ rowDelta.addDeletes(deleteFile);
+ }
+
+ rowDelta.commit();
+ LOG.info("Committed merge with {} data files and {} delete files",
+ dataFiles.size(), deleteFiles.size());
+ }
+
public void finishInsert(NameMapping nameMapping) {
if (LOG.isDebugEnabled()) {
LOG.info("iceberg table {} insert table finished!", nameMapping.getFullLocalName());
@@ -294,7 +531,20 @@ public class IcebergTransaction implements Transaction {
}
public long getUpdateCnt() {
- return commitDataList.stream().mapToLong(TIcebergCommitData::getRowCount).sum();
+ long dataRows = 0;
+ long deleteRows = 0;
+ for (TIcebergCommitData commitData : commitDataList) {
+ if (commitData.isSetFileContent()
+ && commitData.getFileContent() == TFileContent.POSITION_DELETES) {
+ deleteRows += commitData.getRowCount();
+ } else {
+ dataRows += commitData.getRowCount();
+ }
+ }
+ // For UPDATE/MERGE, dataRows includes both inserted and update-inserted rows,
+ // which equals the number of rows affected. Position deletes are internal
+ // implementation details and should not be double-counted.
+ return dataRows > 0 ? dataRows : deleteRows;
}
/**
@@ -347,6 +597,171 @@ public class IcebergTransaction implements Transaction {
appendFiles.commit();
}
+ private Long getSnapshotIdIfPresent(Table icebergTable) {
+ if (icebergTable == null || icebergTable.currentSnapshot() == null) {
+ return null;
+ }
+ return icebergTable.currentSnapshot().snapshotId();
+ }
+
+ private void applyBaseSnapshotValidation(RowDelta rowDelta) {
+ if (baseSnapshotId != null) {
+ rowDelta.validateFromSnapshot(baseSnapshotId);
+ }
+ }
+
+ private void applyRowDeltaValidations(RowDelta rowDelta, Table icebergTable,
+ List<TIcebergCommitData> commitDataList, List<String> referencedDataFiles) {
+ applyBaseSnapshotValidation(rowDelta);
+ applyConflictDetectionFilter(rowDelta, icebergTable, commitDataList);
+ if (isSerializableIsolationLevel(icebergTable)) {
+ rowDelta.validateNoConflictingDataFiles();
+ }
+ rowDelta.validateDeletedFiles();
+ rowDelta.validateNoConflictingDeleteFiles();
+ if (!referencedDataFiles.isEmpty()) {
+ rowDelta.validateDataFilesExist(referencedDataFiles);
+ }
+ }
+
+ private void applyConflictDetectionFilter(RowDelta rowDelta, Table icebergTable,
+ List<TIcebergCommitData> commitDataList) {
+ Optional<Expression> partitionFilter = buildConflictDetectionFilter(icebergTable, commitDataList);
+ Optional<Expression> combined =
+ combineConflictDetectionFilters(conflictDetectionFilter, partitionFilter);
+ combined.ifPresent(rowDelta::conflictDetectionFilter);
+ }
+
+ private Optional<Expression> combineConflictDetectionFilters(Optional<Expression> queryFilter,
+ Optional<Expression> partitionFilter) {
+ if (queryFilter.isPresent() && partitionFilter.isPresent()) {
+ return Optional.of(Expressions.and(queryFilter.get(), partitionFilter.get()));
+ }
+ return queryFilter.isPresent() ? queryFilter : partitionFilter;
+ }
+
+ private Optional<Expression> buildConflictDetectionFilter(Table icebergTable,
+ List<TIcebergCommitData> commitDataList) {
+ if (icebergTable == null || commitDataList == null || commitDataList.isEmpty()) {
+ return Optional.empty();
+ }
+
+ PartitionSpec spec = icebergTable.spec();
+ if (!spec.isPartitioned()) {
+ return Optional.empty();
+ }
+ if (!areAllIdentityPartitions(spec)) {
+ return Optional.empty();
+ }
+
+ Schema schema = icebergTable.schema();
+ int currentSpecId = spec.specId();
+
+ Expression combined = null;
+ for (TIcebergCommitData commitData : commitDataList) {
+ if (commitData.isSetPartitionSpecId()
+ && commitData.getPartitionSpecId() != currentSpecId) {
+ return Optional.empty();
+ }
+ if (!commitData.isSetPartitionSpecId() && spec.isPartitioned()) {
+ return Optional.empty();
+ }
+
+ List<String> partitionValues = extractPartitionValues(commitData);
+ if (partitionValues.isEmpty() || partitionValues.size() != spec.fields().size()) {
+ return Optional.empty();
+ }
+
+ Expression partitionExpr = buildIdentityPartitionExpression(spec, schema, partitionValues);
+ if (partitionExpr == null) {
+ return Optional.empty();
+ }
+ combined = combined == null ? partitionExpr : Expressions.or(combined, partitionExpr);
+ }
+ return combined == null ? Optional.empty() : Optional.of(combined);
+ }
+
+ private boolean areAllIdentityPartitions(PartitionSpec spec) {
+ for (PartitionField field : spec.fields()) {
+ if (!field.transform().isIdentity()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private Expression buildIdentityPartitionExpression(PartitionSpec spec, Schema schema,
+ List<String> partitionValues) {
+ Expression expression = null;
+ List<PartitionField> fields = spec.fields();
+ for (int i = 0; i < fields.size(); i++) {
+ PartitionField field = fields.get(i);
+ Types.NestedField sourceField = schema.findField(field.sourceId());
+ if (sourceField == null) {
+ return null;
+ }
+ String valueStr = partitionValues.get(i);
+ if ("null".equals(valueStr)) {
+ valueStr = null;
+ }
+ Object value = IcebergUtils.parsePartitionValueFromString(valueStr, sourceField.type());
+ Expression predicate = value == null
+ ? Expressions.isNull(sourceField.name())
+ : Expressions.equal(sourceField.name(), value);
+ expression = expression == null ? predicate : Expressions.and(expression, predicate);
+ }
+ return expression;
+ }
+
+ private List<String> extractPartitionValues(TIcebergCommitData commitData) {
+ if (commitData == null) {
+ return Collections.emptyList();
+ }
+ if (commitData.getPartitionValues() != null && !commitData.getPartitionValues().isEmpty()) {
+ return commitData.getPartitionValues();
+ }
+ if (commitData.getPartitionDataJson() != null && !commitData.getPartitionDataJson().isEmpty()) {
+ return IcebergUtils.parsePartitionValuesFromJson(commitData.getPartitionDataJson());
+ }
+ return Collections.emptyList();
+ }
+
+ private boolean isSerializableIsolationLevel(Table icebergTable) {
+ if (icebergTable == null) {
+ return true;
+ }
+ String level = icebergTable.properties()
+ .getOrDefault(DELETE_ISOLATION_LEVEL, DELETE_ISOLATION_LEVEL_DEFAULT);
+ return "serializable".equalsIgnoreCase(level);
+ }
+
+ private List<String> collectReferencedDataFiles(List<TIcebergCommitData> commitDataList) {
+ if (commitDataList == null || commitDataList.isEmpty()) {
+ return Collections.emptyList();
+ }
+
+ List<String> referencedDataFiles = new ArrayList<>();
+ for (TIcebergCommitData commitData : commitDataList) {
+ if (commitData.isSetFileContent()
+ && commitData.getFileContent() != TFileContent.POSITION_DELETES) {
+ continue;
+ }
+ if (commitData.isSetReferencedDataFiles()) {
+ for (String dataFile : commitData.getReferencedDataFiles()) {
+ if (dataFile != null && !dataFile.isEmpty()) {
+ referencedDataFiles.add(dataFile);
+ }
+ }
+ }
+ if (commitData.isSetReferencedDataFilePath()
+ && commitData.getReferencedDataFilePath() != null
+ && !commitData.getReferencedDataFilePath().isEmpty()) {
+ referencedDataFiles.add(commitData.getReferencedDataFilePath());
+ }
+ }
+ return referencedDataFiles;
+ }
+
private void commitReplaceTxn(List<WriteResult> pendingResults) {
if (pendingResults.isEmpty()) {
// such as : insert overwrite table `dst_tb` select * from `empty_tb`
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
index 48a0c96a3a0..dea5859d3bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
@@ -63,6 +63,7 @@ import org.apache.doris.nereids.exceptions.NotSupportedException;
import org.apache.doris.nereids.trees.expressions.literal.Result;
import org.apache.doris.nereids.types.VarBinaryType;
import org.apache.doris.nereids.util.DateUtils;
+import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.thrift.TExprOpcode;
import com.github.benmanes.caffeine.cache.Caffeine;
@@ -73,6 +74,8 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
import com.google.common.collect.Sets;
+import com.google.gson.reflect.TypeToken;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.FileFormat;
@@ -665,6 +668,16 @@ public class IcebergUtils {
return null;
}
+ TypeID partitionTypeId = field.type().typeId();
+ if (partitionTypeId == TypeID.BINARY || partitionTypeId == TypeID.FIXED) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Skip dynamic partition pruning for binary partition field: {}",
+ field.name());
+ }
+ return null;
+ }
+
Object value = partitionData.get(i);
try {
String partitionString = serializePartitionValue(field.type(), value, timeZone);
@@ -678,11 +691,53 @@ public class IcebergUtils {
return partitionInfoMap;
}
+ public static List<String> getPartitionValues(PartitionData partitionData, PartitionSpec partitionSpec,
+ String timeZone) {
+ List<NestedField> fields = partitionData.getPartitionType().asNestedType().fields();
+ Preconditions.checkArgument(fields.size() == partitionSpec.fields().size(),
+ "PartitionData fields size does not match PartitionSpec fields size");
+
+ List<String> partitionValues = new ArrayList<>(fields.size());
+ for (int i = 0; i < fields.size(); i++) {
+ NestedField field = fields.get(i);
+ Object value = partitionData.get(i);
+ try {
+ partitionValues.add(serializePartitionValue(field.type(), value, timeZone));
+ } catch (UnsupportedOperationException e) {
+ LOG.warn("Failed to serialize Iceberg partition value for field {}: {}", field.name(),
+ e.getMessage());
+ partitionValues.add(null);
+ }
+ }
+ return partitionValues;
+ }
+
+ public static String getPartitionDataJson(PartitionData partitionData, PartitionSpec partitionSpec,
+ String timeZone) {
+ List<String> partitionValues = getPartitionValues(partitionData, partitionSpec, timeZone);
+ return GsonUtils.GSON.toJson(partitionValues);
+ }
+
+ public static List<String> parsePartitionValuesFromJson(String partitionDataJson) {
+ if (StringUtils.isBlank(partitionDataJson)) {
+ return Lists.newArrayList();
+ }
+ try {
+ java.lang.reflect.Type listType = new TypeToken<List<String>>() {}.getType();
+ return GsonUtils.GSON.fromJson(partitionDataJson, listType);
+ } catch (Exception e) {
+ LOG.warn("Failed to parse partition data JSON: {}", partitionDataJson, e);
+ return Lists.newArrayList();
+ }
+ }
+
private static String serializePartitionValue(org.apache.iceberg.types.Type type, Object value, String timeZone) {
switch (type.typeId()) {
case BOOLEAN:
case INTEGER:
case LONG:
+ case FLOAT:
+ case DOUBLE:
case STRING:
case UUID:
case DECIMAL:
@@ -1139,6 +1194,12 @@ public class IcebergUtils {
refName = params.getListParams().get(0);
}
SnapshotRef snapshotRef = table.refs().get(refName);
+ LOG.info("[BranchDebug] getQuerySpecSnapshot: refName={}, snapshotId={}, "
+ + "currentSnapshotId={}, allRefs={}",
+ refName,
+ snapshotRef != null ? snapshotRef.snapshotId() : "null",
+ table.currentSnapshot() != null ? table.currentSnapshot().snapshotId() : "null",
+ table.refs());
if (params.isBranch()) {
if (snapshotRef == null || !snapshotRef.isBranch()) {
throw new UserException("Table " + table.name() + " does not have branch named " + refName);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelper.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelper.java
index 9a47874762c..41537f6ca76 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelper.java
@@ -19,13 +19,16 @@ package org.apache.doris.datasource.iceberg.helper;
import org.apache.doris.datasource.iceberg.IcebergUtils;
import org.apache.doris.datasource.statistics.CommonStatistics;
+import org.apache.doris.thrift.TFileContent;
import org.apache.doris.thrift.TIcebergColumnStats;
import org.apache.doris.thrift.TIcebergCommitData;
import com.google.common.base.VerifyException;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.DataFiles;
+import org.apache.iceberg.DeleteFile;
import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.FileMetadata;
import org.apache.iceberg.Metrics;
import org.apache.iceberg.PartitionData;
import org.apache.iceberg.PartitionSpec;
@@ -178,4 +181,74 @@ public class IcebergWriterHelper {
return new Metrics(commitData.getRowCount(), columnSizes, valueCounts,
nullValueCounts, null, lowerBounds, upperBounds);
}
+
+ /**
+ * Convert TIcebergCommitData list to DeleteFile list for delete operations.
+ *
+ * @param format File format (Parquet/ORC)
+ * @param spec Partition specification
+ * @param commitDataList List of commit data from BE
+ * @return List of DeleteFile objects ready to be committed
+ */
+ public static List<DeleteFile> convertToDeleteFiles(
+ FileFormat format,
+ PartitionSpec spec,
+ List<TIcebergCommitData> commitDataList) {
+ List<DeleteFile> deleteFiles = new ArrayList<>();
+
+ for (TIcebergCommitData commitData : commitDataList) {
+ // Only process delete files
+ if (commitData.getFileContent() == null
+ || commitData.getFileContent() == TFileContent.DATA) {
+ continue;
+ }
+
+ String deleteFilePath = commitData.getFilePath();
+ long fileSize = commitData.getFileSize();
+ long recordCount = commitData.getRowCount();
+
+ // Build delete file metadata
+ FileMetadata.Builder deleteBuilder = FileMetadata.deleteFileBuilder(spec)
+ .withPath(deleteFilePath)
+ .withFormat(format)
+ .withFileSizeInBytes(fileSize)
+ .withRecordCount(recordCount);
+
+ // Set delete file content type
+ if (commitData.getFileContent() == TFileContent.POSITION_DELETES) {
+ deleteBuilder.ofPositionDeletes();
+ } else {
+ throw new VerifyException("Iceberg delete only supports position deletes, but got "
+ + commitData.getFileContent());
+ }
+
+ // Add partition information if table is partitioned
+ if (spec.isPartitioned()) {
+ PartitionData partitionData;
+ if (commitData.getPartitionValues() != null && !commitData.getPartitionValues().isEmpty()) {
+ // Convert partition values to PartitionData
+ List<String> partitionValues = commitData.getPartitionValues().stream()
+ .map(s -> s.equals("null") ? null : s)
+ .collect(Collectors.toList());
+ partitionData = convertToPartitionData(partitionValues, spec);
+ } else if (commitData.getPartitionDataJson() != null && !commitData.getPartitionDataJson().isEmpty()) {
+ List<String> partitionValues = IcebergUtils.parsePartitionValuesFromJson(
+ commitData.getPartitionDataJson());
+ if (!partitionValues.isEmpty()) {
+ partitionData = convertToPartitionData(partitionValues, spec);
+ } else {
+ partitionData = new PartitionData(spec.partitionType());
+ }
+ } else {
+ throw new VerifyException("No partition data for partitioned table");
+ }
+ deleteBuilder.withPartition(partitionData);
+ }
+
+ DeleteFile deleteFile = deleteBuilder.build();
+ deleteFiles.add(deleteFile);
+ }
+
+ return deleteFiles;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
index 83c95ecf954..1060b355818 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
@@ -274,6 +274,12 @@ public class IcebergScanNode extends FileQueryScanNode {
TIcebergFileDesc fileDesc = new TIcebergFileDesc();
fileDesc.setFormatVersion(formatVersion);
fileDesc.setOriginalFilePath(icebergSplit.getOriginalPath());
+ if (icebergSplit.getPartitionSpecId() != null) {
+ fileDesc.setPartitionSpecId(icebergSplit.getPartitionSpecId());
+ }
+ if (icebergSplit.getPartitionDataJson() != null) {
+ fileDesc.setPartitionDataJson(icebergSplit.getPartitionDataJson());
+ }
if (formatVersion < MIN_DELETE_FILE_SUPPORT_VERSION) {
fileDesc.setContent(FileContent.DATA.id());
} else {
@@ -776,14 +782,17 @@ public class IcebergScanNode extends FileQueryScanNode {
split.setTableFormatType(TableFormatType.ICEBERG);
split.setTargetSplitSize(targetSplitSize);
if (isPartitionedTable) {
+ int specId = fileScanTask.file().specId();
+ PartitionSpec partitionSpec = icebergTable.specs().get(specId);
+ Preconditions.checkNotNull(partitionSpec, "Partition spec with specId %s not found for table %s",
+ specId, icebergTable.name());
PartitionData partitionData = (PartitionData) fileScanTask.file().partition();
+ if (partitionData != null) {
+ split.setPartitionSpecId(specId);
+ split.setPartitionDataJson(IcebergUtils.getPartitionDataJson(
+ partitionData, partitionSpec, sessionVariable.getTimeZone()));
+ }
if (sessionVariable.isEnableRuntimeFilterPartitionPrune()) {
- // Get specId and corresponding PartitionSpec to handle partition evolution
- int specId = fileScanTask.file().specId();
- PartitionSpec partitionSpec = icebergTable.specs().get(specId);
-
- Preconditions.checkNotNull(partitionSpec, "Partition spec with specId %s not found for table %s",
- specId, icebergTable.name());
Map<String, String> partitionInfoMap = partitionMapInfos.computeIfAbsent(
partitionData, k -> {
return IcebergUtils.getPartitionInfoMap(partitionData, partitionSpec,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java
index 0d08f53b3e8..345bbf58c8c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java
@@ -43,6 +43,8 @@ public class IcebergSplit extends FileSplit {
private long tableLevelRowCount = -1;
// Partition values are used to do runtime filter partition pruning.
private Map<String, String> icebergPartitionValues = null;
+ private Integer partitionSpecId = null;
+ private String partitionDataJson = null;
// File path will be changed if the file is modified, so there's no need to get modification time.
public IcebergSplit(LocationPath file, long start, long length, long fileLength, String[] hosts,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index cb9b4d4a7f2..028007eab6f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -58,6 +58,7 @@ import org.apache.doris.datasource.hive.HMSExternalTable.DLAType;
import org.apache.doris.datasource.hive.source.HiveScanNode;
import org.apache.doris.datasource.hudi.source.HudiScanNode;
import org.apache.doris.datasource.iceberg.IcebergExternalTable;
+import org.apache.doris.datasource.iceberg.IcebergMergeOperation;
import org.apache.doris.datasource.iceberg.source.IcebergScanNode;
import org.apache.doris.datasource.jdbc.JdbcExternalTable;
import org.apache.doris.datasource.jdbc.sink.JdbcTableSink;
@@ -86,6 +87,7 @@ import org.apache.doris.nereids.properties.DistributionSpecGather;
import org.apache.doris.nereids.properties.DistributionSpecHash;
import org.apache.doris.nereids.properties.DistributionSpecHiveTableSinkHashPartitioned;
import org.apache.doris.nereids.properties.DistributionSpecHiveTableSinkUnPartitioned;
+import org.apache.doris.nereids.properties.DistributionSpecMerge;
import org.apache.doris.nereids.properties.DistributionSpecOlapTableSinkHashPartitioned;
import org.apache.doris.nereids.properties.DistributionSpecReplicated;
import org.apache.doris.nereids.properties.DistributionSpecStorageAny;
@@ -143,6 +145,8 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHiveTableSink;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHudiScan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergDeleteSink;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergMergeSink;
import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergTableSink;
import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
@@ -199,6 +203,8 @@ import org.apache.doris.planner.GroupCommitBlockSink;
import org.apache.doris.planner.HashJoinNode;
import org.apache.doris.planner.HashJoinNode.DistributionMode;
import org.apache.doris.planner.HiveTableSink;
+import org.apache.doris.planner.IcebergDeleteSink;
+import org.apache.doris.planner.IcebergMergeSink;
import org.apache.doris.planner.IcebergTableSink;
import org.apache.doris.planner.IntersectNode;
import org.apache.doris.planner.JoinNodeBase;
@@ -615,6 +621,47 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
return rootFragment;
}
+ @Override
+ public PlanFragment visitPhysicalIcebergDeleteSink(PhysicalIcebergDeleteSink<? extends Plan> icebergDeleteSink,
+ PlanTranslatorContext context) {
+ PlanFragment rootFragment = icebergDeleteSink.child().accept(this, context);
+ rootFragment.setOutputPartition(DataPartition.UNPARTITIONED);
+ IcebergDeleteSink sink = new IcebergDeleteSink(
+ (IcebergExternalTable) icebergDeleteSink.getTargetTable(),
+ icebergDeleteSink.getDeleteContext());
+ rootFragment.setSink(sink);
+ return rootFragment;
+ }
+
+ @Override
+ public PlanFragment visitPhysicalIcebergMergeSink(PhysicalIcebergMergeSink<? extends Plan> icebergMergeSink,
+ PlanTranslatorContext context) {
+ PlanFragment rootFragment = icebergMergeSink.child().accept(this, context);
+ rootFragment.setOutputPartition(DataPartition.UNPARTITIONED);
+ List<Expr> outputExprs = Lists.newArrayList();
+ for (Slot slot : icebergMergeSink.getOutput()) {
+ SlotRef slotRef = Objects.requireNonNull(context.findSlotRef(slot.getExprId()),
+ "Missing slot ref for iceberg merge sink output");
+ SlotDescriptor slotDesc = slotRef.getDesc();
+ if (slotDesc != null && slotDesc.getColumn() == null) {
+ String label = slotDesc.getLabel();
+ if (label != null && !label.isEmpty()) {
+ if (IcebergMergeOperation.OPERATION_COLUMN.equalsIgnoreCase(label)
+ || Column.ICEBERG_ROWID_COL.equalsIgnoreCase(label)) {
+ slotDesc.setMaterializedColumnName(label);
+ }
+ }
+ }
+ outputExprs.add(slotRef);
+ }
+ rootFragment.setOutputExprs(outputExprs);
+ IcebergMergeSink sink = new IcebergMergeSink(
+ (IcebergExternalTable) icebergMergeSink.getTargetTable(),
+ icebergMergeSink.getDeleteContext());
+ rootFragment.setSink(sink);
+ return rootFragment;
+ }
+
@Override
public PlanFragment visitPhysicalJdbcTableSink(PhysicalJdbcTableSink<? extends Plan> jdbcTableSink,
PlanTranslatorContext context) {
@@ -3187,6 +3234,30 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
return new DataPartition(TPartitionType.HIVE_TABLE_SINK_HASH_PARTITIONED, partitionExprs);
} else if (distributionSpec instanceof DistributionSpecHiveTableSinkUnPartitioned) {
return new DataPartition(TPartitionType.HIVE_TABLE_SINK_UNPARTITIONED);
+ } else if (distributionSpec instanceof DistributionSpecMerge) {
+ DistributionSpecMerge mergeSpec = (DistributionSpecMerge) distributionSpec;
+ Expr operationExpr = context.findSlotRef(mergeSpec.getOperationExprId());
+ List<Expr> insertPartitionExprs = Lists.newArrayList();
+ for (ExprId exprId : mergeSpec.getInsertPartitionExprIds()) {
+ insertPartitionExprs.add(context.findSlotRef(exprId));
+ }
+ List<Expr> deletePartitionExprs = Lists.newArrayList();
+ for (ExprId exprId : mergeSpec.getDeletePartitionExprIds()) {
+ deletePartitionExprs.add(context.findSlotRef(exprId));
+ }
+ List<DataPartition.IcebergPartitionField> insertPartitionFields = Lists.newArrayList();
+ for (DistributionSpecMerge.IcebergPartitionField field : mergeSpec.getInsertPartitionFields()) {
+ Expr sourceExpr = context.findSlotRef(field.getSourceExprId());
+ insertPartitionFields.add(new DataPartition.IcebergPartitionField(
+ sourceExpr,
+ field.getTransform(),
+ field.getParam(),
+ field.getName(),
+ field.getSourceId()));
+ }
+ return new DataPartition(TPartitionType.MERGE_PARTITIONED, operationExpr,
+ insertPartitionExprs, deletePartitionExprs, mergeSpec.isInsertRandom(),
+ insertPartitionFields, mergeSpec.getPartitionSpecId());
} else {
throw new RuntimeException("Unknown DistributionSpec: " + distributionSpec);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecMerge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecMerge.java
new file mode 100644
index 00000000000..c6ea4e37a8b
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecMerge.java
@@ -0,0 +1,189 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.properties;
+
+import org.apache.doris.nereids.trees.expressions.ExprId;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Merge-style distribution: route insert and delete/update rows using different partition keys.
+ */
+public class DistributionSpecMerge extends DistributionSpec {
+ /**
+ * Iceberg partition field metadata for merge insert routing.
+ */
+ public static class IcebergPartitionField {
+ private final String transform;
+ private final ExprId sourceExprId;
+ private final Integer param;
+ private final String name;
+ private final Integer sourceId;
+
+ /**
+ * Create a partition field mapping for merge insert routing.
+ */
+ public IcebergPartitionField(String transform, ExprId sourceExprId, Integer param,
+ String name, Integer sourceId) {
+ this.transform = Objects.requireNonNull(transform, "transform should not be null");
+ this.sourceExprId = Objects.requireNonNull(sourceExprId, "sourceExprId should not be null");
+ this.param = param;
+ this.name = name;
+ this.sourceId = sourceId;
+ }
+
+ public String getTransform() {
+ return transform;
+ }
+
+ public ExprId getSourceExprId() {
+ return sourceExprId;
+ }
+
+ public Integer getParam() {
+ return param;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Integer getSourceId() {
+ return sourceId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ IcebergPartitionField that = (IcebergPartitionField) o;
+ return transform.equals(that.transform)
+ && sourceExprId.equals(that.sourceExprId)
+ && Objects.equals(param, that.param)
+ && Objects.equals(name, that.name)
+ && Objects.equals(sourceId, that.sourceId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(transform, sourceExprId, param, name, sourceId);
+ }
+ }
+
+ private final ExprId operationExprId;
+ private final ImmutableList<ExprId> insertPartitionExprIds;
+ private final ImmutableList<ExprId> deletePartitionExprIds;
+ private final boolean insertRandom;
+ private final ImmutableList<IcebergPartitionField> insertPartitionFields;
+ private final Integer partitionSpecId;
+
+ /**
+ * Create merge distribution spec for Iceberg DML routing.
+ */
+ public DistributionSpecMerge(ExprId operationExprId, List<ExprId> insertPartitionExprIds,
+ List<ExprId> deletePartitionExprIds, boolean insertRandom,
+ List<IcebergPartitionField> insertPartitionFields, Integer partitionSpecId) {
+ this.operationExprId = Objects.requireNonNull(operationExprId, "operationExprId should not be null");
+ this.insertPartitionExprIds = ImmutableList.copyOf(
+ Objects.requireNonNull(insertPartitionExprIds, "insertPartitionExprIds should not be null"));
+ this.deletePartitionExprIds = ImmutableList.copyOf(
+ Objects.requireNonNull(deletePartitionExprIds, "deletePartitionExprIds should not be null"));
+ this.insertRandom = insertRandom;
+ this.insertPartitionFields = ImmutableList.copyOf(
+ Objects.requireNonNull(insertPartitionFields, "insertPartitionFields should not be null"));
+ this.partitionSpecId = partitionSpecId;
+ Preconditions.checkState(!deletePartitionExprIds.isEmpty(), "deletePartitionExprIds should not be empty");
+ }
+
+ public ExprId getOperationExprId() {
+ return operationExprId;
+ }
+
+ public List<ExprId> getInsertPartitionExprIds() {
+ return insertPartitionExprIds;
+ }
+
+ public List<ExprId> getDeletePartitionExprIds() {
+ return deletePartitionExprIds;
+ }
+
+ public boolean isInsertRandom() {
+ return insertRandom;
+ }
+
+ public List<IcebergPartitionField> getInsertPartitionFields() {
+ return insertPartitionFields;
+ }
+
+ public Integer getPartitionSpecId() {
+ return partitionSpecId;
+ }
+
+ @Override
+ public boolean satisfy(DistributionSpec required) {
+ if (required instanceof DistributionSpecAny) {
+ return true;
+ }
+ if (!(required instanceof DistributionSpecMerge)) {
+ return false;
+ }
+ DistributionSpecMerge other = (DistributionSpecMerge) required;
+ return insertRandom == other.insertRandom
+ && operationExprId.equals(other.operationExprId)
+ && insertPartitionExprIds.equals(other.insertPartitionExprIds)
+ && deletePartitionExprIds.equals(other.deletePartitionExprIds)
+ && insertPartitionFields.equals(other.insertPartitionFields)
+ && Objects.equals(partitionSpecId, other.partitionSpecId);
+ }
+
+ @Override
+ public String shapeInfo() {
+ return "DistributionSpecMerge";
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ DistributionSpecMerge that = (DistributionSpecMerge) o;
+ return insertRandom == that.insertRandom
+ && operationExprId.equals(that.operationExprId)
+ && insertPartitionExprIds.equals(that.insertPartitionExprIds)
+ && deletePartitionExprIds.equals(that.deletePartitionExprIds)
+ && insertPartitionFields.equals(that.insertPartitionFields)
+ && Objects.equals(partitionSpecId, that.partitionSpecId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(operationExprId, insertPartitionExprIds, deletePartitionExprIds,
+ insertRandom, insertPartitionFields, partitionSpecId);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java
index 505732ab1ad..7610d75a22d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java
@@ -47,6 +47,8 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHiveTableSink;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergDeleteSink;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergMergeSink;
import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergTableSink;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcTableSink;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
@@ -182,6 +184,28 @@ public class RequestPropertyDeriver extends PlanVisitor<Void, PlanContext> {
return null;
}
+ @Override
+ public Void visitPhysicalIcebergDeleteSink(
+ PhysicalIcebergDeleteSink<? extends Plan> icebergDeleteSink, PlanContext context) {
+ if (connectContext != null && !connectContext.getSessionVariable().enableStrictConsistencyDml) {
+ addRequestPropertyToChildren(PhysicalProperties.ANY);
+ } else {
+ addRequestPropertyToChildren(icebergDeleteSink.getRequirePhysicalProperties());
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitPhysicalIcebergMergeSink(
+ PhysicalIcebergMergeSink<? extends Plan> icebergMergeSink, PlanContext context) {
+ if (connectContext != null && !connectContext.getSessionVariable().enableStrictConsistencyDml) {
+ addRequestPropertyToChildren(PhysicalProperties.ANY);
+ } else {
+ addRequestPropertyToChildren(icebergMergeSink.getRequirePhysicalProperties());
+ }
+ return null;
+ }
+
@Override
public Void visitPhysicalJdbcTableSink(
PhysicalJdbcTableSink<? extends Plan> jdbcTableSink, PlanContext context) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index c07682b5609..d1c9166179c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -76,6 +76,8 @@ import org.apache.doris.nereids.rules.implementation.LogicalFilterToPhysicalFilt
import org.apache.doris.nereids.rules.implementation.LogicalGenerateToPhysicalGenerate;
import org.apache.doris.nereids.rules.implementation.LogicalHiveTableSinkToPhysicalHiveTableSink;
import org.apache.doris.nereids.rules.implementation.LogicalHudiScanToPhysicalHudiScan;
+import org.apache.doris.nereids.rules.implementation.LogicalIcebergDeleteSinkToPhysicalIcebergDeleteSink;
+import org.apache.doris.nereids.rules.implementation.LogicalIcebergMergeSinkToPhysicalIcebergMergeSink;
import org.apache.doris.nereids.rules.implementation.LogicalIcebergTableSinkToPhysicalIcebergTableSink;
import org.apache.doris.nereids.rules.implementation.LogicalIntersectToPhysicalIntersect;
import org.apache.doris.nereids.rules.implementation.LogicalJdbcScanToPhysicalJdbcScan;
@@ -230,6 +232,8 @@ public class RuleSet {
.add(new LogicalHiveTableSinkToPhysicalHiveTableSink())
.add(new LogicalIcebergTableSinkToPhysicalIcebergTableSink())
.add(new LogicalMaxComputeTableSinkToPhysicalMaxComputeTableSink())
+ .add(new LogicalIcebergDeleteSinkToPhysicalIcebergDeleteSink())
+ .add(new LogicalIcebergMergeSinkToPhysicalIcebergMergeSink())
.add(new LogicalJdbcTableSinkToPhysicalJdbcTableSink())
.add(new LogicalFileSinkToPhysicalFileSink())
.add(new LogicalResultSinkToPhysicalResultSink())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 15cf4c1e898..6d7f8f25b23 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -35,6 +35,8 @@ public enum RuleType {
// binding rules
BINDING_RESULT_SINK(RuleTypeClass.REWRITE),
+ BINDING_ICEBERG_DELETE_SINK_OUTPUT(RuleTypeClass.REWRITE),
+ BINDING_ICEBERG_MERGE_SINK_OUTPUT(RuleTypeClass.REWRITE),
BINDING_INSERT_BLACKHOLE_SINK(RuleTypeClass.REWRITE),
BINDING_INSERT_HIVE_TABLE(RuleTypeClass.REWRITE),
BINDING_INSERT_ICEBERG_TABLE(RuleTypeClass.REWRITE),
@@ -558,6 +560,8 @@ public enum RuleType {
LOGICAL_HIVE_TABLE_SINK_TO_PHYSICAL_HIVE_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_ICEBERG_TABLE_SINK_TO_PHYSICAL_ICEBERG_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_MAX_COMPUTE_TABLE_SINK_TO_PHYSICAL_MAX_COMPUTE_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
+ LOGICAL_ICEBERG_DELETE_SINK_TO_PHYSICAL_ICEBERG_DELETE_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
+ LOGICAL_ICEBERG_MERGE_SINK_TO_PHYSICAL_ICEBERG_MERGE_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_JDBC_TABLE_SINK_TO_PHYSICAL_JDBC_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_RESULT_SINK_TO_PHYSICAL_RESULT_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_DEFER_MATERIALIZE_RESULT_SINK_TO_PHYSICAL_DEFER_MATERIALIZE_RESULT_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
index 9540a12d40f..39150d69f64 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
@@ -17,14 +17,17 @@
package org.apache.doris.nereids.rules.analysis;
+import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.FunctionRegistry;
import org.apache.doris.common.Pair;
+import org.apache.doris.datasource.iceberg.IcebergMergeOperation;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.SqlCacheContext;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.analyzer.MappingSlot;
import org.apache.doris.nereids.analyzer.Scope;
+import org.apache.doris.nereids.analyzer.Unbound;
import org.apache.doris.nereids.analyzer.UnboundFunction;
import org.apache.doris.nereids.analyzer.UnboundOneRowRelation;
import org.apache.doris.nereids.analyzer.UnboundResultSink;
@@ -77,6 +80,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate;
import org.apache.doris.nereids.trees.plans.logical.LogicalHaving;
+import org.apache.doris.nereids.trees.plans.logical.LogicalIcebergDeleteSink;
+import org.apache.doris.nereids.trees.plans.logical.LogicalIcebergMergeSink;
import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalLoadProject;
@@ -231,6 +236,12 @@ public class BindExpression implements AnalysisRuleFactory {
RuleType.BINDING_SUBQUERY_ALIAS_SLOT.build(
logicalSubQueryAlias().thenApply(this::bindSubqueryAlias)
),
+ RuleType.BINDING_ICEBERG_DELETE_SINK_OUTPUT.build(
+ logicalIcebergDeleteSink().thenApply(this::bindIcebergDeleteSink)
+ ),
+ RuleType.BINDING_ICEBERG_MERGE_SINK_OUTPUT.build(
+ logicalIcebergMergeSink().thenApply(this::bindIcebergMergeSink)
+ ),
RuleType.BINDING_RESULT_SINK.build(
unboundResultSink().thenApply(this::bindResultSink)
)
@@ -261,6 +272,92 @@ public class BindExpression implements AnalysisRuleFactory {
return ctx.root;
}
+ private LogicalIcebergDeleteSink<Plan> bindIcebergDeleteSink(
+ MatchingContext<LogicalIcebergDeleteSink<Plan>> ctx) {
+ LogicalIcebergDeleteSink<Plan> sink = ctx.root;
+ if (hasUnboundPlan(sink.child())) {
+ return sink;
+ }
+ List<NamedExpression> outputExprs = sink.child().getOutput().stream()
+ .map(NamedExpression.class::cast)
+ .collect(ImmutableList.toImmutableList());
+ if (sink.getOutputExprs().equals(outputExprs)) {
+ return sink;
+ }
+ return sink.withOutputExprs(outputExprs);
+ }
+
+ private LogicalIcebergMergeSink<Plan> bindIcebergMergeSink(
+ MatchingContext<LogicalIcebergMergeSink<Plan>> ctx) {
+ LogicalIcebergMergeSink<Plan> sink = ctx.root;
+ if (hasUnboundPlan(sink.child())) {
+ return sink;
+ }
+ List<NamedExpression> outputExprs = sink.child().getOutput().stream()
+ .map(NamedExpression.class::cast)
+ .collect(ImmutableList.toImmutableList());
+ List<Column> visibleColumns = sink.getCols().stream()
+ .filter(Column::isVisible)
+ .collect(ImmutableList.toImmutableList());
+ int dataExprCount = 0;
+ for (NamedExpression expr : outputExprs) {
+ if (!isIcebergMergeMetaColumn(expr.getName())) {
+ dataExprCount++;
+ }
+ }
+ if (dataExprCount != visibleColumns.size()) {
+ if (sink.getOutputExprs().equals(outputExprs)) {
+ return sink;
+ }
+ return sink.withOutputExprs(outputExprs);
+ }
+
+ boolean changed = false;
+ int columnIndex = 0;
+ List<NamedExpression> castExprs = Lists.newArrayListWithCapacity(outputExprs.size());
+ for (NamedExpression expr : outputExprs) {
+ if (isIcebergMergeMetaColumn(expr.getName())) {
+ castExprs.add(expr);
+ continue;
+ }
+ Column column = visibleColumns.get(columnIndex++);
+ DataType targetType = DataType.fromCatalogType(column.getType());
+ Expression castExpr = TypeCoercionUtils.castIfNotSameType(expr, targetType);
+ NamedExpression namedExpr;
+ if (castExpr instanceof NamedExpression) {
+ namedExpr = (NamedExpression) castExpr;
+ if (!column.getName().equalsIgnoreCase(namedExpr.getName())) {
+ namedExpr = new Alias(namedExpr, column.getName());
+ }
+ } else {
+ namedExpr = new Alias(castExpr, column.getName());
+ }
+ if (!namedExpr.equals(expr)) {
+ changed = true;
+ }
+ castExprs.add(namedExpr);
+ }
+ if (!changed) {
+ if (sink.getOutputExprs().equals(outputExprs)) {
+ return sink;
+ }
+ return sink.withOutputExprs(outputExprs);
+ }
+ LogicalProject<?> project = new LogicalProject<>(castExprs, sink.child());
+ return (LogicalIcebergMergeSink<Plan>) sink.withChildAndUpdateOutput(project);
+ }
+
+ private boolean isIcebergMergeMetaColumn(String name) {
+ if (IcebergMergeOperation.OPERATION_COLUMN.equalsIgnoreCase(name)) {
+ return true;
+ }
+ return Column.ICEBERG_ROWID_COL.equalsIgnoreCase(name);
+ }
+
+ private static boolean hasUnboundPlan(Plan plan) {
+ return plan.anyMatch(node -> node instanceof Unbound || ((Plan) node).hasUnboundExpression());
+ }
+
private LogicalPlan bindGenerate(MatchingContext<LogicalGenerate<Plan>> ctx) {
LogicalGenerate<Plan> generate = ctx.root;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java
index f9aaf46d5af..d8d903384ba 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java
@@ -111,6 +111,7 @@ public class ExpressionRewrite implements RewriteRuleFactory {
new LogicalHiveTableSinkRewrite().build(),
new LogicalIcebergTableSinkRewrite().build(),
new LogicalMaxComputeTableSinkRewrite().build(),
+ new LogicalIcebergMergeSinkRewrite().build(),
new LogicalJdbcTableSinkRewrite().build(),
new LogicalOlapTableSinkRewrite().build(),
new LogicalDictionarySinkRewrite().build(),
@@ -527,6 +528,14 @@ public class ExpressionRewrite implements RewriteRuleFactory {
}
}
+ private class LogicalIcebergMergeSinkRewrite extends OneRewriteRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalIcebergMergeSink().thenApply(ExpressionRewrite.this::applyRewriteToSink)
+ .toRule(RuleType.REWRITE_SINK_EXPRESSION);
+ }
+ }
+
private class LogicalJdbcTableSinkRewrite extends OneRewriteRuleFactory {
@Override
public Rule build() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergDeleteSinkToPhysicalIcebergDeleteSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergDeleteSinkToPhysicalIcebergDeleteSink.java
new file mode 100644
index 00000000000..73a7284b155
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergDeleteSinkToPhysicalIcebergDeleteSink.java
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.implementation;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalIcebergDeleteSink;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergDeleteSink;
+
+import java.util.Optional;
+
+/**
+ * Implementation rule that convert logical IcebergDeleteSink to physical IcebergDeleteSink.
+ */
+public class LogicalIcebergDeleteSinkToPhysicalIcebergDeleteSink extends OneImplementationRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalIcebergDeleteSink().thenApply(ctx -> {
+ LogicalIcebergDeleteSink<? extends Plan> sink = ctx.root;
+ return new PhysicalIcebergDeleteSink<>(
+ sink.getDatabase(),
+ sink.getTargetTable(),
+ sink.getCols(),
+ sink.getOutputExprs(),
+ sink.getDeleteContext(),
+ Optional.empty(),
+ sink.getLogicalProperties(),
+ null,
+ null,
+ sink.child());
+ }).toRule(RuleType.LOGICAL_ICEBERG_DELETE_SINK_TO_PHYSICAL_ICEBERG_DELETE_SINK_RULE);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergMergeSinkToPhysicalIcebergMergeSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergMergeSinkToPhysicalIcebergMergeSink.java
new file mode 100644
index 00000000000..9447aaf09d1
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergMergeSinkToPhysicalIcebergMergeSink.java
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.implementation;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalIcebergMergeSink;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergMergeSink;
+
+import java.util.Optional;
+
+/**
+ * Implementation rule that convert logical IcebergMergeSink to physical IcebergMergeSink.
+ */
+public class LogicalIcebergMergeSinkToPhysicalIcebergMergeSink extends OneImplementationRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalIcebergMergeSink().thenApply(ctx -> {
+ LogicalIcebergMergeSink<? extends Plan> sink = ctx.root;
+ return new PhysicalIcebergMergeSink<>(
+ sink.getDatabase(),
+ sink.getTargetTable(),
+ sink.getCols(),
+ sink.getOutputExprs(),
+ sink.getDeleteContext(),
+ Optional.empty(),
+ sink.getLogicalProperties(),
+ null,
+ null,
+ sink.child());
+ }).toRule(RuleType.LOGICAL_ICEBERG_MERGE_SINK_TO_PHYSICAL_ICEBERG_MERGE_SINK_RULE);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 3c0ee3371a0..949283816a2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -51,6 +51,8 @@ public enum PlanType {
LOGICAL_HIVE_TABLE_SINK,
LOGICAL_ICEBERG_TABLE_SINK,
LOGICAL_MAX_COMPUTE_TABLE_SINK,
+ LOGICAL_ICEBERG_DELETE_SINK,
+ LOGICAL_ICEBERG_MERGE_SINK,
LOGICAL_JDBC_TABLE_SINK,
LOGICAL_RESULT_SINK,
LOGICAL_BLACKHOLE_SINK,
@@ -118,6 +120,8 @@ public enum PlanType {
PHYSICAL_HIVE_TABLE_SINK,
PHYSICAL_ICEBERG_TABLE_SINK,
PHYSICAL_MAX_COMPUTE_TABLE_SINK,
+ PHYSICAL_ICEBERG_DELETE_SINK,
+ PHYSICAL_ICEBERG_MERGE_SINK,
PHYSICAL_JDBC_TABLE_SINK,
PHYSICAL_RESULT_SINK,
PHYSICAL_BLACKHOLE_SINK,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
index be067378310..57e62da8aa9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
@@ -38,6 +38,7 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.NereidsPlanner;
@@ -67,6 +68,7 @@ import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
import org.apache.doris.nereids.trees.plans.Explainable;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext;
import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
@@ -128,6 +130,30 @@ public class DeleteFromCommand extends Command implements ForwardWithSync, Expla
@Override
public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+ // Check if target table is Iceberg table and route to IcebergDeleteCommand if so
+ List<String> qualifiedTableName = RelationUtil.getQualifierName(ctx, nameParts);
+ TableIf table = null;
+ try {
+ table = RelationUtil.getTable(qualifiedTableName, ctx.getEnv(), Optional.empty());
+ } catch (Exception e) {
+ // Table not found, will be handled by regular error flow
+ }
+
+ // Route to IcebergDeleteCommand for Iceberg tables
+ if (table instanceof org.apache.doris.datasource.iceberg.IcebergExternalTable) {
+ LOG.info("Routing DELETE to IcebergDeleteCommand for table: {}", table.getName());
+ org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext deleteCtx =
+ new org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext();
+ deleteCtx.setDeleteFileType(org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext
+ .DeleteFileType.POSITION_DELETE);
+ IcebergDeleteCommand icebergDeleteCommand = new IcebergDeleteCommand(
+ nameParts, tableAlias, isTempPart, partitions, logicalQuery,
+ deleteCtx);
+ icebergDeleteCommand.run(ctx, executor);
+ return;
+ }
+
+ // Continue with OLAP table delete logic
LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(logicalQuery, ctx.getStatementContext());
updateSessionVariableForDelete(ctx.getSessionVariable());
StatementContext statementContext = ctx.getStatementContext();
@@ -457,6 +483,15 @@ public class DeleteFromCommand extends Command implements ForwardWithSync, Expla
@Override
public Plan getExplainPlan(ConnectContext ctx) {
+ List<String> qualifiedTableName = RelationUtil.getQualifierName(ctx, nameParts);
+ TableIf table = RelationUtil.getTable(qualifiedTableName, ctx.getEnv(), Optional.empty());
+ if (table instanceof IcebergExternalTable) {
+ DeleteCommandContext deleteCtx = new DeleteCommandContext();
+ deleteCtx.setDeleteFileType(DeleteCommandContext.DeleteFileType.POSITION_DELETE);
+ IcebergDeleteCommand icebergDeleteCommand = new IcebergDeleteCommand(
+ nameParts, tableAlias, isTempPart, partitions, logicalQuery, deleteCtx);
+ return icebergDeleteCommand.getExplainPlan(ctx);
+ }
return completeQueryPlan(ctx, logicalQuery);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
index badd660734c..e2194048e48 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
@@ -27,6 +27,8 @@ import org.apache.doris.nereids.trees.plans.Explainable;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
import org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand;
+import org.apache.doris.nereids.trees.plans.logical.LogicalIcebergDeleteSink;
+import org.apache.doris.nereids.trees.plans.logical.LogicalIcebergMergeSink;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.planner.ScanNode;
@@ -95,24 +97,45 @@ public class ExplainCommand extends Command implements NoForward {
new NereidsPlanner(ctx.getStatementContext())
);
- LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(explainPlan, ctx.getStatementContext());
- ExplainOptions explainOptions = new ExplainOptions(level, showPlanProcess);
- logicalPlanAdapter.setIsExplain(explainOptions);
- executor.setParsedStmt(logicalPlanAdapter);
- if (ctx.getSessionVariable().isEnableMaterializedViewRewrite()) {
- ctx.getStatementContext().addPlannerHook(InitMaterializationContextHook.INSTANCE);
+ long previousTargetTableId = ctx.getIcebergRowIdTargetTableId();
+ boolean resetTargetTableId = false;
+ if (explainPlan instanceof LogicalIcebergDeleteSink) {
+ if (previousTargetTableId < 0) {
+ ctx.setIcebergRowIdTargetTableId(
+ ((LogicalIcebergDeleteSink<?>) explainPlan).getTargetTable().getId());
+ resetTargetTableId = true;
+ }
+ } else if (explainPlan instanceof LogicalIcebergMergeSink) {
+ if (previousTargetTableId < 0) {
+ ctx.setIcebergRowIdTargetTableId(
+ ((LogicalIcebergMergeSink<?>) explainPlan).getTargetTable().getId());
+ resetTargetTableId = true;
+ }
}
- planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift());
- executor.setPlanner(planner);
- // Skip SQL block rules check for EXPLAIN statements since they only show
- // the execution plan without actually executing the query
- if (showPlanProcess) {
- executor.handleExplainPlanProcessStmt(planner.getCascadesContext().getPlanProcesses());
- } else {
- executor.handleExplainStmt(planner.getExplainString(explainOptions), true);
- }
- for (ScanNode scanNode : planner.getScanNodes()) {
- scanNode.stop();
+ try {
+ LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(explainPlan, ctx.getStatementContext());
+ ExplainOptions explainOptions = new ExplainOptions(level, showPlanProcess);
+ logicalPlanAdapter.setIsExplain(explainOptions);
+ executor.setParsedStmt(logicalPlanAdapter);
+ if (ctx.getSessionVariable().isEnableMaterializedViewRewrite()) {
+ ctx.getStatementContext().addPlannerHook(InitMaterializationContextHook.INSTANCE);
+ }
+ planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift());
+ executor.setPlanner(planner);
+ // Skip SQL block rules check for EXPLAIN statements since they only show
+ // the execution plan without actually executing the query
+ if (showPlanProcess) {
+ executor.handleExplainPlanProcessStmt(planner.getCascadesContext().getPlanProcesses());
+ } else {
+ executor.handleExplainStmt(planner.getExplainString(explainOptions), true);
+ }
+ for (ScanNode scanNode : planner.getScanNodes()) {
+ scanNode.stop();
+ }
+ } finally {
+ if (resetTargetTableId) {
+ ctx.setIcebergRowIdTargetTableId(previousTargetTableId);
+ }
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergDeleteCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergDeleteCommand.java
new file mode 100644
index 00000000000..f7da7f5dc4f
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergDeleteCommand.java
@@ -0,0 +1,288 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.StmtType;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.iceberg.IcebergConflictDetectionFilterUtils;
+import org.apache.doris.datasource.iceberg.IcebergExternalDatabase;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
+import org.apache.doris.datasource.iceberg.IcebergMergeOperation;
+import org.apache.doris.datasource.iceberg.IcebergNereidsUtils;
+import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.nereids.analyzer.UnboundAlias;
+import org.apache.doris.nereids.analyzer.UnboundSlot;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.glue.LogicalPlanAdapter;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
+import org.apache.doris.nereids.trees.plans.Explainable;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext;
+import org.apache.doris.nereids.trees.plans.commands.insert.IcebergDeleteExecutor;
+import org.apache.doris.nereids.trees.plans.logical.LogicalIcebergDeleteSink;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergDeleteSink;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalSink;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.RelationUtil;
+import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.planner.DataSink;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * DELETE command for Iceberg tables.
+ *
+ * This command converts DELETE operations to INSERT operations that generate
+ * position DeleteFile entries instead of data files.
+ *
+ * Example:
+ * DELETE FROM iceberg_table WHERE id = 1
+ *
+ * This will:
+ * 1. Scan rows matching the WHERE condition
+ * 2. Generate DeleteFile containing the matching rows
+ * 3. Commit the DeleteFile to Iceberg table using RowDelta API
+ */
+public class IcebergDeleteCommand extends Command implements ForwardWithSync, Explainable {
+
+ protected final List<String> nameParts;
+ protected final String tableAlias;
+ protected final boolean isTempPart;
+ protected final List<String> partitions;
+ protected final LogicalPlan logicalQuery;
+ protected final DeleteCommandContext deleteCtx;
+
+ /**
+ * constructor
+ */
+ public IcebergDeleteCommand(
+ List<String> nameParts,
+ String tableAlias,
+ boolean isTempPart,
+ List<String> partitions,
+ LogicalPlan logicalQuery,
+ DeleteCommandContext deleteCtx) {
+ super(PlanType.DELETE_COMMAND);
+ this.nameParts = Utils.copyRequiredList(nameParts);
+ this.tableAlias = tableAlias;
+ this.isTempPart = isTempPart;
+ this.partitions = Utils.copyRequiredList(partitions);
+ this.logicalQuery = logicalQuery;
+ this.deleteCtx = deleteCtx != null ? deleteCtx : new DeleteCommandContext();
+ }
+
+ @Override
+ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+ // Check if target table is Iceberg table
+ List<String> qualifiedTableName = RelationUtil.getQualifierName(ctx, nameParts);
+ TableIf table = RelationUtil.getTable(qualifiedTableName, ctx.getEnv(), Optional.empty());
+
+ if (!(table instanceof IcebergExternalTable)) {
+ throw new AnalysisException("DELETE command can only be used on Iceberg tables. "
+ + "Table " + Util.getTempTableDisplayName(table.getName()) + " is not an Iceberg table.");
+ }
+
+ IcebergExternalTable icebergTable = (IcebergExternalTable) table;
+
+ // Verify table format version (must be v2+ for delete support)
+ // org.apache.iceberg.Table icebergTableObj = icebergTable.getIcebergTable();
+ // String formatVersionStr = icebergTableObj.properties().get("format-version");
+ // int formatVersion = formatVersionStr != null ? Integer.parseInt(formatVersionStr) : 1;
+ // if (formatVersion < 2) {
+ // throw new AnalysisException("Iceberg table DELETE requires format version >= 2. "
+ // + "Current format version: " + formatVersion);
+ // }
+
+ long previousTargetTableId = ctx.getIcebergRowIdTargetTableId();
+ ctx.setIcebergRowIdTargetTableId(icebergTable.getId());
+ try {
+ // Build query plan with DELETE sink
+ LogicalPlan deleteQueryPlan = completeQueryPlan(ctx, logicalQuery, icebergTable);
+
+ // Create planner and plan the delete operation
+ NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext());
+ LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(deleteQueryPlan, ctx.getStatementContext());
+
+ // Plan the delete query to generate physical plan and distributed plan
+ planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift());
+
+ // Set planner in executor for later use
+ executor.setPlanner(planner);
+ executor.checkBlockRules();
+ Optional<org.apache.iceberg.expressions.Expression> conflictFilter =
+ IcebergConflictDetectionFilterUtils.buildConflictDetectionFilter(
+ planner.getAnalyzedPlan(), icebergTable);
+
+ PhysicalSink<?> physicalSink = getPhysicalSink(planner);
+ PlanFragment fragment = planner.getFragments().get(0);
+ DataSink dataSink = fragment.getSink();
+ boolean emptyInsert = childIsEmptyRelation(physicalSink);
+ String label = String.format("iceberg_delete_%x_%x", ctx.queryId().hi, ctx.queryId().lo);
+
+ // Create IcebergDeleteExecutor and execute
+ IcebergDeleteExecutor deleteExecutor = new IcebergDeleteExecutor(
+ ctx,
+ icebergTable,
+ label,
+ planner,
+ emptyInsert,
+ -1L);
+ deleteExecutor.setConflictDetectionFilter(conflictFilter);
+
+ if (deleteExecutor.isEmptyInsert()) {
+ return;
+ }
+
+ deleteExecutor.beginTransaction();
+ deleteExecutor.finalizeSinkForDelete(fragment, dataSink, physicalSink);
+ deleteExecutor.getCoordinator().setTxnId(deleteExecutor.getTxnId());
+ executor.setCoord(deleteExecutor.getCoordinator());
+ deleteExecutor.executeSingleInsert(executor);
+ } finally {
+ ctx.setIcebergRowIdTargetTableId(previousTargetTableId);
+ }
+ }
+
+ /**
+ * Complete the query plan by adding necessary columns for position delete operation.
+ * Select $row_id (file_path, row_position, partition info).
+ */
+ private LogicalPlan completeQueryPlan(ConnectContext ctx, LogicalPlan logicalQuery,
+ IcebergExternalTable icebergTable) {
+ LogicalPlan queryPlan = buildPositionDeletePlan(ctx, logicalQuery, icebergTable);
+
+ // Convert output to NamedExpression list
+ List<NamedExpression> outputExprs;
+ if (!IcebergNereidsUtils.hasUnboundPlan(queryPlan)) {
+ outputExprs = queryPlan.getOutput().stream()
+ .map(slot -> (NamedExpression) slot)
+ .collect(java.util.stream.Collectors.toList());
+ } else if (queryPlan instanceof LogicalProject) {
+ outputExprs = ((LogicalProject<?>) queryPlan).getProjects();
+ } else {
+ outputExprs = ImmutableList.of();
+ }
+
+ // Wrap query plan with LogicalIcebergDeleteSink
+ LogicalIcebergDeleteSink<LogicalPlan> deleteSink = new LogicalIcebergDeleteSink<>(
+ (IcebergExternalDatabase) icebergTable.getDatabase(),
+ icebergTable,
+ icebergTable.getBaseSchema(true), // cols
+ outputExprs, // outputExprs
+ deleteCtx,
+ Optional.empty(), // groupExpression
+ Optional.empty(), // logicalProperties
+ queryPlan // child
+ );
+
+ return deleteSink;
+ }
+
+ /**
+ * Build query plan for position delete.
+ * Add $row_id column to select list.
+ *
+ * This follows Trino's approach:
+ * 1. Original query filters rows based on WHERE clause
+ * 2. We project $row_id metadata column from matching rows
+ * 3. The $row_id contains (file_path, row_position, partition_spec_id, partition_data)
+ * 4. These will be written to Position Delete file
+ */
+ private LogicalPlan buildPositionDeletePlan(ConnectContext ctx, LogicalPlan logicalQuery,
+ IcebergExternalTable icebergTable) {
+ // Step 1: Inject $row_id metadata column into the scan
+ LogicalPlan planWithRowId = IcebergNereidsUtils.injectRowIdColumn(logicalQuery);
+
+ // Step 2: Project operation + __DORIS_ICEBERG_ROWID_COL__
+ Optional<Slot> rowIdSlot = Optional.empty();
+ if (!IcebergNereidsUtils.hasUnboundPlan(planWithRowId)) {
+ rowIdSlot = IcebergNereidsUtils.findRowIdSlot(planWithRowId.getOutput());
+ }
+ NamedExpression operationColumn = new UnboundAlias(
+ new TinyIntLiteral(IcebergMergeOperation.DELETE_OPERATION_NUMBER),
+ IcebergMergeOperation.OPERATION_COLUMN);
+ NamedExpression rowIdColumn = rowIdSlot.isPresent()
+ ? (NamedExpression) rowIdSlot.get()
+ : new UnboundSlot(Column.ICEBERG_ROWID_COL);
+ List<NamedExpression> projectItems = ImmutableList.of(operationColumn, rowIdColumn);
+
+ return new LogicalProject<>(projectItems, planWithRowId);
+ }
+
+ private PhysicalSink<?> getPhysicalSink(NereidsPlanner planner) {
+ Optional<PhysicalSink<?>> plan = planner.getPhysicalPlan()
+ .<PhysicalSink<?>>collect(PhysicalSink.class::isInstance).stream().findAny();
+ if (!plan.isPresent()) {
+ throw new AnalysisException("DELETE command must contain target table");
+ }
+ PhysicalSink<?> sink = plan.get();
+ if (!(sink instanceof PhysicalIcebergDeleteSink)) {
+ throw new AnalysisException("DELETE plan must use Iceberg delete sink");
+ }
+ return sink;
+ }
+
+ private boolean childIsEmptyRelation(PhysicalSink<?> sink) {
+ return sink.children() != null && sink.children().size() == 1
+ && sink.child(0) instanceof PhysicalEmptyRelation;
+ }
+
+ @Override
+ public Plan getExplainPlan(ConnectContext ctx) {
+ List<String> qualifiedTableName = RelationUtil.getQualifierName(ctx, nameParts);
+ TableIf table = RelationUtil.getTable(qualifiedTableName, ctx.getEnv(), Optional.empty());
+ if (!(table instanceof IcebergExternalTable)) {
+ throw new AnalysisException("Table must be IcebergExternalTable in DELETE command");
+ }
+ long previousTargetTableId = ctx.getIcebergRowIdTargetTableId();
+ ctx.setIcebergRowIdTargetTableId(table.getId());
+ try {
+ return completeQueryPlan(ctx, logicalQuery, (IcebergExternalTable) table);
+ } finally {
+ ctx.setIcebergRowIdTargetTableId(previousTargetTableId);
+ }
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitCommand(this, context);
+ }
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DELETE;
+ }
+
+ public DeleteCommandContext getDeleteCtx() {
+ return deleteCtx;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergMergeCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergMergeCommand.java
new file mode 100644
index 00000000000..c54d6a8889b
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergMergeCommand.java
@@ -0,0 +1,519 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.StmtType;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.iceberg.IcebergConflictDetectionFilterUtils;
+import org.apache.doris.datasource.iceberg.IcebergExternalDatabase;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
+import org.apache.doris.datasource.iceberg.IcebergMergeOperation;
+import org.apache.doris.datasource.iceberg.IcebergNereidsUtils;
+import org.apache.doris.datasource.iceberg.IcebergRowId;
+import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.nereids.analyzer.UnboundAlias;
+import org.apache.doris.nereids.analyzer.UnboundRelation;
+import org.apache.doris.nereids.analyzer.UnboundSlot;
+import org.apache.doris.nereids.analyzer.UnboundStar;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.glue.LogicalPlanAdapter;
+import org.apache.doris.nereids.parser.LogicalPlanBuilderAssistant;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.rules.exploration.join.JoinReorderContext;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.IsNull;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Not;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.If;
+import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
+import org.apache.doris.nereids.trees.plans.Explainable;
+import org.apache.doris.nereids.trees.plans.JoinType;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext;
+import org.apache.doris.nereids.trees.plans.commands.insert.IcebergMergeExecutor;
+import org.apache.doris.nereids.trees.plans.commands.merge.MergeMatchedClause;
+import org.apache.doris.nereids.trees.plans.commands.merge.MergeNotMatchedClause;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalIcebergMergeSink;
+import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergMergeSink;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalSink;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.IntegerType;
+import org.apache.doris.nereids.util.RelationUtil;
+import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.planner.DataSink;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.QueryState;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * MERGE INTO command for Iceberg tables.
+ */
+public class IcebergMergeCommand extends Command implements ForwardWithSync, Explainable {
+ private static final String BRANCH_LABEL = "__DORIS_ICEBERG_MERGE_INTO_BRANCH_LABEL__";
+
+ private final List<String> targetNameParts;
+ private final Optional<String> targetAlias;
+ private final List<String> targetNameInPlan;
+ private final Optional<LogicalPlan> cte;
+ private final LogicalPlan source;
+ private final Expression onClause;
+ private final List<MergeMatchedClause> matchedClauses;
+ private final List<MergeNotMatchedClause> notMatchedClauses;
+ private final DeleteCommandContext deleteCtx;
+
+ /**
+ * constructor.
+ */
+ public IcebergMergeCommand(List<String> targetNameParts, Optional<String> targetAlias,
+ Optional<LogicalPlan> cte, LogicalPlan source, Expression onClause,
+ List<MergeMatchedClause> matchedClauses, List<MergeNotMatchedClause> notMatchedClauses) {
+ super(PlanType.MERGE_INTO_COMMAND);
+ this.targetNameParts = Utils.copyRequiredList(targetNameParts);
+ this.targetAlias = Objects.requireNonNull(targetAlias, "targetAlias should not be null");
+ if (targetAlias.isPresent()) {
+ this.targetNameInPlan = ImmutableList.of(targetAlias.get());
+ } else {
+ this.targetNameInPlan = ImmutableList.copyOf(targetNameParts);
+ }
+ this.cte = Objects.requireNonNull(cte, "cte should not be null");
+ this.source = Objects.requireNonNull(source, "source should not be null");
+ this.onClause = Objects.requireNonNull(onClause, "onClause should not be null");
+ this.matchedClauses = Utils.fastToImmutableList(
+ Objects.requireNonNull(matchedClauses, "matchedClauses should not be null"));
+ this.notMatchedClauses = Utils.fastToImmutableList(
+ Objects.requireNonNull(notMatchedClauses, "notMatchedClauses should not be null"));
+ this.deleteCtx = new DeleteCommandContext();
+ }
+
+ @Override
+ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+ TableIf table = getTargetTable(ctx);
+ if (!(table instanceof IcebergExternalTable)) {
+ throw new AnalysisException("MERGE INTO can only be used on Iceberg tables. "
+ + "Table " + Util.getTempTableDisplayName(table.getName()) + " is not an Iceberg table.");
+ }
+ IcebergExternalTable icebergTable = (IcebergExternalTable) table;
+ long previousTargetTableId = ctx.getIcebergRowIdTargetTableId();
+ ctx.setIcebergRowIdTargetTableId(icebergTable.getId());
+ try {
+ LogicalPlan mergePlan = buildMergePlan(ctx, icebergTable);
+ executeMergePlan(ctx, executor, icebergTable, mergePlan);
+ } finally {
+ ctx.setIcebergRowIdTargetTableId(previousTargetTableId);
+ }
+ }
+
+ @Override
+ public Plan getExplainPlan(ConnectContext ctx) {
+ TableIf table = getTargetTable(ctx);
+ if (!(table instanceof IcebergExternalTable)) {
+ throw new AnalysisException("MERGE INTO can only be used on Iceberg tables. "
+ + "Table " + Util.getTempTableDisplayName(table.getName()) + " is not an Iceberg table.");
+ }
+ long previousTargetTableId = ctx.getIcebergRowIdTargetTableId();
+ ctx.setIcebergRowIdTargetTableId(((IcebergExternalTable) table).getId());
+ try {
+ return buildMergePlan(ctx, (IcebergExternalTable) table);
+ } finally {
+ ctx.setIcebergRowIdTargetTableId(previousTargetTableId);
+ }
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitCommand(this, context);
+ }
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.MERGE_INTO;
+ }
+
+ private TableIf getTargetTable(ConnectContext ctx) {
+ List<String> qualifiedTableName = RelationUtil.getQualifierName(ctx, targetNameParts);
+ return RelationUtil.getTable(qualifiedTableName, ctx.getEnv(), Optional.empty());
+ }
+
+ private LogicalPlan generateBasePlan() {
+ LogicalPlan targetPlan = LogicalPlanBuilderAssistant.withCheckPolicy(
+ new UnboundRelation(
+ StatementScopeIdGenerator.newRelationId(),
+ targetNameParts
+ )
+ );
+ if (targetAlias.isPresent()) {
+ targetPlan = new LogicalSubQueryAlias<>(targetAlias.get(), targetPlan);
+ }
+ // Use INNER JOIN when there are no WHEN NOT MATCHED clauses, since unmatched
+ // source rows are not needed. This allows early filtering for better performance.
+ JoinType joinType = notMatchedClauses.isEmpty()
+ ? JoinType.INNER_JOIN : JoinType.LEFT_OUTER_JOIN;
+ return new LogicalJoin<>(joinType,
+ ImmutableList.of(), ImmutableList.of(onClause),
+ source, targetPlan, JoinReorderContext.EMPTY);
+ }
+
+ private NamedExpression generateBranchLabel(Expression rowIdExpr) {
+ Expression matchedLabel = new NullLiteral(IntegerType.INSTANCE);
+ for (int i = matchedClauses.size() - 1; i >= 0; i--) {
+ MergeMatchedClause clause = matchedClauses.get(i);
+ if (i != matchedClauses.size() - 1 && !clause.getCasePredicate().isPresent()) {
+ throw new AnalysisException("Only the last matched clause could without case predicate.");
+ }
+ Expression currentResult = new IntegerLiteral(i);
+ if (clause.getCasePredicate().isPresent()) {
+ matchedLabel = new If(clause.getCasePredicate().get(), currentResult, matchedLabel);
+ } else {
+ matchedLabel = currentResult;
+ }
+ }
+
+ Expression notMatchedLabel = new NullLiteral(IntegerType.INSTANCE);
+ for (int i = notMatchedClauses.size() - 1; i >= 0; i--) {
+ MergeNotMatchedClause clause = notMatchedClauses.get(i);
+ if (i != notMatchedClauses.size() - 1 && !clause.getCasePredicate().isPresent()) {
+ throw new AnalysisException("Only the last not matched clause could without case predicate.");
+ }
+ Expression currentResult = new IntegerLiteral(i + matchedClauses.size());
+ if (clause.getCasePredicate().isPresent()) {
+ notMatchedLabel = new If(clause.getCasePredicate().get(), currentResult, notMatchedLabel);
+ } else {
+ notMatchedLabel = currentResult;
+ }
+ }
+
+ return new UnboundAlias(new If(new Not(new IsNull(rowIdExpr)), matchedLabel, notMatchedLabel),
+ BRANCH_LABEL);
+ }
+
+ private List<Expression> buildDeleteProjection(Expression rowIdExpr, List<Column> columns) {
+ List<Expression> projection = new ArrayList<>();
+ projection.add(new TinyIntLiteral(IcebergMergeOperation.DELETE_OPERATION_NUMBER));
+ projection.add(rowIdExpr);
+ for (Column column : columns) {
+ if (!column.isVisible()) {
+ continue;
+ }
+ List<String> nameParts = Lists.newArrayList(targetNameInPlan);
+ nameParts.add(column.getName());
+ projection.add(new UnboundSlot(nameParts));
+ }
+ return projection;
+ }
+
+ private List<Expression> buildUpdateProjection(MergeMatchedClause clause, Expression rowIdExpr,
+ List<Column> columns, ConnectContext ctx) {
+ Map<String, Expression> colNameToExpression = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
+ for (EqualTo equalTo : clause.getAssignments()) {
+ List<String> nameParts = ((UnboundSlot) equalTo.left()).getNameParts();
+ UpdateCommand.checkAssignmentColumn(ctx, nameParts, targetNameParts, targetAlias.orElse(null));
+ String columnName = nameParts.get(nameParts.size() - 1);
+ if (colNameToExpression.put(columnName, equalTo.right()) != null) {
+ throw new AnalysisException("Duplicate column name in update: " + columnName);
+ }
+ }
+ List<Expression> projection = new ArrayList<>();
+ projection.add(new TinyIntLiteral(IcebergMergeOperation.UPDATE_OPERATION_NUMBER));
+ projection.add(rowIdExpr);
+ for (Column column : columns) {
+ if (!column.isVisible()) {
+ continue;
+ }
+ if (column.isGeneratedColumn()) {
+ throw new AnalysisException("The value specified for generated column '"
+ + column.getName() + "' in table '" + getTargetTable(ctx).getName() + "' is not allowed.");
+ }
+ if (colNameToExpression.containsKey(column.getName())) {
+ projection.add(colNameToExpression.remove(column.getName()));
+ } else {
+ List<String> nameParts = Lists.newArrayList(targetNameInPlan);
+ nameParts.add(column.getName());
+ projection.add(new UnboundSlot(nameParts));
+ }
+ }
+ if (!colNameToExpression.isEmpty()) {
+ throw new AnalysisException("unknown column in assignment list: "
+ + String.join(", ", colNameToExpression.keySet()));
+ }
+ return projection;
+ }
+
+ private List<Expression> buildInsertProjection(MergeNotMatchedClause clause,
+ List<Column> columns, ConnectContext ctx, DataType rowIdType) {
+ Map<String, Expression> colNameToExpression = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
+ if (!clause.getColNames().isEmpty()) {
+ if (clause.getColNames().size() != clause.getRow().size()) {
+ throw new AnalysisException("Column count doesn't match value count");
+ }
+ for (int i = 0; i < clause.getColNames().size(); i++) {
+ String targetColumnName = clause.getColNames().get(i);
+ NamedExpression rowItem = clause.getRow().get(i);
+ Expression value = rowItem instanceof UnboundAlias ? rowItem.child(0) : rowItem;
+ if (rowItem instanceof Alias) {
+ value = rowItem.child(0);
+ }
+ if (colNameToExpression.put(targetColumnName, value) != null) {
+ throw new AnalysisException("insert has duplicate column names");
+ }
+ }
+ } else {
+ long visibleColumnCount = columns.stream().filter(Column::isVisible).count();
+ if (visibleColumnCount != clause.getRow().size()) {
+ throw new AnalysisException("Column count doesn't match value count");
+ }
+ }
+
+ List<Expression> projection = new ArrayList<>();
+ projection.add(new TinyIntLiteral(IcebergMergeOperation.INSERT_OPERATION_NUMBER));
+ projection.add(new NullLiteral(rowIdType));
+
+ int visibleIndex = 0;
+ for (Column column : columns) {
+ if (!column.isVisible()) {
+ continue;
+ }
+ if (column.isGeneratedColumn()) {
+ throw new AnalysisException("The value specified for generated column '"
+ + column.getName() + "' in table '" + getTargetTable(ctx).getName() + "' is not allowed.");
+ }
+ Expression value = null;
+ if (!clause.getColNames().isEmpty()) {
+ value = colNameToExpression.remove(column.getName());
+ } else {
+ NamedExpression rowItem = clause.getRow().get(visibleIndex++);
+ value = rowItem instanceof UnboundAlias ? rowItem.child(0) : rowItem;
+ if (rowItem instanceof Alias) {
+ value = rowItem.child(0);
+ }
+ }
+ if (value == null) {
+ if (column.getDefaultValueSql() != null) {
+ Expression unboundDefaultValue = new NereidsParser()
+ .parseExpression(column.getDefaultValueSql());
+ if (unboundDefaultValue instanceof UnboundAlias) {
+ unboundDefaultValue = unboundDefaultValue.child(0);
+ }
+ value = unboundDefaultValue;
+ } else if (column.isAllowNull()) {
+ value = new NullLiteral(DataType.fromCatalogType(column.getType()));
+ } else {
+ throw new AnalysisException("Column has no default value, column=" + column.getName());
+ }
+ }
+ projection.add(value);
+ }
+ if (!colNameToExpression.isEmpty()) {
+ throw new AnalysisException("unknown column in target table: "
+ + String.join(", ", colNameToExpression.keySet()));
+ }
+ return projection;
+ }
+
+ private List<NamedExpression> generateFinalProjections(List<String> colNames,
+ List<List<Expression>> finalProjections) {
+ for (List<Expression> projection : finalProjections) {
+ if (projection.size() != finalProjections.get(0).size()) {
+ throw new AnalysisException("Column count doesn't match each other");
+ }
+ }
+ List<NamedExpression> output = new ArrayList<>();
+ for (int i = 0; i < finalProjections.get(0).size(); i++) {
+ Expression project = new NullLiteral();
+ for (int j = 0; j < finalProjections.size(); j++) {
+ project = new If(new EqualTo(new UnboundSlot(BRANCH_LABEL), new IntegerLiteral(j)),
+ finalProjections.get(j).get(i), project);
+ }
+ output.add(new UnboundAlias(project, colNames.get(i)));
+ }
+ return output;
+ }
+
+ private LogicalPlan buildMergeProjectPlan(ConnectContext ctx, IcebergExternalTable icebergTable) {
+ List<Column> columns = icebergTable.getBaseSchema(true);
+
+ LogicalPlan plan = generateBasePlan();
+ plan = injectRowIdColumn(plan, icebergTable);
+
+ Expression rowIdExpr = getTargetRowIdSlot();
+ if (!IcebergNereidsUtils.hasUnboundPlan(plan)) {
+ Optional<Slot> rowIdSlot = IcebergNereidsUtils.findRowIdSlot(plan.getOutput());
+ if (rowIdSlot.isPresent()) {
+ rowIdExpr = rowIdSlot.get();
+ }
+ }
+ List<NamedExpression> outputProjections = new ArrayList<>();
+ outputProjections.add(new UnboundStar(ImmutableList.of()));
+ if (!Util.showHiddenColumns()) {
+ outputProjections.add((NamedExpression) rowIdExpr);
+ }
+ outputProjections.add(generateBranchLabel(rowIdExpr));
+ plan = new LogicalProject<>(outputProjections, plan);
+
+ plan = new LogicalFilter<>(ImmutableSet.of(new Not(new IsNull(new UnboundSlot(BRANCH_LABEL)))), plan);
+
+ List<List<Expression>> finalProjections = new ArrayList<>();
+ for (MergeMatchedClause clause : matchedClauses) {
+ if (clause.isDelete()) {
+ finalProjections.add(buildDeleteProjection(rowIdExpr, columns));
+ } else {
+ finalProjections.add(buildUpdateProjection(clause, rowIdExpr, columns, ctx));
+ }
+ }
+
+ DataType rowIdType = DataType.fromCatalogType(IcebergRowId.getRowIdType());
+ for (MergeNotMatchedClause clause : notMatchedClauses) {
+ finalProjections.add(buildInsertProjection(clause, columns, ctx, rowIdType));
+ }
+
+ List<String> colNames = new ArrayList<>();
+ colNames.add(IcebergMergeOperation.OPERATION_COLUMN);
+ colNames.add(Column.ICEBERG_ROWID_COL);
+ for (Column column : columns) {
+ if (column.isVisible()) {
+ colNames.add(column.getName());
+ }
+ }
+ plan = new LogicalProject<>(generateFinalProjections(colNames, finalProjections), plan);
+
+ if (cte.isPresent()) {
+ plan = (LogicalPlan) cte.get().withChildren(plan);
+ }
+ return plan;
+ }
+
+ private LogicalPlan buildMergePlan(ConnectContext ctx, IcebergExternalTable icebergTable) {
+ LogicalPlan projectPlan = buildMergeProjectPlan(ctx, icebergTable);
+
+ List<NamedExpression> outputExprs;
+ if (!IcebergNereidsUtils.hasUnboundPlan(projectPlan)) {
+ outputExprs = projectPlan.getOutput().stream()
+ .map(NamedExpression.class::cast)
+ .collect(ImmutableList.toImmutableList());
+ } else if (projectPlan instanceof LogicalProject) {
+ outputExprs = ((LogicalProject<?>) projectPlan).getProjects();
+ } else {
+ outputExprs = ImmutableList.of();
+ }
+
+ return new LogicalIcebergMergeSink<>(
+ (IcebergExternalDatabase) icebergTable.getDatabase(),
+ icebergTable,
+ icebergTable.getBaseSchema(true),
+ outputExprs,
+ deleteCtx,
+ Optional.empty(),
+ Optional.empty(),
+ projectPlan);
+ }
+
+ private boolean executeMergePlan(ConnectContext ctx, StmtExecutor executor,
+ IcebergExternalTable icebergTable,
+ LogicalPlan logicalPlan) throws Exception {
+ LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(logicalPlan, ctx.getStatementContext());
+ NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext());
+ planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift());
+ executor.setPlanner(planner);
+ executor.checkBlockRules();
+ Optional<org.apache.iceberg.expressions.Expression> conflictFilter =
+ IcebergConflictDetectionFilterUtils.buildConflictDetectionFilter(
+ planner.getAnalyzedPlan(), icebergTable);
+
+ PhysicalSink<?> physicalSink = getPhysicalMergeSink(planner);
+ PlanFragment fragment = planner.getFragments().get(0);
+ DataSink dataSink = fragment.getSink();
+ boolean emptyInsert = childIsEmptyRelation(physicalSink);
+ String label = String.format("iceberg_merge_into_%x_%x", ctx.queryId().hi, ctx.queryId().lo);
+
+ IcebergMergeExecutor insertExecutor =
+ new IcebergMergeExecutor(ctx, icebergTable, label, planner, emptyInsert, -1L);
+ insertExecutor.setConflictDetectionFilter(conflictFilter);
+
+ if (insertExecutor.isEmptyInsert()) {
+ return true;
+ }
+
+ insertExecutor.beginTransaction();
+ insertExecutor.finalizeSinkForMerge(fragment, dataSink, physicalSink);
+ insertExecutor.getCoordinator().setTxnId(insertExecutor.getTxnId());
+ executor.setCoord(insertExecutor.getCoordinator());
+ insertExecutor.executeSingleInsert(executor);
+ return ctx.getState().getStateType() != QueryState.MysqlStateType.ERR;
+ }
+
+ private PhysicalSink<?> getPhysicalMergeSink(NereidsPlanner planner) {
+ Optional<PhysicalSink<?>> plan = planner.getPhysicalPlan()
+ .<PhysicalSink<?>>collect(PhysicalSink.class::isInstance).stream().findAny();
+ if (!plan.isPresent()) {
+ throw new AnalysisException("MERGE INTO command must contain target table");
+ }
+ PhysicalSink<?> sink = plan.get();
+ if (!(sink instanceof PhysicalIcebergMergeSink)) {
+ throw new AnalysisException("MERGE INTO plan must use Iceberg merge sink");
+ }
+ return sink;
+ }
+
+ private boolean childIsEmptyRelation(PhysicalSink<?> sink) {
+ return sink.children() != null && sink.children().size() == 1
+ && sink.child(0) instanceof PhysicalEmptyRelation;
+ }
+
+ private LogicalPlan injectRowIdColumn(LogicalPlan plan, IcebergExternalTable targetTable) {
+ if (IcebergNereidsUtils.hasUnboundPlan(plan)) {
+ return plan;
+ }
+ return IcebergNereidsUtils.injectRowIdColumn(plan, targetTable);
+ }
+
+ private Expression getTargetRowIdSlot() {
+ return new UnboundSlot(Column.ICEBERG_ROWID_COL);
+ }
+
+ private static Column getRowIdColumn(IcebergExternalTable table) {
+ return IcebergNereidsUtils.getRowIdColumn(table);
+ }
+
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergUpdateCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergUpdateCommand.java
new file mode 100644
index 00000000000..8e488862a82
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/IcebergUpdateCommand.java
@@ -0,0 +1,305 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.StmtType;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.iceberg.IcebergConflictDetectionFilterUtils;
+import org.apache.doris.datasource.iceberg.IcebergExternalDatabase;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
+import org.apache.doris.datasource.iceberg.IcebergMergeOperation;
+import org.apache.doris.datasource.iceberg.IcebergNereidsUtils;
+import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.nereids.analyzer.UnboundAlias;
+import org.apache.doris.nereids.analyzer.UnboundSlot;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.glue.LogicalPlanAdapter;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
+import org.apache.doris.nereids.trees.plans.Explainable;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext;
+import org.apache.doris.nereids.trees.plans.commands.insert.IcebergMergeExecutor;
+import org.apache.doris.nereids.trees.plans.logical.LogicalIcebergMergeSink;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergMergeSink;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalSink;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.RelationUtil;
+import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.planner.DataSink;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.QueryState;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * UPDATE command for Iceberg tables.
+ *
+ * UPDATE operations are implemented as a single scan + merge sink:
+ * 1. Scan rows matching WHERE condition with row_id injected
+ * 2. Project operation + row_id + updated columns
+ * 3. Merge sink writes position deletes and new data files
+ * 4. RowDelta commits delete + insert atomically
+ */
+public class IcebergUpdateCommand extends Command implements ForwardWithSync, Explainable {
+
+ private final List<EqualTo> assignments;
+ private final List<String> nameParts;
+ private final String tableAlias;
+ private final LogicalPlan logicalQuery;
+ private final DeleteCommandContext deleteCtx;
+
+ /**
+ * constructor
+ */
+ public IcebergUpdateCommand(
+ List<String> nameParts,
+ String tableAlias,
+ List<EqualTo> assignments,
+ LogicalPlan logicalQuery,
+ DeleteCommandContext deleteCtx) {
+ super(PlanType.UPDATE_COMMAND);
+ this.nameParts = Utils.copyRequiredList(nameParts);
+ this.assignments = Utils.copyRequiredList(assignments);
+ this.tableAlias = tableAlias;
+ this.logicalQuery = logicalQuery;
+ this.deleteCtx = deleteCtx != null ? deleteCtx : new DeleteCommandContext();
+ }
+
+ @Override
+ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+ // Check if target table is Iceberg table
+ List<String> qualifiedTableName = RelationUtil.getQualifierName(ctx, nameParts);
+ TableIf table = RelationUtil.getTable(qualifiedTableName, ctx.getEnv(), Optional.empty());
+
+ if (!(table instanceof IcebergExternalTable)) {
+ throw new AnalysisException("UPDATE command can only be used on Iceberg tables. "
+ + "Table " + Util.getTempTableDisplayName(table.getName()) + " is not an Iceberg table.");
+ }
+
+ IcebergExternalTable icebergTable = (IcebergExternalTable) table;
+
+ // Verify table format version (must be v2+ for update support)
+ // org.apache.iceberg.Table icebergTableObj = icebergTable.getIcebergTable();
+ // String formatVersionStr = icebergTableObj.properties().get("format-version");
+ // int formatVersion = formatVersionStr != null ? Integer.parseInt(formatVersionStr) : 1;
+ // if (formatVersion < 2) {
+ // throw new AnalysisException("Iceberg table UPDATE requires format version >= 2. "
+ // + "Current format version: " + formatVersion);
+ // }
+
+ long previousTargetTableId = ctx.getIcebergRowIdTargetTableId();
+ ctx.setIcebergRowIdTargetTableId(icebergTable.getId());
+ try {
+ // UPDATE is implemented as a single merge plan (delete + insert in one scan)
+ LogicalPlan mergePlan = buildMergePlan(ctx, logicalQuery, assignments, icebergTable);
+ executeMergePlan(ctx, executor, icebergTable, mergePlan);
+ } finally {
+ ctx.setIcebergRowIdTargetTableId(previousTargetTableId);
+ }
+ }
+
+ private boolean executeMergePlan(ConnectContext ctx, StmtExecutor executor,
+ IcebergExternalTable icebergTable,
+ LogicalPlan logicalPlan) throws Exception {
+ LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(logicalPlan, ctx.getStatementContext());
+ NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext());
+ planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift());
+ executor.setPlanner(planner);
+ executor.checkBlockRules();
+ Optional<org.apache.iceberg.expressions.Expression> conflictFilter =
+ IcebergConflictDetectionFilterUtils.buildConflictDetectionFilter(
+ planner.getAnalyzedPlan(), icebergTable);
+
+ PhysicalSink<?> physicalSink = getPhysicalMergeSink(planner);
+ PlanFragment fragment = planner.getFragments().get(0);
+ DataSink dataSink = fragment.getSink();
+ boolean emptyInsert = childIsEmptyRelation(physicalSink);
+ String label = String.format("iceberg_update_merge_%x_%x", ctx.queryId().hi, ctx.queryId().lo);
+
+ IcebergMergeExecutor insertExecutor =
+ new IcebergMergeExecutor(ctx, icebergTable, label, planner, emptyInsert, -1L);
+ insertExecutor.setConflictDetectionFilter(conflictFilter);
+
+ if (insertExecutor.isEmptyInsert()) {
+ return true;
+ }
+
+ insertExecutor.beginTransaction();
+ insertExecutor.finalizeSinkForMerge(fragment, dataSink, physicalSink);
+ insertExecutor.getCoordinator().setTxnId(insertExecutor.getTxnId());
+ executor.setCoord(insertExecutor.getCoordinator());
+ insertExecutor.executeSingleInsert(executor);
+ return ctx.getState().getStateType() != QueryState.MysqlStateType.ERR;
+ }
+
+ @VisibleForTesting
+ LogicalPlan buildMergeProjectPlan(ConnectContext ctx, LogicalPlan logicalQuery,
+ List<EqualTo> assignments, List<Column> columns, String tableName) {
+ Map<String, Expression> colNameToExpression = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
+ for (EqualTo equalTo : assignments) {
+ List<String> colNameParts = ((UnboundSlot) equalTo.left()).getNameParts();
+ UpdateCommand.checkAssignmentColumn(ctx, colNameParts, this.nameParts, this.tableAlias);
+ colNameToExpression.put(colNameParts.get(colNameParts.size() - 1), equalTo.right());
+ }
+ List<NamedExpression> updateColumns = buildUpdateSelectItems(colNameToExpression, columns, tableName);
+ LogicalPlan planWithRowId = IcebergNereidsUtils.injectRowIdColumn(logicalQuery);
+ NamedExpression rowIdColumn = getRowIdColumnExpr(planWithRowId);
+ NamedExpression operationColumn = new UnboundAlias(
+ new TinyIntLiteral(IcebergMergeOperation.UPDATE_OPERATION_NUMBER),
+ IcebergMergeOperation.OPERATION_COLUMN);
+
+ List<NamedExpression> projectItems = new ArrayList<>(2 + updateColumns.size());
+ projectItems.add(operationColumn);
+ projectItems.add(rowIdColumn);
+ projectItems.addAll(updateColumns);
+ return new LogicalProject<>(projectItems, planWithRowId);
+ }
+
+ private LogicalPlan buildMergePlan(ConnectContext ctx, LogicalPlan logicalQuery,
+ List<EqualTo> assignments, IcebergExternalTable icebergTable) {
+ String tableName = tableAlias != null
+ ? tableAlias
+ : Util.getTempTableDisplayName(icebergTable.getName());
+ LogicalPlan queryPlan = buildMergeProjectPlan(ctx, logicalQuery, assignments,
+ icebergTable.getBaseSchema(true), tableName);
+
+ List<NamedExpression> outputExprs;
+ if (!IcebergNereidsUtils.hasUnboundPlan(queryPlan)) {
+ outputExprs = queryPlan.getOutput().stream()
+ .map(NamedExpression.class::cast)
+ .collect(Collectors.toList());
+ } else if (queryPlan instanceof LogicalProject) {
+ outputExprs = ((LogicalProject<?>) queryPlan).getProjects();
+ } else {
+ outputExprs = ImmutableList.of();
+ }
+
+ return new LogicalIcebergMergeSink<>(
+ (IcebergExternalDatabase) icebergTable.getDatabase(),
+ icebergTable,
+ icebergTable.getBaseSchema(true),
+ outputExprs,
+ deleteCtx,
+ Optional.empty(),
+ Optional.empty(),
+ queryPlan);
+ }
+
+ private NamedExpression getRowIdColumnExpr(LogicalPlan planWithRowId) {
+ if (!IcebergNereidsUtils.hasUnboundPlan(planWithRowId)) {
+ Optional<Slot> rowIdSlot = IcebergNereidsUtils.findRowIdSlot(planWithRowId.getOutput());
+ if (rowIdSlot.isPresent()) {
+ return (NamedExpression) rowIdSlot.get();
+ }
+ }
+ return new UnboundSlot(Column.ICEBERG_ROWID_COL);
+ }
+
+ @VisibleForTesting
+ List<NamedExpression> buildUpdateSelectItems(Map<String, Expression> colNameToExpression,
+ List<Column> columns, String tableName) {
+ List<NamedExpression> selectItems = new ArrayList<>();
+ for (Column column : columns) {
+ if (!column.isVisible()) {
+ continue;
+ }
+ if (colNameToExpression.containsKey(column.getName())) {
+ Expression expr = colNameToExpression.get(column.getName());
+ selectItems.add(expr instanceof UnboundSlot
+ ? ((NamedExpression) expr)
+ : new UnboundAlias(expr));
+ colNameToExpression.remove(column.getName());
+ } else {
+ selectItems.add(new UnboundSlot(tableName, column.getName()));
+ }
+ }
+ if (!colNameToExpression.isEmpty()) {
+ throw new AnalysisException("unknown column in assignment list: "
+ + String.join(", ", colNameToExpression.keySet()));
+ }
+ return selectItems;
+ }
+
+ private PhysicalSink<?> getPhysicalMergeSink(NereidsPlanner planner) {
+ Optional<PhysicalSink<?>> plan = planner.getPhysicalPlan()
+ .<PhysicalSink<?>>collect(PhysicalSink.class::isInstance).stream().findAny();
+ if (!plan.isPresent()) {
+ throw new AnalysisException("UPDATE command must contain target table");
+ }
+ PhysicalSink<?> sink = plan.get();
+ if (!(sink instanceof PhysicalIcebergMergeSink)) {
+ throw new AnalysisException("UPDATE merge plan must use Iceberg merge sink");
+ }
+ return sink;
+ }
+
+ private boolean childIsEmptyRelation(PhysicalSink<?> sink) {
+ return sink.children() != null && sink.children().size() == 1
+ && sink.child(0) instanceof PhysicalEmptyRelation;
+ }
+
+ @Override
+ public Plan getExplainPlan(ConnectContext ctx) {
+ List<String> qualifiedTableName = RelationUtil.getQualifierName(ctx, nameParts);
+ TableIf table = RelationUtil.getTable(qualifiedTableName, ctx.getEnv(), Optional.empty());
+ if (!(table instanceof IcebergExternalTable)) {
+ throw new AnalysisException("Table must be IcebergExternalTable in UPDATE command");
+ }
+ long previousTargetTableId = ctx.getIcebergRowIdTargetTableId();
+ ctx.setIcebergRowIdTargetTableId(table.getId());
+ try {
+ return buildMergePlan(ctx, logicalQuery, assignments, (IcebergExternalTable) table);
+ } finally {
+ ctx.setIcebergRowIdTargetTableId(previousTargetTableId);
+ }
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitCommand(this, context);
+ }
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.UPDATE;
+ }
+
+ public DeleteCommandContext getDeleteCtx() {
+ return deleteCtx;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java
index 9e40034cef6..4e717bbb1d6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java
@@ -25,6 +25,7 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
import org.apache.doris.nereids.analyzer.UnboundAlias;
import org.apache.doris.nereids.analyzer.UnboundSlot;
import org.apache.doris.nereids.analyzer.UnboundTableSinkCreator;
@@ -37,6 +38,7 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Explainable;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext;
import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType;
import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
@@ -97,6 +99,26 @@ public class UpdateCommand extends Command implements ForwardWithSync, Explainab
@Override
public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+ // Check if target table is Iceberg table and route to IcebergUpdateCommand if so
+ List<String> qualifiedTableName = RelationUtil.getQualifierName(ctx, nameParts);
+ TableIf table = null;
+ try {
+ table = RelationUtil.getTable(qualifiedTableName, ctx.getEnv(), Optional.empty());
+ } catch (Exception e) {
+ // Table not found, will be handled by regular error flow
+ }
+
+ // Route to IcebergUpdateCommand for Iceberg tables
+ if (table instanceof IcebergExternalTable) {
+ DeleteCommandContext deleteCtx = new DeleteCommandContext();
+ deleteCtx.setDeleteFileType(DeleteCommandContext.DeleteFileType.POSITION_DELETE);
+ IcebergUpdateCommand icebergUpdateCommand = new IcebergUpdateCommand(
+ nameParts, tableAlias, assignments, logicalQuery,
+ deleteCtx);
+ icebergUpdateCommand.run(ctx, executor);
+ return;
+ }
+
// NOTE: update command is executed as insert command, so txn insert can support it
new InsertIntoTableCommand(completeQueryPlan(ctx, logicalQuery), Optional.empty(), Optional.empty(),
Optional.empty(), true, Optional.empty()).run(ctx, executor);
@@ -257,6 +279,15 @@ public class UpdateCommand extends Command implements ForwardWithSync, Explainab
@Override
public Plan getExplainPlan(ConnectContext ctx) {
+ List<String> qualifiedTableName = RelationUtil.getQualifierName(ctx, nameParts);
+ TableIf table = RelationUtil.getTable(qualifiedTableName, ctx.getEnv(), Optional.empty());
+ if (table instanceof IcebergExternalTable) {
+ DeleteCommandContext deleteCtx = new DeleteCommandContext();
+ deleteCtx.setDeleteFileType(DeleteCommandContext.DeleteFileType.POSITION_DELETE);
+ IcebergUpdateCommand icebergUpdateCommand = new IcebergUpdateCommand(
+ nameParts, tableAlias, assignments, logicalQuery, deleteCtx);
+ return icebergUpdateCommand.getExplainPlan(ctx);
+ }
return completeQueryPlan(ctx, logicalQuery);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/delete/DeleteCommandContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/delete/DeleteCommandContext.java
new file mode 100644
index 00000000000..6dd1a09d6ee
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/delete/DeleteCommandContext.java
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.delete;
+
+import org.apache.doris.thrift.TFileContent;
+
+/**
+ * Context for Iceberg delete operations.
+ * Stores information about delete file type.
+ */
+public class DeleteCommandContext {
+ /**
+ * Type of delete file to generate
+ */
+ public enum DeleteFileType {
+ /**
+ * Position delete: delete by file path and row position
+ */
+ POSITION_DELETE
+ }
+
+ private DeleteFileType deleteFileType = DeleteFileType.POSITION_DELETE;
+
+ public DeleteFileType getDeleteFileType() {
+ return deleteFileType;
+ }
+
+ public void setDeleteFileType(DeleteFileType deleteFileType) {
+ this.deleteFileType = deleteFileType;
+ }
+
+ /**
+ * Convert to Thrift file content type
+ */
+ public TFileContent toTFileContent() {
+ return TFileContent.POSITION_DELETES;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutor.java
new file mode 100644
index 00000000000..c761993583a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergDeleteExecutor.java
@@ -0,0 +1,105 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.insert;
+
+import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
+import org.apache.doris.datasource.iceberg.IcebergTransaction;
+import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalSink;
+import org.apache.doris.planner.DataSink;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.transaction.TransactionType;
+
+import org.apache.iceberg.expressions.Expression;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.Optional;
+
+/**
+ * Executor for Iceberg DELETE operations.
+ *
+ * DELETE is implemented by generating Position Delete files
+ * instead of rewriting data files.
+ *
+ * Flow:
+ * 1. Execute query to get rows matching WHERE clause (with $row_id column)
+ * 2. BE writes position delete files and returns TIcebergCommitData
+ * 3. FE commits DeleteFiles to Iceberg table using RowDelta API
+ */
+public class IcebergDeleteExecutor extends BaseExternalTableInsertExecutor {
+ private static final Logger LOG = LogManager.getLogger(IcebergDeleteExecutor.class);
+ private Optional<Expression> conflictDetectionFilter = Optional.empty();
+
+ public IcebergDeleteExecutor(ConnectContext ctx, IcebergExternalTable table,
+ String labelName, NereidsPlanner planner,
+ boolean emptyInsert, long jobId) {
+ // BaseExternalTableInsertExecutor requires Optional<InsertCommandContext>
+ // For DELETE operations, we pass Optional.empty().
+ super(ctx, table, labelName, planner, Optional.empty(), emptyInsert, jobId);
+ }
+
+ public void finalizeSinkForDelete(PlanFragment fragment, DataSink sink, PhysicalSink<?> physicalSink) {
+ super.finalizeSink(fragment, sink, physicalSink);
+ }
+
+ public void setConflictDetectionFilter(Optional<Expression> filter) {
+ conflictDetectionFilter = filter == null ? Optional.empty() : filter;
+ }
+
+ @Override
+ protected void beforeExec() throws UserException {
+ IcebergTransaction transaction = (IcebergTransaction) transactionManager.getTransaction(txnId);
+ transaction.beginDelete((IcebergExternalTable) table);
+ if (conflictDetectionFilter.isPresent()) {
+ transaction.setConflictDetectionFilter(conflictDetectionFilter.get());
+ } else {
+ transaction.clearConflictDetectionFilter();
+ }
+ }
+
+ @Override
+ protected void doBeforeCommit() throws UserException {
+ IcebergExternalTable dorisTable = (IcebergExternalTable) table;
+ IcebergTransaction transaction = (IcebergTransaction) transactionManager.getTransaction(txnId);
+
+ // Position delete files are written by BE and returned as TIcebergCommitData.
+ // FE only needs to use commitDataList to update loaded rows and commit RowDelta.
+ LOG.info("Processing Position Delete commit data for table: {}", dorisTable.getName());
+
+ this.loadedRows = transaction.getUpdateCnt();
+
+ // Finish delete and commit
+ org.apache.doris.datasource.NameMapping nameMapping =
+ new org.apache.doris.datasource.NameMapping(
+ dorisTable.getCatalog().getId(),
+ dorisTable.getDbName(),
+ dorisTable.getName(),
+ dorisTable.getRemoteDbName(),
+ dorisTable.getRemoteName());
+
+ transaction.finishDelete(nameMapping);
+ }
+
+ @Override
+ protected TransactionType transactionType() {
+ return TransactionType.ICEBERG;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutor.java
new file mode 100644
index 00000000000..be82a2f82fb
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergMergeExecutor.java
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.insert;
+
+import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.NameMapping;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
+import org.apache.doris.datasource.iceberg.IcebergTransaction;
+import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalSink;
+import org.apache.doris.planner.DataSink;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.transaction.TransactionType;
+
+import org.apache.iceberg.expressions.Expression;
+
+import java.util.Optional;
+
+/**
+ * Executor for Iceberg UPDATE merge operations (single scan + merge sink).
+ */
+public class IcebergMergeExecutor extends BaseExternalTableInsertExecutor {
+ private Optional<Expression> conflictDetectionFilter = Optional.empty();
+
+ public IcebergMergeExecutor(ConnectContext ctx, IcebergExternalTable table,
+ String labelName, NereidsPlanner planner,
+ boolean emptyInsert, long jobId) {
+ super(ctx, table, labelName, planner, Optional.empty(), emptyInsert, jobId);
+ }
+
+ public void finalizeSinkForMerge(PlanFragment fragment, DataSink sink, PhysicalSink<?> physicalSink) {
+ super.finalizeSink(fragment, sink, physicalSink);
+ }
+
+ public void setConflictDetectionFilter(Optional<Expression> filter) {
+ conflictDetectionFilter = filter == null ? Optional.empty() : filter;
+ }
+
+ @Override
+ protected void beforeExec() throws UserException {
+ IcebergTransaction transaction = (IcebergTransaction) transactionManager.getTransaction(txnId);
+ transaction.beginMerge((IcebergExternalTable) table);
+ if (conflictDetectionFilter.isPresent()) {
+ transaction.setConflictDetectionFilter(conflictDetectionFilter.get());
+ } else {
+ transaction.clearConflictDetectionFilter();
+ }
+ }
+
+ @Override
+ protected void doBeforeCommit() throws UserException {
+ IcebergExternalTable dorisTable = (IcebergExternalTable) table;
+ IcebergTransaction transaction = (IcebergTransaction) transactionManager.getTransaction(txnId);
+ this.loadedRows = transaction.getUpdateCnt();
+
+ NameMapping nameMapping = new NameMapping(
+ dorisTable.getCatalog().getId(),
+ dorisTable.getDbName(),
+ dorisTable.getName(),
+ dorisTable.getRemoteDbName(),
+ dorisTable.getRemoteName());
+ transaction.finishMerge(nameMapping);
+ }
+
+ @Override
+ protected TransactionType transactionType() {
+ return TransactionType.ICEBERG;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/merge/MergeIntoCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/merge/MergeIntoCommand.java
index 9b562492c85..4a68f739651 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/merge/MergeIntoCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/merge/MergeIntoCommand.java
@@ -23,6 +23,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.Type;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
import org.apache.doris.nereids.analyzer.UnboundAlias;
import org.apache.doris.nereids.analyzer.UnboundRelation;
import org.apache.doris.nereids.analyzer.UnboundSlot;
@@ -52,6 +53,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.Command;
import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync;
+import org.apache.doris.nereids.trees.plans.commands.IcebergMergeCommand;
import org.apache.doris.nereids.trees.plans.commands.UpdateCommand;
import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType;
import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
@@ -121,6 +123,12 @@ public class MergeIntoCommand extends Command implements ForwardWithSync, Explai
@Override
public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+ TableIf table = getTargetTableIf(ctx);
+ if (table instanceof IcebergExternalTable) {
+ new IcebergMergeCommand(targetNameParts, targetAlias, cte,
+ source, onClause, matchedClauses, notMatchedClauses).run(ctx, executor);
+ return;
+ }
new InsertIntoTableCommand(completeQueryPlan(ctx), Optional.empty(), Optional.empty(),
Optional.empty(), true, Optional.empty()).run(ctx, executor);
}
@@ -132,12 +140,21 @@ public class MergeIntoCommand extends Command implements ForwardWithSync, Explai
@Override
public Plan getExplainPlan(ConnectContext ctx) {
+ TableIf table = getTargetTableIf(ctx);
+ if (table instanceof IcebergExternalTable) {
+ return new IcebergMergeCommand(targetNameParts, targetAlias, cte,
+ source, onClause, matchedClauses, notMatchedClauses).getExplainPlan(ctx);
+ }
return completeQueryPlan(ctx);
}
- private OlapTable getTargetTable(ConnectContext ctx) {
+ private TableIf getTargetTableIf(ConnectContext ctx) {
List<String> qualifiedTableName = RelationUtil.getQualifierName(ctx, targetNameParts);
- TableIf table = RelationUtil.getTable(qualifiedTableName, ctx.getEnv(), Optional.empty());
+ return RelationUtil.getTable(qualifiedTableName, ctx.getEnv(), Optional.empty());
+ }
+
+ private OlapTable getTargetTable(ConnectContext ctx) {
+ TableIf table = getTargetTableIf(ctx);
if (!(table instanceof OlapTable) || !((OlapTable) table).getEnableUniqueKeyMergeOnWrite()) {
throw new AnalysisException("merge into command only support MOW unique key olapTable");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergDeleteSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergDeleteSink.java
new file mode 100644
index 00000000000..17904c330dc
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergDeleteSink.java
@@ -0,0 +1,151 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.logical;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.datasource.iceberg.IcebergExternalDatabase;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.PropagateFuncDeps;
+import org.apache.doris.nereids.trees.plans.algebra.Sink;
+import org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.Utils;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Logical Iceberg Delete Sink for DELETE operations.
+ * This sink is responsible for writing position delete files.
+ */
+public class LogicalIcebergDeleteSink<CHILD_TYPE extends Plan> extends LogicalTableSink<CHILD_TYPE>
+ implements Sink, PropagateFuncDeps {
+ private final IcebergExternalDatabase database;
+ private final IcebergExternalTable targetTable;
+ private final DeleteCommandContext deleteContext;
+
+ /**
+ * Constructor
+ */
+ public LogicalIcebergDeleteSink(IcebergExternalDatabase database,
+ IcebergExternalTable targetTable,
+ List<Column> cols,
+ List<NamedExpression> outputExprs,
+ DeleteCommandContext deleteContext,
+ Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties,
+ CHILD_TYPE child) {
+ super(PlanType.LOGICAL_ICEBERG_DELETE_SINK, outputExprs, groupExpression, logicalProperties, cols, child);
+ this.database = Objects.requireNonNull(database, "database != null in LogicalIcebergDeleteSink");
+ this.targetTable = Objects.requireNonNull(targetTable, "targetTable != null in LogicalIcebergDeleteSink");
+ this.deleteContext = Objects.requireNonNull(deleteContext, "deleteContext != null in LogicalIcebergDeleteSink");
+ }
+
+ public Plan withChildAndUpdateOutput(Plan child) {
+ List<NamedExpression> output = child.getOutput().stream()
+ .map(NamedExpression.class::cast)
+ .collect(ImmutableList.toImmutableList());
+ return new LogicalIcebergDeleteSink<>(database, targetTable, cols, output,
+ deleteContext, Optional.empty(), Optional.empty(), child);
+ }
+
+ @Override
+ public Plan withChildren(List<Plan> children) {
+ Preconditions.checkArgument(children.size() == 1, "LogicalIcebergDeleteSink only accepts one child");
+ return new LogicalIcebergDeleteSink<>(database, targetTable, cols, outputExprs,
+ deleteContext, Optional.empty(), Optional.empty(), children.get(0));
+ }
+
+ public LogicalIcebergDeleteSink<CHILD_TYPE> withOutputExprs(List<NamedExpression> outputExprs) {
+ return new LogicalIcebergDeleteSink<>(database, targetTable, cols, outputExprs,
+ deleteContext, Optional.empty(), Optional.empty(), child());
+ }
+
+ public IcebergExternalDatabase getDatabase() {
+ return database;
+ }
+
+ public IcebergExternalTable getTargetTable() {
+ return targetTable;
+ }
+
+ public DeleteCommandContext getDeleteContext() {
+ return deleteContext;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ LogicalIcebergDeleteSink<?> that = (LogicalIcebergDeleteSink<?>) o;
+ return Objects.equals(database, that.database)
+ && Objects.equals(targetTable, that.targetTable)
+ && Objects.equals(deleteContext, that.deleteContext)
+ && Objects.equals(cols, that.cols);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), database, targetTable, cols, deleteContext);
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toSqlString("LogicalIcebergDeleteSink[" + id.asInt() + "]",
+ "outputExprs", outputExprs,
+ "database", database.getFullName(),
+ "targetTable", targetTable.getName(),
+ "cols", cols,
+ "deleteFileType", deleteContext.getDeleteFileType()
+ );
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitLogicalIcebergDeleteSink(this, context);
+ }
+
+ @Override
+ public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
+ return new LogicalIcebergDeleteSink<>(database, targetTable, cols, outputExprs,
+ deleteContext, groupExpression, Optional.of(getLogicalProperties()), child());
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan> children) {
+ return new LogicalIcebergDeleteSink<>(database, targetTable, cols, outputExprs,
+ deleteContext, groupExpression, logicalProperties, children.get(0));
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergMergeSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergMergeSink.java
new file mode 100644
index 00000000000..7f528020890
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergMergeSink.java
@@ -0,0 +1,150 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.logical;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.datasource.iceberg.IcebergExternalDatabase;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.PropagateFuncDeps;
+import org.apache.doris.nereids.trees.plans.algebra.Sink;
+import org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.Utils;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Logical Iceberg Merge Sink for UPDATE operations.
+ * This sink is responsible for routing rows to position delete and data insert.
+ */
+public class LogicalIcebergMergeSink<CHILD_TYPE extends Plan> extends LogicalTableSink<CHILD_TYPE>
+ implements Sink, PropagateFuncDeps {
+ private final IcebergExternalDatabase database;
+ private final IcebergExternalTable targetTable;
+ private final DeleteCommandContext deleteContext;
+
+ /**
+ * Constructor
+ */
+ public LogicalIcebergMergeSink(IcebergExternalDatabase database,
+ IcebergExternalTable targetTable,
+ List<Column> cols,
+ List<NamedExpression> outputExprs,
+ DeleteCommandContext deleteContext,
+ Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties,
+ CHILD_TYPE child) {
+ super(PlanType.LOGICAL_ICEBERG_MERGE_SINK, outputExprs, groupExpression, logicalProperties, cols, child);
+ this.database = Objects.requireNonNull(database, "database != null in LogicalIcebergMergeSink");
+ this.targetTable = Objects.requireNonNull(targetTable, "targetTable != null in LogicalIcebergMergeSink");
+ this.deleteContext = Objects.requireNonNull(deleteContext, "deleteContext != null in LogicalIcebergMergeSink");
+ }
+
+ public Plan withChildAndUpdateOutput(Plan child) {
+ List<NamedExpression> output = child.getOutput().stream()
+ .map(NamedExpression.class::cast)
+ .collect(ImmutableList.toImmutableList());
+ return new LogicalIcebergMergeSink<>(database, targetTable, cols, output,
+ deleteContext, Optional.empty(), Optional.empty(), child);
+ }
+
+ @Override
+ public Plan withChildren(List<Plan> children) {
+ Preconditions.checkArgument(children.size() == 1, "LogicalIcebergMergeSink only accepts one child");
+ return new LogicalIcebergMergeSink<>(database, targetTable, cols, outputExprs,
+ deleteContext, Optional.empty(), Optional.empty(), children.get(0));
+ }
+
+ public LogicalIcebergMergeSink<CHILD_TYPE> withOutputExprs(List<NamedExpression> outputExprs) {
+ return new LogicalIcebergMergeSink<>(database, targetTable, cols, outputExprs,
+ deleteContext, Optional.empty(), Optional.empty(), child());
+ }
+
+ public IcebergExternalDatabase getDatabase() {
+ return database;
+ }
+
+ public IcebergExternalTable getTargetTable() {
+ return targetTable;
+ }
+
+ public DeleteCommandContext getDeleteContext() {
+ return deleteContext;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ LogicalIcebergMergeSink<?> that = (LogicalIcebergMergeSink<?>) o;
+ return Objects.equals(database, that.database)
+ && Objects.equals(targetTable, that.targetTable)
+ && Objects.equals(deleteContext, that.deleteContext)
+ && Objects.equals(cols, that.cols);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), database, targetTable, cols, deleteContext);
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toSqlString("LogicalIcebergMergeSink[" + id.asInt() + "]",
+ "outputExprs", outputExprs,
+ "database", database.getFullName(),
+ "targetTable", targetTable.getName(),
+ "cols", cols,
+ "deleteFileType", deleteContext.getDeleteFileType());
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitLogicalIcebergMergeSink(this, context);
+ }
+
+ @Override
+ public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
+ return new LogicalIcebergMergeSink<>(database, targetTable, cols, outputExprs,
+ deleteContext, groupExpression, Optional.of(getLogicalProperties()), child());
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan> children) {
+ return new LogicalIcebergMergeSink<>(database, targetTable, cols, outputExprs,
+ deleteContext, groupExpression, logicalProperties, children.get(0));
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergDeleteSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergDeleteSink.java
new file mode 100644
index 00000000000..67192dbb4fb
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergDeleteSink.java
@@ -0,0 +1,175 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.physical;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.datasource.iceberg.IcebergExternalDatabase;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
+import org.apache.doris.datasource.iceberg.IcebergMergeOperation;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType;
+import org.apache.doris.nereids.properties.DistributionSpecMerge;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.ExprId;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.statistics.Statistics;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Physical Iceberg Delete Sink for DELETE operations.
+ * This sink is responsible for writing position delete files.
+ */
+public class PhysicalIcebergDeleteSink<CHILD_TYPE extends Plan> extends PhysicalBaseExternalTableSink<CHILD_TYPE> {
+ private final DeleteCommandContext deleteContext;
+
+ /**
+ * Constructor
+ */
+ public PhysicalIcebergDeleteSink(IcebergExternalDatabase database,
+ IcebergExternalTable targetTable,
+ List<Column> cols,
+ List<NamedExpression> outputExprs,
+ DeleteCommandContext deleteContext,
+ Optional<GroupExpression> groupExpression,
+ LogicalProperties logicalProperties,
+ CHILD_TYPE child) {
+ this(database, targetTable, cols, outputExprs, deleteContext, groupExpression, logicalProperties,
+ PhysicalProperties.GATHER, null, child);
+ }
+
+ /**
+ * Constructor
+ */
+ public PhysicalIcebergDeleteSink(IcebergExternalDatabase database,
+ IcebergExternalTable targetTable,
+ List<Column> cols,
+ List<NamedExpression> outputExprs,
+ DeleteCommandContext deleteContext,
+ Optional<GroupExpression> groupExpression,
+ LogicalProperties logicalProperties,
+ PhysicalProperties physicalProperties,
+ Statistics statistics,
+ CHILD_TYPE child) {
+ super(PlanType.PHYSICAL_ICEBERG_DELETE_SINK, database, targetTable, cols, outputExprs, groupExpression,
+ logicalProperties, physicalProperties, statistics, child);
+ this.deleteContext = Objects.requireNonNull(
+ deleteContext, "deleteContext != null in PhysicalIcebergDeleteSink");
+ }
+
+ public DeleteCommandContext getDeleteContext() {
+ return deleteContext;
+ }
+
+ @Override
+ public Plan withChildren(List<Plan> children) {
+ return new PhysicalIcebergDeleteSink<>(
+ (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable,
+ cols, outputExprs, deleteContext, groupExpression,
+ getLogicalProperties(), physicalProperties, statistics, children.get(0));
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitPhysicalIcebergDeleteSink(this, context);
+ }
+
+ @Override
+ public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
+ return new PhysicalIcebergDeleteSink<>(
+ (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, cols, outputExprs,
+ deleteContext, groupExpression, getLogicalProperties(), child());
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan> children) {
+ return new PhysicalIcebergDeleteSink<>(
+ (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, cols, outputExprs,
+ deleteContext, groupExpression, logicalProperties.get(), children.get(0));
+ }
+
+ @Override
+ public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) {
+ return new PhysicalIcebergDeleteSink<>(
+ (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, cols, outputExprs,
+ deleteContext, groupExpression, getLogicalProperties(), physicalProperties, statistics, child());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ PhysicalIcebergDeleteSink<?> that = (PhysicalIcebergDeleteSink<?>) o;
+ return Objects.equals(deleteContext, that.deleteContext);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), deleteContext);
+ }
+
+ /**
+ * Get output physical properties.
+ */
+ @Override
+ public PhysicalProperties getRequirePhysicalProperties() {
+ ExprId rowIdExprId = null;
+ ExprId operationExprId = null;
+ for (Slot slot : child().getOutput()) {
+ String name = slot.getName();
+ if (operationExprId == null && IcebergMergeOperation.OPERATION_COLUMN.equalsIgnoreCase(name)) {
+ operationExprId = slot.getExprId();
+ }
+ if (rowIdExprId == null && Column.ICEBERG_ROWID_COL.equalsIgnoreCase(name)) {
+ rowIdExprId = slot.getExprId();
+ }
+ }
+
+ if (rowIdExprId != null && operationExprId != null) {
+ return new PhysicalProperties(new DistributionSpecMerge(
+ operationExprId,
+ ImmutableList.of(),
+ ImmutableList.of(rowIdExprId),
+ true,
+ ImmutableList.of(),
+ null));
+ }
+ if (rowIdExprId != null) {
+ return PhysicalProperties.createHash(ImmutableList.of(rowIdExprId), ShuffleType.REQUIRE);
+ }
+ return PhysicalProperties.GATHER;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergMergeSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergMergeSink.java
new file mode 100644
index 00000000000..0281ad23243
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergMergeSink.java
@@ -0,0 +1,338 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.physical;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.datasource.iceberg.IcebergExternalDatabase;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
+import org.apache.doris.datasource.iceberg.IcebergMergeOperation;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType;
+import org.apache.doris.nereids.properties.DistributionSpecMerge;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.ExprId;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.delete.DeleteCommandContext;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.statistics.Statistics;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.iceberg.PartitionField;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.types.Types;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.TreeMap;
+
+/**
+ * Physical Iceberg Merge Sink for UPDATE operations.
+ * This sink is responsible for writing position delete files and data files.
+ */
+public class PhysicalIcebergMergeSink<CHILD_TYPE extends Plan> extends PhysicalBaseExternalTableSink<CHILD_TYPE> {
+ private final DeleteCommandContext deleteContext;
+
+ /**
+ * Constructor
+ */
+ public PhysicalIcebergMergeSink(IcebergExternalDatabase database,
+ IcebergExternalTable targetTable,
+ List<Column> cols,
+ List<NamedExpression> outputExprs,
+ DeleteCommandContext deleteContext,
+ Optional<GroupExpression> groupExpression,
+ LogicalProperties logicalProperties,
+ CHILD_TYPE child) {
+ this(database, targetTable, cols, outputExprs, deleteContext, groupExpression, logicalProperties,
+ PhysicalProperties.GATHER, null, child);
+ }
+
+ /**
+ * Constructor
+ */
+ public PhysicalIcebergMergeSink(IcebergExternalDatabase database,
+ IcebergExternalTable targetTable,
+ List<Column> cols,
+ List<NamedExpression> outputExprs,
+ DeleteCommandContext deleteContext,
+ Optional<GroupExpression> groupExpression,
+ LogicalProperties logicalProperties,
+ PhysicalProperties physicalProperties,
+ Statistics statistics,
+ CHILD_TYPE child) {
+ super(PlanType.PHYSICAL_ICEBERG_MERGE_SINK, database, targetTable, cols, outputExprs, groupExpression,
+ logicalProperties, physicalProperties, statistics, child);
+ this.deleteContext = Objects.requireNonNull(
+ deleteContext, "deleteContext != null in PhysicalIcebergMergeSink");
+ }
+
... 4767 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org