You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2026/03/10 04:25:20 UTC
(doris) branch tmp-branch-2.1.11-rc01-paimon-cpp updated: [feat](paimon) backport paimon-cpp reader integration chain for 2.1.11-rc01 (#61125)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch tmp-branch-2.1.11-rc01-paimon-cpp
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/tmp-branch-2.1.11-rc01-paimon-cpp by this push:
new aa776d0fec0 [feat](paimon) backport paimon-cpp reader integration chain for 2.1.11-rc01 (#61125)
aa776d0fec0 is described below
commit aa776d0fec0c7d336b31ee2c5d30fc77f31be7e0
Author: Chenjunwei <13...@users.noreply.github.com>
AuthorDate: Tue Mar 10 12:25:13 2026 +0800
[feat](paimon) backport paimon-cpp reader integration chain for 2.1.11-rc01 (#61125)
## Summary
Backport paimon-cpp reader integration chain into
tmp-branch-2.1.11-rc01-paimon-cpp as a single squashed change.
Included upstream PRs:
- #60296
- #60676
- #60711
- #60730
- #60795
- #60883
- #60946
## Notes
- This PR is intentionally squashed into one commit for release-branch
delivery.
- Local uncommitted changes in the original workspace were not touched;
work was done in an isolated worktree.
---
.github/workflows/build-thirdparty.yml | 43 +-
be/CMakeLists.txt | 44 ++
be/cmake/thirdparty.cmake | 21 +-
be/src/vec/exec/format/table/paimon_cpp_reader.cpp | 324 +++++++++
be/src/vec/exec/format/table/paimon_cpp_reader.h | 93 +++
.../exec/format/table/paimon_doris_file_system.cpp | 721 +++++++++++++++++++++
.../exec/format/table/paimon_doris_file_system.h | 25 +
.../format/table/paimon_predicate_converter.cpp | 654 +++++++++++++++++++
.../exec/format/table/paimon_predicate_converter.h | 88 +++
be/src/vec/exec/scan/vfile_scanner.cpp | 26 +-
.../exec/format/table/paimon_cpp_reader_test.cpp | 95 +++
.../java/org/apache/doris/paimon/PaimonUtils.java | 14 +-
.../datasource/paimon/source/PaimonScanNode.java | 28 +-
.../datasource/paimon/source/PaimonSource.java | 9 +
.../java/org/apache/doris/qe/SessionVariable.java | 24 +-
gensrc/thrift/PaloInternalService.thrift | 2 +
.../paimon/test_paimon_cpp_reader.groovy | 70 ++
thirdparty/build-thirdparty.sh | 227 +++++++
thirdparty/download-thirdparty.sh | 137 ++++
thirdparty/paimon-cpp-cache.cmake | 136 ++++
.../patches/apache-arrow-17.0.0-paimon.patch | 224 +++++++
.../paimon-cpp-buildutils-static-deps.patch | 331 ++++++++++
thirdparty/vars.sh | 25 +-
23 files changed, 3330 insertions(+), 31 deletions(-)
diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml
index caca2ece6a6..6258be8554f 100644
--- a/.github/workflows/build-thirdparty.yml
+++ b/.github/workflows/build-thirdparty.yml
@@ -64,16 +64,17 @@ jobs:
run: |
git clone -b v7 https://github.com/easimon/maximize-build-space
- - name: Maximize build space
- uses: ./maximize-build-space
- with:
- root-reserve-mb: 4096
- swap-size-mb: 8192
- remove-dotnet: 'true'
- remove-android: 'true'
- remove-haskell: 'true'
- remove-codeql: 'true'
- remove-docker-images: 'true'
+ #- name: Maximize build space
+ # uses: ./maximize-build-space
+ # with:
+ # root-reserve-mb: 4096
+ # temp-reserve-mb: 4096
+ # swap-size-mb: 8192
+ # remove-dotnet: 'true'
+ # remove-android: 'true'
+ # remove-haskell: 'true'
+ # remove-codeql: 'true'
+ # remove-docker-images: 'true'
- name: Checkout ${{ github.ref }}
uses: actions/checkout@v3
@@ -131,8 +132,18 @@ jobs:
export JAVA_HOME="$(find /usr/lib/jvm/java-8-openjdk* -maxdepth 0)"
export DORIS_TOOLCHAIN=gcc
+ #export TMPDIR=/home/runner/work/doris/doris/.tmp
+ #export TMP=$TMPDIR
+ #export TEMP=$TMPDIR
+ #mkdir -p $TMPDIR
+
+ df -h
+ echo $TMPDIR
+ echo $RUNNER_TEMP
+
cd thirdparty
- ./build-thirdparty.sh -j "$(nproc)"
+ #./build-thirdparty.sh -j "$(nproc)"
+ ./build-thirdparty.sh -j 2
build_macos:
name: Build Third Party Libraries (macOS)
@@ -180,7 +191,13 @@ jobs:
- name: Build
run: |
export MACOSX_DEPLOYMENT_TARGET=12.0
+ export CMAKE_POLICY_VERSION_MINIMUM=3.10
+ export CUSTOM_CMAKE="/usr/local/bin/cmake"
- cd thirdparty
- ./build-thirdparty.sh -j "$(nproc)"
+ df -h
+ echo $TMPDIR
+ echo $RUNNER_TEMP
+ cd thirdparty
+ #./build-thirdparty.sh -j "$(nproc)"
+ ./build-thirdparty.sh -j 2
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 11a416158a8..23edf356d7d 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -135,6 +135,22 @@ option(WITH_MYSQL "Support access MySQL" ON)
option(BUILD_FS_BENCHMARK "ON for building fs benchmark tool or OFF for not" OFF)
message(STATUS "build fs benchmark tool: ${BUILD_FS_BENCHMARK}")
+option(BUILD_TASK_EXECUTOR_SIMULATOR "ON for building task executor simulator or OFF for not" OFF)
+message(STATUS "build task executor simulator: ${BUILD_TASK_EXECUTOR_SIMULATOR}")
+
+option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file cache lru tool or OFF for not" OFF)
+message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}")
+
+option(ENABLE_PAIMON_CPP "Enable Paimon C++ integration" ON)
+set(PAIMON_HOME "" CACHE PATH "Paimon install prefix")
+
+# Allow env to override when reconfiguring (avoid picking /usr/local).
+if (DEFINED ENV{ENABLE_PAIMON_CPP})
+ set(ENABLE_PAIMON_CPP "$ENV{ENABLE_PAIMON_CPP}" CACHE BOOL "" FORCE)
+endif()
+if (DEFINED ENV{PAIMON_HOME} AND NOT PAIMON_HOME)
+ set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "" FORCE)
+endif()
set(CMAKE_SKIP_RPATH TRUE)
set(Boost_USE_STATIC_LIBS ON)
set(Boost_USE_STATIC_RUNTIME ON)
@@ -495,6 +511,10 @@ set(COMMON_THIRDPARTY
${COMMON_THIRDPARTY}
)
+if (ENABLE_PAIMON_CPP)
+ message(STATUS "Paimon C++ enabled: legacy thirdparty static linkage mode")
+endif()
+
if ((ARCH_AMD64 OR ARCH_AARCH64) AND OS_LINUX)
add_library(hadoop_hdfs STATIC IMPORTED)
set_target_properties(hadoop_hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/hadoop_hdfs/native/libhdfs.a)
@@ -548,6 +568,14 @@ if (MAKE_TEST)
)
endif()
+set(PAIMON_FACTORY_REGISTRY_LIBS)
+if (ENABLE_PAIMON_CPP)
+ # Doris Arrow is built with COMPUTE/DATASET/ACERO/FILESYSTEM enabled for
+ # this branch, so paimon-cpp should reuse Doris's single Arrow stack.
+ set(PAIMON_FACTORY_REGISTRY_LIBS paimon_parquet_file_format)
+ list(REMOVE_ITEM COMMON_THIRDPARTY ${PAIMON_FACTORY_REGISTRY_LIBS})
+endif()
+
set(DORIS_DEPENDENCIES
${DORIS_DEPENDENCIES}
${WL_START_GROUP}
@@ -569,6 +597,22 @@ set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-core-static)
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-shared-static)
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-contribs-lib)
+if (ENABLE_PAIMON_CPP)
+ if (PAIMON_FACTORY_REGISTRY_LIBS)
+ set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES}
+ -Wl,--whole-archive
+ ${PAIMON_FACTORY_REGISTRY_LIBS}
+ -Wl,--no-whole-archive)
+ endif()
+
+ # paimon-cpp internal dependencies (renamed with _paimon suffix)
+ # These must come after paimon libraries to resolve symbols.
+ set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} roaring_bitmap_paimon)
+ set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} xxhash_paimon)
+ set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} fmt_paimon)
+ set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} tbb_paimon)
+endif()
+
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_END_GROUP})
# Add all external dependencies. They should come after the palo libs.
diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake
index 886ca5a240d..8aac51bfdf0 100644
--- a/be/cmake/thirdparty.cmake
+++ b/be/cmake/thirdparty.cmake
@@ -104,6 +104,8 @@ add_thirdparty(zstd LIB64)
add_thirdparty(arrow LIB64)
add_thirdparty(arrow_flight LIB64)
add_thirdparty(arrow_flight_sql LIB64)
+add_thirdparty(arrow_dataset LIB64)
+add_thirdparty(arrow_acero LIB64)
add_thirdparty(parquet LIB64)
add_thirdparty(brpc LIB64)
add_thirdparty(rocksdb)
@@ -164,4 +166,21 @@ endif()
add_thirdparty(icuuc LIB64)
add_thirdparty(icui18n LIB64)
-add_thirdparty(icudata LIB64)
\ No newline at end of file
+add_thirdparty(icudata LIB64)
+
+add_thirdparty(pugixml LIB64)
+
+if (ENABLE_PAIMON_CPP)
+ add_thirdparty(paimon LIB64)
+ add_thirdparty(paimon_parquet_file_format LIB64)
+ add_thirdparty(paimon_orc_file_format LIB64)
+ add_thirdparty(paimon_blob_file_format LIB64)
+ add_thirdparty(paimon_local_file_system LIB64)
+ add_thirdparty(paimon_file_index LIB64)
+ add_thirdparty(paimon_global_index LIB64)
+
+ add_thirdparty(roaring_bitmap_paimon LIB64)
+ add_thirdparty(xxhash_paimon LIB64)
+ add_thirdparty(fmt_paimon LIB64)
+ add_thirdparty(tbb_paimon LIB64)
+endif()
diff --git a/be/src/vec/exec/format/table/paimon_cpp_reader.cpp b/be/src/vec/exec/format/table/paimon_cpp_reader.cpp
new file mode 100644
index 00000000000..8b7f6e107a6
--- /dev/null
+++ b/be/src/vec/exec/format/table/paimon_cpp_reader.cpp
@@ -0,0 +1,324 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "paimon_cpp_reader.h"
+
+#include <algorithm>
+#include <mutex>
+#include <utility>
+
+#include "arrow/c/bridge.h"
+#include "arrow/record_batch.h"
+#include "arrow/result.h"
+#include "paimon/defs.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/read_context.h"
+#include "paimon/table/source/table_read.h"
+#include "runtime/descriptors.h"
+#include "runtime/runtime_state.h"
+#include "util/url_coding.h"
+#include "vec/core/block.h"
+#include "vec/core/column_with_type_and_name.h"
+#include "vec/exec/format/table/paimon_doris_file_system.h"
+
+namespace doris::vectorized {
+
+namespace {
+constexpr const char* VALUE_KIND_FIELD = "_VALUE_KIND";
+
+} // namespace
+
+PaimonCppReader::PaimonCppReader(const std::vector<SlotDescriptor*>& file_slot_descs,
+ RuntimeState* state, RuntimeProfile* profile,
+ const TFileRangeDesc& range,
+ const TFileScanRangeParams* range_params)
+ : _file_slot_descs(file_slot_descs),
+ _state(state),
+ _profile(profile),
+ _range(range),
+ _range_params(range_params),
+ _remaining_table_level_row_count(-1) {
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, _ctzz);
+}
+
+PaimonCppReader::~PaimonCppReader() = default;
+
+Status PaimonCppReader::init_reader() {
+ if (_push_down_agg_type == TPushAggOp::type::COUNT && _remaining_table_level_row_count >= 0) {
+ return Status::OK();
+ }
+ return _init_paimon_reader();
+}
+
+Status PaimonCppReader::get_next_block(Block* block, size_t* read_rows, bool* eof) {
+ if (_push_down_agg_type == TPushAggOp::type::COUNT && _remaining_table_level_row_count >= 0) {
+ auto rows = std::min(_remaining_table_level_row_count,
+ (int64_t)_state->query_options().batch_size);
+ _remaining_table_level_row_count -= rows;
+ auto mutate_columns = block->mutate_columns();
+ for (auto& col : mutate_columns) {
+ col->resize(rows);
+ }
+ block->set_columns(std::move(mutate_columns));
+ *read_rows = rows;
+ *eof = false;
+ if (_remaining_table_level_row_count == 0) {
+ *eof = true;
+ }
+ return Status::OK();
+ }
+
+ if (!_batch_reader) {
+ return Status::InternalError("paimon-cpp reader is not initialized");
+ }
+
+ auto batch_result = _batch_reader->NextBatch();
+ if (!batch_result.ok()) {
+ return Status::InternalError("paimon-cpp read batch failed: {}",
+ batch_result.status().ToString());
+ }
+ auto batch = std::move(batch_result).value();
+ if (paimon::BatchReader::IsEofBatch(batch)) {
+ *read_rows = 0;
+ *eof = true;
+ return Status::OK();
+ }
+
+ arrow::Result<std::shared_ptr<arrow::RecordBatch>> import_result =
+ arrow::ImportRecordBatch(batch.first.get(), batch.second.get());
+ if (!import_result.ok()) {
+ return Status::InternalError("failed to import paimon-cpp arrow batch: {}",
+ import_result.status().message());
+ }
+
+ auto record_batch = std::move(import_result).ValueUnsafe();
+ const auto num_rows = static_cast<size_t>(record_batch->num_rows());
+ const auto num_rows_as_int = static_cast<int>(num_rows);
+ const auto num_columns = record_batch->num_columns();
+ for (int c = 0; c < num_columns; ++c) {
+ const auto& field = record_batch->schema()->field(c);
+ if (field->name() == VALUE_KIND_FIELD) {
+ continue;
+ }
+
+ auto* column_with_name = block->try_get_by_name(field->name());
+ if (column_with_name == nullptr) {
+ // Skip columns that are not in the block (e.g., partition columns handled elsewhere)
+ continue;
+ }
+ try {
+ column_with_name->type->get_serde()->read_column_from_arrow(
+ column_with_name->column->assume_mutable_ref(), record_batch->column(c).get(),
+ 0, num_rows_as_int, _ctzz);
+ } catch (Exception& e) {
+ return Status::InternalError("Failed to convert from arrow to block: {}", e.what());
+ }
+ }
+
+ *read_rows = num_rows;
+ *eof = false;
+ return Status::OK();
+}
+
+Status PaimonCppReader::get_columns(std::unordered_map<std::string, TypeDescriptor>* name_to_type,
+ std::unordered_set<std::string>* missing_cols) {
+ for (const auto& slot : _file_slot_descs) {
+ name_to_type->emplace(slot->col_name(), slot->type());
+ }
+ return Status::OK();
+}
+
+Status PaimonCppReader::close() {
+ if (_batch_reader) {
+ _batch_reader->Close();
+ }
+ return Status::OK();
+}
+
+Status PaimonCppReader::_init_paimon_reader() {
+ register_paimon_doris_file_system();
+ RETURN_IF_ERROR(_decode_split(&_split));
+
+ auto table_path_opt = _resolve_table_path();
+ if (!table_path_opt.has_value()) {
+ return Status::InternalError(
+ "paimon-cpp missing paimon_table; cannot resolve paimon table root path");
+ }
+ auto options = _build_options();
+ auto read_columns = _build_read_columns();
+
+ // Avoid moving strings across module boundaries to prevent allocator mismatches in ASAN builds.
+ std::string table_path = table_path_opt.value();
+ static std::once_flag options_log_once;
+ std::call_once(options_log_once, [&]() {
+ auto has_key = [&](const char* key) {
+ auto it = options.find(key);
+ return (it != options.end() && !it->second.empty()) ? "set" : "empty";
+ };
+ auto value_or = [&](const char* key) {
+ auto it = options.find(key);
+ return it != options.end() ? it->second : std::string("<unset>");
+ };
+ LOG(INFO) << "paimon-cpp options summary: table_path=" << table_path
+ << " AWS_ACCESS_KEY=" << has_key("AWS_ACCESS_KEY")
+ << " AWS_SECRET_KEY=" << has_key("AWS_SECRET_KEY")
+ << " AWS_TOKEN=" << has_key("AWS_TOKEN")
+ << " AWS_ENDPOINT=" << value_or("AWS_ENDPOINT")
+ << " AWS_REGION=" << value_or("AWS_REGION")
+ << " use_path_style=" << value_or("use_path_style")
+ << " fs.oss.endpoint=" << value_or("fs.oss.endpoint")
+ << " fs.s3a.endpoint=" << value_or("fs.s3a.endpoint");
+ });
+ paimon::ReadContextBuilder builder(table_path);
+ if (!read_columns.empty()) {
+ builder.SetReadSchema(read_columns);
+ }
+ if (!options.empty()) {
+ builder.SetOptions(options);
+ }
+ if (_predicate) {
+ builder.SetPredicate(_predicate);
+ builder.EnablePredicateFilter(true);
+ }
+
+ auto context_result = builder.Finish();
+ if (!context_result.ok()) {
+ return Status::InternalError("paimon-cpp build read context failed: {}",
+ context_result.status().ToString());
+ }
+ auto context = std::move(context_result).value();
+
+ auto table_read_result = paimon::TableRead::Create(std::move(context));
+ if (!table_read_result.ok()) {
+ return Status::InternalError("paimon-cpp create table read failed: {}",
+ table_read_result.status().ToString());
+ }
+ auto table_read = std::move(table_read_result).value();
+ auto reader_result = table_read->CreateReader(_split);
+ if (!reader_result.ok()) {
+ return Status::InternalError("paimon-cpp create reader failed: {}",
+ reader_result.status().ToString());
+ }
+ _table_read = std::move(table_read);
+ _batch_reader = std::move(reader_result).value();
+ return Status::OK();
+}
+
+Status PaimonCppReader::_decode_split(std::shared_ptr<paimon::Split>* split) {
+ if (!_range.__isset.table_format_params || !_range.table_format_params.__isset.paimon_params ||
+ !_range.table_format_params.paimon_params.__isset.paimon_split) {
+ return Status::InternalError("paimon-cpp missing paimon_split in scan range");
+ }
+ const auto& encoded_split = _range.table_format_params.paimon_params.paimon_split;
+ std::string decoded_split;
+ if (!base64_decode(encoded_split, &decoded_split)) {
+ return Status::InternalError("paimon-cpp base64 decode paimon_split failed");
+ }
+ auto pool = paimon::GetDefaultPool();
+ auto split_result =
+ paimon::Split::Deserialize(decoded_split.data(), decoded_split.size(), pool);
+ if (!split_result.ok()) {
+ return Status::InternalError("paimon-cpp deserialize split failed: {}",
+ split_result.status().ToString());
+ }
+ *split = std::move(split_result).value();
+ return Status::OK();
+}
+
+std::optional<std::string> PaimonCppReader::_resolve_table_path() const {
+ if (_range.__isset.table_format_params && _range.table_format_params.__isset.paimon_params &&
+ _range.table_format_params.paimon_params.__isset.paimon_table &&
+ !_range.table_format_params.paimon_params.paimon_table.empty()) {
+ return _range.table_format_params.paimon_params.paimon_table;
+ }
+ return std::nullopt;
+}
+
+std::vector<std::string> PaimonCppReader::_build_read_columns() const {
+ std::vector<std::string> columns;
+ columns.reserve(_file_slot_descs.size());
+ for (const auto& slot : _file_slot_descs) {
+ columns.emplace_back(slot->col_name());
+ }
+ return columns;
+}
+
+std::map<std::string, std::string> PaimonCppReader::_build_options() const {
+ std::map<std::string, std::string> options;
+ if (_range.__isset.table_format_params && _range.table_format_params.__isset.paimon_params &&
+ _range.table_format_params.paimon_params.__isset.paimon_options) {
+ options.insert(_range.table_format_params.paimon_params.paimon_options.begin(),
+ _range.table_format_params.paimon_params.paimon_options.end());
+ }
+
+ if (_range_params && _range_params->__isset.properties && !_range_params->properties.empty()) {
+ for (const auto& kv : _range_params->properties) {
+ options[kv.first] = kv.second;
+ }
+ } else if (_range.__isset.table_format_params &&
+ _range.table_format_params.__isset.paimon_params &&
+ _range.table_format_params.paimon_params.__isset.hadoop_conf) {
+ for (const auto& kv : _range.table_format_params.paimon_params.hadoop_conf) {
+ options[kv.first] = kv.second;
+ }
+ }
+
+ auto copy_if_missing = [&](const char* from_key, const char* to_key) {
+ if (options.find(to_key) != options.end()) {
+ return;
+ }
+ auto it = options.find(from_key);
+ if (it != options.end() && !it->second.empty()) {
+ options[to_key] = it->second;
+ }
+ };
+
+ // Map common OSS/S3 Hadoop configs to Doris S3 property keys.
+ copy_if_missing("fs.oss.accessKeyId", "AWS_ACCESS_KEY");
+ copy_if_missing("fs.oss.accessKeySecret", "AWS_SECRET_KEY");
+ copy_if_missing("fs.oss.sessionToken", "AWS_TOKEN");
+ copy_if_missing("fs.oss.endpoint", "AWS_ENDPOINT");
+ copy_if_missing("fs.oss.region", "AWS_REGION");
+ copy_if_missing("fs.s3a.access.key", "AWS_ACCESS_KEY");
+ copy_if_missing("fs.s3a.secret.key", "AWS_SECRET_KEY");
+ copy_if_missing("fs.s3a.session.token", "AWS_TOKEN");
+ copy_if_missing("fs.s3a.endpoint", "AWS_ENDPOINT");
+ copy_if_missing("fs.s3a.region", "AWS_REGION");
+ copy_if_missing("fs.s3a.path.style.access", "use_path_style");
+
+ // FE currently does not pass paimon_options in scan ranges.
+ // Backfill file.format/manifest.format from split file_format to avoid
+ // paimon-cpp falling back to default manifest.format=avro.
+ if (_range.__isset.table_format_params && _range.table_format_params.__isset.paimon_params &&
+ _range.table_format_params.paimon_params.__isset.file_format &&
+ !_range.table_format_params.paimon_params.file_format.empty()) {
+ const auto& split_file_format = _range.table_format_params.paimon_params.file_format;
+ auto file_format_it = options.find(paimon::Options::FILE_FORMAT);
+ if (file_format_it == options.end() || file_format_it->second.empty()) {
+ options[paimon::Options::FILE_FORMAT] = split_file_format;
+ }
+ auto manifest_format_it = options.find(paimon::Options::MANIFEST_FORMAT);
+ if (manifest_format_it == options.end() || manifest_format_it->second.empty()) {
+ options[paimon::Options::MANIFEST_FORMAT] = split_file_format;
+ }
+ }
+
+ options[paimon::Options::FILE_SYSTEM] = "doris";
+ return options;
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/paimon_cpp_reader.h b/be/src/vec/exec/format/table/paimon_cpp_reader.h
new file mode 100644
index 00000000000..49be99cbc3c
--- /dev/null
+++ b/be/src/vec/exec/format/table/paimon_cpp_reader.h
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstddef>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "cctz/time_zone.h"
+#include "common/status.h"
+#include "exec/olap_common.h"
+#include "paimon/reader/batch_reader.h"
+#include "paimon/table/source/split.h"
+#include "vec/exec/format/generic_reader.h"
+
+namespace paimon {
+class TableRead;
+class Predicate;
+} // namespace paimon
+
+namespace doris {
+class RuntimeProfile;
+class RuntimeState;
+class SlotDescriptor;
+} // namespace doris
+
+namespace doris::vectorized {
+
+class Block;
+
+class PaimonCppReader : public GenericReader {
+ ENABLE_FACTORY_CREATOR(PaimonCppReader);
+
+public:
+ PaimonCppReader(const std::vector<SlotDescriptor*>& file_slot_descs, RuntimeState* state,
+ RuntimeProfile* profile, const TFileRangeDesc& range,
+ const TFileScanRangeParams* range_params);
+ ~PaimonCppReader() override;
+
+ Status init_reader();
+ Status get_next_block(Block* block, size_t* read_rows, bool* eof) override;
+ Status get_columns(std::unordered_map<std::string, TypeDescriptor>* name_to_type,
+ std::unordered_set<std::string>* missing_cols) override;
+ Status close() override;
+ void set_predicate(std::shared_ptr<paimon::Predicate> predicate) {
+ _predicate = std::move(predicate);
+ }
+
+private:
+ Status _init_paimon_reader();
+ Status _decode_split(std::shared_ptr<paimon::Split>* split);
+ // Resolve paimon table root path for schema/manifest lookup.
+ std::optional<std::string> _resolve_table_path() const;
+ std::vector<std::string> _build_read_columns() const;
+ std::map<std::string, std::string> _build_options() const;
+
+ const std::vector<SlotDescriptor*>& _file_slot_descs;
+ RuntimeState* _state = nullptr;
+ [[maybe_unused]] RuntimeProfile* _profile = nullptr;
+ const TFileRangeDesc& _range;
+ const TFileScanRangeParams* _range_params = nullptr;
+
+ std::shared_ptr<paimon::Split> _split;
+ std::unique_ptr<paimon::TableRead> _table_read;
+ std::unique_ptr<paimon::BatchReader> _batch_reader;
+ std::shared_ptr<paimon::Predicate> _predicate;
+
+ std::unordered_map<std::string, uint32_t> _col_name_to_block_idx;
+ int64_t _remaining_table_level_row_count = -1;
+ cctz::time_zone _ctzz;
+};
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/paimon_doris_file_system.cpp b/be/src/vec/exec/format/table/paimon_doris_file_system.cpp
new file mode 100644
index 00000000000..d0cf4f406f2
--- /dev/null
+++ b/be/src/vec/exec/format/table/paimon_doris_file_system.cpp
@@ -0,0 +1,721 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "paimon_doris_file_system.h"
+
+#include <algorithm>
+#include <cctype>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "common/status.h"
+#include "gen_cpp/Types_types.h"
+#include "io/file_factory.h"
+#include "io/fs/broker_file_system.h"
+#include "io/fs/file_reader.h"
+#include "io/fs/file_system.h"
+#include "io/fs/file_writer.h"
+#include "io/fs/hdfs_file_system.h"
+#include "io/fs/local_file_system.h"
+#include "io/fs/s3_file_system.h"
+#include "io/hdfs_builder.h"
+#include "paimon/factories/factory.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/fs/file_system_factory.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+#include "util/s3_uri.h"
+#include "util/s3_util.h"
+
+namespace paimon {
+
+struct ParsedUri {
+ std::string scheme;
+ std::string authority;
+};
+
+std::string to_lower(std::string value) {
+ std::ranges::transform(value, value.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ return value;
+}
+
+ParsedUri parse_uri(const std::string& path) {
+ ParsedUri parsed;
+ size_t scheme_pos = path.find("://");
+ size_t delim_len = 3;
+ if (scheme_pos == std::string::npos) {
+ scheme_pos = path.find(":/");
+ delim_len = 2;
+ }
+ if (scheme_pos == std::string::npos || scheme_pos == 0) {
+ return parsed;
+ }
+ parsed.scheme = to_lower(path.substr(0, scheme_pos));
+ size_t authority_start = scheme_pos + delim_len;
+ if (authority_start >= path.size() || path[authority_start] == '/') {
+ return parsed;
+ }
+ size_t next_slash = path.find('/', authority_start);
+ if (next_slash == std::string::npos) {
+ parsed.authority = path.substr(authority_start);
+ } else {
+ parsed.authority = path.substr(authority_start, next_slash - authority_start);
+ }
+ return parsed;
+}
+
+bool is_s3_scheme(const std::string& scheme) {
+ return scheme == "s3" || scheme == "s3a" || scheme == "s3n" || scheme == "oss" ||
+ scheme == "obs" || scheme == "cos" || scheme == "cosn" || scheme == "gs" ||
+ scheme == "abfs" || scheme == "abfss" || scheme == "wasb" || scheme == "wasbs";
+}
+
+bool is_hdfs_scheme(const std::string& scheme) {
+ return scheme == "hdfs" || scheme == "viewfs" || scheme == "local";
+}
+
+bool is_http_scheme(const std::string& scheme) {
+ return scheme == "http" || scheme == "https";
+}
+
+doris::TFileType::type map_scheme_to_file_type(const std::string& scheme) {
+ if (scheme.empty()) {
+ return doris::TFileType::FILE_HDFS;
+ }
+ if (scheme == "file") {
+ return doris::TFileType::FILE_LOCAL;
+ }
+ if (is_hdfs_scheme(scheme)) {
+ return doris::TFileType::FILE_HDFS;
+ }
+ if (is_s3_scheme(scheme)) {
+ return doris::TFileType::FILE_S3;
+ }
+ if (is_http_scheme(scheme)) {
+ return doris::TFileType::FILE_S3;
+ }
+ if (scheme == "ofs" || scheme == "gfs" || scheme == "jfs") {
+ return doris::TFileType::FILE_BROKER;
+ }
+ return doris::TFileType::FILE_HDFS;
+}
+
+std::string replace_scheme(const std::string& path, const std::string& scheme) {
+ size_t scheme_pos = path.find("://");
+ size_t delim_len = 3;
+ if (scheme_pos == std::string::npos) {
+ scheme_pos = path.find(":/");
+ delim_len = 2;
+ }
+ if (scheme_pos == std::string::npos) {
+ return path;
+ }
+ return scheme + "://" + path.substr(scheme_pos + delim_len);
+}
+
+std::string normalize_local_path(const std::string& path) {
+ if (!path.starts_with("file:")) {
+ return path;
+ }
+ constexpr size_t file_prefix_len = 5;
+ size_t start = file_prefix_len;
+ if (path.compare(start, 2, "//") == 0 && path.size() - start > 2) {
+ size_t next_slash = path.find('/', start + 2);
+ if (next_slash == std::string::npos) {
+ return "";
+ }
+ start = next_slash;
+ }
+ return path.substr(start);
+}
+
+std::string normalize_path_for_type(const std::string& path, const std::string& scheme,
+ doris::TFileType::type type) {
+ if (type == doris::TFileType::FILE_LOCAL) {
+ return normalize_local_path(path);
+ }
+ if (type == doris::TFileType::FILE_S3 && scheme != "s3" && !is_http_scheme(scheme)) {
+ return replace_scheme(path, "s3");
+ }
+ return path;
+}
+
+std::string build_fs_cache_key(doris::TFileType::type type, const ParsedUri& uri,
+ const std::string& default_fs_name,
+ const std::string& normalized_path) {
+ switch (type) {
+ case doris::TFileType::FILE_LOCAL:
+ return "local";
+ case doris::TFileType::FILE_S3: {
+ doris::S3URI s3_uri(normalized_path);
+ if (s3_uri.parse().ok()) {
+ if (is_http_scheme(uri.scheme)) {
+ return uri.scheme + "://" + uri.authority + "/" + s3_uri.get_bucket();
+ }
+ return "s3://" + s3_uri.get_bucket();
+ }
+ return normalized_path;
+ }
+ case doris::TFileType::FILE_BROKER:
+ return "broker";
+ case doris::TFileType::FILE_HDFS:
+ default:
+ if (!uri.scheme.empty() || !uri.authority.empty()) {
+ return uri.scheme + "://" + uri.authority;
+ }
+ return default_fs_name;
+ }
+}
+
+paimon::Status to_paimon_status(const doris::Status& status) {
+ if (status.ok()) {
+ return paimon::Status::OK();
+ }
+ switch (status.code()) {
+ case doris::ErrorCode::NOT_FOUND:
+ case doris::ErrorCode::DIR_NOT_EXIST:
+ return paimon::Status::NotExist(status.to_string());
+ case doris::ErrorCode::ALREADY_EXIST:
+ case doris::ErrorCode::FILE_ALREADY_EXIST:
+ return paimon::Status::Exist(status.to_string());
+ case doris::ErrorCode::INVALID_ARGUMENT:
+ return paimon::Status::Invalid(status.to_string());
+ case doris::ErrorCode::NOT_IMPLEMENTED_ERROR:
+ return paimon::Status::NotImplemented(status.to_string());
+ default:
+ return paimon::Status::IOError(status.to_string());
+ }
+}
+
+std::string join_path(const std::string& base, const std::string& child) {
+ if (base.empty()) {
+ return child;
+ }
+ if (base.back() == '/') {
+ return base + child;
+ }
+ return base + "/" + child;
+}
+
+std::string parent_path_no_scheme(const std::string& path) {
+ if (path.empty()) {
+ return "";
+ }
+ size_t end = path.size();
+ while (end > 1 && path[end - 1] == '/') {
+ --end;
+ }
+ size_t pos = path.rfind('/', end - 1);
+ if (pos == std::string::npos) {
+ return "";
+ }
+ if (pos == 0) {
+ return "/";
+ }
+ return path.substr(0, pos);
+}
+
+std::string parent_path(const std::string& path) {
+ ParsedUri uri = parse_uri(path);
+ if (uri.scheme.empty()) {
+ return parent_path_no_scheme(path);
+ }
+ size_t scheme_pos = path.find("://");
+ size_t delim_len = 3;
+ if (scheme_pos == std::string::npos) {
+ scheme_pos = path.find(":/");
+ delim_len = 2;
+ }
+ if (scheme_pos == std::string::npos) {
+ return parent_path_no_scheme(path);
+ }
+ size_t start = scheme_pos + delim_len;
+ size_t slash = path.find('/', start);
+ if (slash == std::string::npos) {
+ return "";
+ }
+ std::string path_part = path.substr(slash);
+ std::string parent_part = parent_path_no_scheme(path_part);
+ if (parent_part.empty()) {
+ return "";
+ }
+ std::string prefix = uri.scheme + "://";
+ if (!uri.authority.empty()) {
+ prefix += uri.authority;
+ }
+ return prefix + parent_part;
+}
+
+std::string hdfs_fs_name(const ParsedUri& uri, const std::string& default_fs_name) {
+ if (!uri.scheme.empty() && !uri.authority.empty()) {
+ return uri.scheme + "://" + uri.authority;
+ }
+ return default_fs_name;
+}
+
+class DorisInputStream : public InputStream {
+public:
+ DorisInputStream(doris::io::FileReaderSPtr reader, std::string path)
+ : reader_(std::move(reader)), path_(std::move(path)) {}
+
+ Status Seek(int64_t offset, SeekOrigin origin) override {
+ int64_t target = 0;
+ if (origin == SeekOrigin::FS_SEEK_SET) {
+ target = offset;
+ } else if (origin == SeekOrigin::FS_SEEK_CUR) {
+ target = position_ + offset;
+ } else if (origin == SeekOrigin::FS_SEEK_END) {
+ target = static_cast<int64_t>(reader_->size()) + offset;
+ } else {
+ return Status::Invalid("unknown seek origin");
+ }
+ if (target < 0) {
+ return Status::Invalid("seek position is negative");
+ }
+ position_ = target;
+ return Status::OK();
+ }
+
+ Result<int64_t> GetPos() const override { return position_; }
+
+ Result<int32_t> Read(char* buffer, uint32_t size) override {
+ size_t bytes_read = 0;
+ doris::Status status = reader_->read_at(position_, doris::Slice(buffer, size), &bytes_read);
+ if (!status.ok()) {
+ return to_paimon_status(status);
+ }
+ position_ += static_cast<int64_t>(bytes_read);
+ return static_cast<int32_t>(bytes_read);
+ }
+
+ Result<int32_t> Read(char* buffer, uint32_t size, uint64_t offset) override {
+ size_t bytes_read = 0;
+ doris::Status status = reader_->read_at(offset, doris::Slice(buffer, size), &bytes_read);
+ if (!status.ok()) {
+ return to_paimon_status(status);
+ }
+ return static_cast<int32_t>(bytes_read);
+ }
+
+ void ReadAsync(char* buffer, uint32_t size, uint64_t offset,
+ std::function<void(Status)>&& callback) override {
+ Result<int32_t> result = Read(buffer, size, offset);
+ Status status = Status::OK();
+ if (!result.ok()) {
+ status = result.status();
+ }
+ callback(status);
+ }
+
+ Result<std::string> GetUri() const override { return path_; }
+
+ Result<uint64_t> Length() const override { return static_cast<uint64_t>(reader_->size()); }
+
+ Status Close() override { return to_paimon_status(reader_->close()); }
+
+private:
+ doris::io::FileReaderSPtr reader_;
+ std::string path_;
+ int64_t position_ = 0;
+};
+
+class DorisOutputStream : public OutputStream {
+public:
+ DorisOutputStream(doris::io::FileWriterPtr writer, std::string path)
+ : writer_(std::move(writer)), path_(std::move(path)) {}
+
+ Result<int32_t> Write(const char* buffer, uint32_t size) override {
+ doris::Status status = writer_->append(doris::Slice(buffer, size));
+ if (!status.ok()) {
+ return to_paimon_status(status);
+ }
+ return static_cast<int32_t>(size);
+ }
+
+ Status Flush() override { return Status::OK(); }
+
+ Result<int64_t> GetPos() const override {
+ return static_cast<int64_t>(writer_->bytes_appended());
+ }
+
+ Result<std::string> GetUri() const override { return path_; }
+
+ Status Close() override { return to_paimon_status(writer_->close()); }
+
+private:
+ doris::io::FileWriterPtr writer_;
+ std::string path_;
+};
+
+class DorisBasicFileStatus : public BasicFileStatus {
+public:
+ DorisBasicFileStatus(std::string path, bool is_dir) : path_(std::move(path)), is_dir_(is_dir) {}
+
+ bool IsDir() const override { return is_dir_; }
+ std::string GetPath() const override { return path_; }
+
+private:
+ std::string path_;
+ bool is_dir_;
+};
+
+class DorisFileStatus : public FileStatus {
+public:
+ DorisFileStatus(std::string path, bool is_dir, uint64_t length, int64_t mtime)
+ : path_(std::move(path)), is_dir_(is_dir), length_(length), mtime_(mtime) {}
+
+ uint64_t GetLen() const override { return length_; }
+ bool IsDir() const override { return is_dir_; }
+ std::string GetPath() const override { return path_; }
+ int64_t GetModificationTime() const override { return mtime_; }
+
+private:
+ std::string path_;
+ bool is_dir_;
+ uint64_t length_;
+ int64_t mtime_;
+};
+
+class DorisFileSystem : public FileSystem {
+public:
+ explicit DorisFileSystem(std::map<std::string, std::string> options)
+ : options_(std::move(options)) {
+ auto it = options_.find("fs.defaultFS");
+ if (it != options_.end()) {
+ default_fs_name_ = it->second;
+ }
+ }
+
+ Result<std::unique_ptr<InputStream>> Open(const std::string& path) const override {
+ PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path));
+ auto& fs = resolved.first;
+ auto& normalized_path = resolved.second;
+ doris::io::FileReaderSPtr reader;
+ doris::io::FileReaderOptions reader_options = doris::io::FileReaderOptions::DEFAULT;
+ doris::Status status = fs->open_file(normalized_path, &reader, &reader_options);
+ if (!status.ok()) {
+ return to_paimon_status(status);
+ }
+ return std::make_unique<DorisInputStream>(std::move(reader), normalized_path);
+ }
+
+ Result<std::unique_ptr<OutputStream>> Create(const std::string& path,
+ bool overwrite) const override {
+ PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path));
+ auto& fs = resolved.first;
+ auto& normalized_path = resolved.second;
+ if (!overwrite) {
+ bool exists = false;
+ doris::Status exists_status = fs->exists(normalized_path, &exists);
+ if (!exists_status.ok()) {
+ return to_paimon_status(exists_status);
+ }
+ if (exists) {
+ return Status::Exist("file already exists: ", normalized_path);
+ }
+ }
+ std::string parent = parent_path(normalized_path);
+ if (!parent.empty()) {
+ doris::Status mkdir_status = fs->create_directory(parent);
+ if (!mkdir_status.ok()) {
+ return to_paimon_status(mkdir_status);
+ }
+ }
+ doris::io::FileWriterPtr writer;
+ doris::Status status = fs->create_file(normalized_path, &writer);
+ if (!status.ok()) {
+ return to_paimon_status(status);
+ }
+ return std::make_unique<DorisOutputStream>(std::move(writer), normalized_path);
+ }
+
+ Status Mkdirs(const std::string& path) const override {
+ PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path));
+ doris::Status status = resolved.first->create_directory(resolved.second);
+ return to_paimon_status(status);
+ }
+
+ Status Rename(const std::string& src, const std::string& dst) const override {
+ PAIMON_ASSIGN_OR_RAISE(auto src_resolved, resolve_path(src));
+ PAIMON_ASSIGN_OR_RAISE(auto dst_resolved, resolve_path(dst));
+ doris::Status status = src_resolved.first->rename(src_resolved.second, dst_resolved.second);
+ return to_paimon_status(status);
+ }
+
+ Status Delete(const std::string& path, bool recursive = true) const override {
+ PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path));
+ bool exists = false;
+ doris::Status exists_status = resolved.first->exists(resolved.second, &exists);
+ if (!exists_status.ok()) {
+ return to_paimon_status(exists_status);
+ }
+ if (!exists) {
+ return Status::OK();
+ }
+ int64_t size = 0;
+ doris::Status size_status = resolved.first->file_size(resolved.second, &size);
+ if (size_status.ok()) {
+ return to_paimon_status(resolved.first->delete_file(resolved.second));
+ }
+ if (recursive) {
+ return to_paimon_status(resolved.first->delete_directory(resolved.second));
+ }
+ return to_paimon_status(size_status);
+ }
+
+ Result<std::unique_ptr<FileStatus>> GetFileStatus(const std::string& path) const override {
+ ParsedUri uri = parse_uri(path);
+ doris::TFileType::type type = map_scheme_to_file_type(uri.scheme);
+ PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path));
+ bool exists = false;
+ doris::Status exists_status = resolved.first->exists(resolved.second, &exists);
+ if (!exists_status.ok()) {
+ return to_paimon_status(exists_status);
+ }
+ if (!exists) {
+ if (type != doris::TFileType::FILE_S3) {
+ return Status::NotExist("path not exists: ", resolved.second);
+ }
+ std::vector<doris::io::FileInfo> files;
+ bool list_exists = false;
+ doris::Status list_status =
+ resolved.first->list(resolved.second, false, &files, &list_exists);
+ if (!list_status.ok()) {
+ return to_paimon_status(list_status);
+ }
+ if (!list_exists && files.empty()) {
+ return Status::NotExist("path not exists: ", resolved.second);
+ }
+ return std::make_unique<DorisFileStatus>(resolved.second, true, 0, 0);
+ }
+ int64_t size = 0;
+ doris::Status size_status = resolved.first->file_size(resolved.second, &size);
+ if (size_status.ok()) {
+ return std::make_unique<DorisFileStatus>(resolved.second, false,
+ static_cast<uint64_t>(size), 0);
+ }
+ std::vector<doris::io::FileInfo> files;
+ bool list_exists = false;
+ doris::Status list_status =
+ resolved.first->list(resolved.second, false, &files, &list_exists);
+ if (!list_status.ok()) {
+ return to_paimon_status(list_status);
+ }
+ if (!list_exists && files.empty()) {
+ return Status::NotExist("path not exists: ", resolved.second);
+ }
+ return std::make_unique<DorisFileStatus>(resolved.second, true, 0, 0);
+ }
+
+ Status ListDir(const std::string& directory,
+ std::vector<std::unique_ptr<BasicFileStatus>>* status_list) const override {
+ PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(directory));
+ auto file_status = GetFileStatus(directory);
+ if (file_status.ok() && !file_status.value()->IsDir()) {
+ return Status::IOError("path is not a directory: ", directory);
+ }
+ std::vector<doris::io::FileInfo> files;
+ bool exists = false;
+ doris::Status status = resolved.first->list(resolved.second, false, &files, &exists);
+ if (!status.ok()) {
+ return to_paimon_status(status);
+ }
+ if (!exists) {
+ return Status::OK();
+ }
+ status_list->reserve(status_list->size() + files.size());
+ for (const auto& file : files) {
+ status_list->emplace_back(std::make_unique<DorisBasicFileStatus>(
+ join_path(resolved.second, file.file_name), !file.is_file));
+ }
+ return Status::OK();
+ }
+
+ Status ListFileStatus(const std::string& path,
+ std::vector<std::unique_ptr<FileStatus>>* status_list) const override {
+ PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path));
+ auto self_status = GetFileStatus(path);
+ if (!self_status.ok()) {
+ if (self_status.status().IsNotExist()) {
+ return Status::OK();
+ }
+ return self_status.status();
+ }
+ if (!self_status.value()->IsDir()) {
+ status_list->emplace_back(std::move(self_status).value());
+ return Status::OK();
+ }
+ std::vector<doris::io::FileInfo> files;
+ bool exists = false;
+ doris::Status list_status = resolved.first->list(resolved.second, false, &files, &exists);
+ if (!list_status.ok()) {
+ return to_paimon_status(list_status);
+ }
+ if (!exists) {
+ return Status::OK();
+ }
+ status_list->reserve(status_list->size() + files.size());
+ for (const auto& file : files) {
+ uint64_t length = file.is_file ? static_cast<uint64_t>(file.file_size) : 0;
+ status_list->emplace_back(std::make_unique<DorisFileStatus>(
+ join_path(resolved.second, file.file_name), !file.is_file, length, 0));
+ }
+ return Status::OK();
+ }
+
+ Result<bool> Exists(const std::string& path) const override {
+ ParsedUri uri = parse_uri(path);
+ doris::TFileType::type type = map_scheme_to_file_type(uri.scheme);
+ PAIMON_ASSIGN_OR_RAISE(auto resolved, resolve_path(path));
+ bool exists = false;
+ doris::Status status = resolved.first->exists(resolved.second, &exists);
+ if (!status.ok()) {
+ return to_paimon_status(status);
+ }
+ if (!exists && type == doris::TFileType::FILE_S3) {
+ std::vector<doris::io::FileInfo> files;
+ bool list_exists = false;
+ doris::Status list_status =
+ resolved.first->list(resolved.second, false, &files, &list_exists);
+ if (!list_status.ok()) {
+ return to_paimon_status(list_status);
+ }
+ return list_exists || !files.empty();
+ }
+ return exists;
+ }
+
+private:
+ Result<doris::io::FileSystemSPtr> create_fs(doris::TFileType::type type, const ParsedUri& uri,
+ const std::string& normalized_path) const {
+ switch (type) {
+ case doris::TFileType::FILE_LOCAL:
+ return std::static_pointer_cast<doris::io::FileSystem>(
+ doris::io::global_local_filesystem());
+ case doris::TFileType::FILE_S3: {
+ doris::S3URI s3_uri(normalized_path);
+ auto status = s3_uri.parse();
+ if (!status.ok()) {
+ return to_paimon_status(status);
+ }
+ doris::S3Conf s3_conf;
+ status = doris::S3ClientFactory::convert_properties_to_s3_conf(options_, s3_uri,
+ &s3_conf);
+ if (!status.ok()) {
+ return to_paimon_status(status);
+ }
+ std::shared_ptr<doris::io::S3FileSystem> fs;
+ status = doris::io::S3FileSystem::create(std::move(s3_conf), "", nullptr, &fs);
+ if (!status.ok()) {
+ return to_paimon_status(status);
+ }
+ return std::static_pointer_cast<doris::io::FileSystem>(std::move(fs));
+ }
+ case doris::TFileType::FILE_BROKER: {
+ if (broker_addresses_.empty()) {
+ return Status::Invalid("broker addresses are empty for paimon path: ",
+ normalized_path);
+ }
+ std::shared_ptr<doris::io::BrokerFileSystem> fs;
+ auto status = doris::io::BrokerFileSystem::create(broker_addresses_[0], options_, &fs);
+ if (!status.ok()) {
+ return to_paimon_status(status);
+ }
+ return std::static_pointer_cast<doris::io::FileSystem>(std::move(fs));
+ }
+ case doris::TFileType::FILE_HDFS: {
+ doris::THdfsParams hdfs_params = doris::parse_properties(options_);
+ std::shared_ptr<doris::io::HdfsFileSystem> fs;
+ auto status = doris::io::HdfsFileSystem::create(
+ hdfs_params, "", hdfs_fs_name(uri, default_fs_name_), nullptr, &fs);
+ if (!status.ok()) {
+ return to_paimon_status(status);
+ }
+ return std::static_pointer_cast<doris::io::FileSystem>(std::move(fs));
+ }
+ default:
+ return Status::NotImplemented("unsupported paimon Doris file system type: ",
+ std::to_string(type));
+ }
+ }
+
+ Result<std::pair<doris::io::FileSystemSPtr, std::string>> resolve_path(
+ const std::string& path) const {
+ auto uri = parse_uri(path);
+ doris::TFileType::type type = map_scheme_to_file_type(uri.scheme);
+ std::string normalized_path = normalize_path_for_type(path, uri.scheme, type);
+ if (type == doris::TFileType::FILE_LOCAL) {
+ doris::io::FileSystemSPtr fs = doris::io::global_local_filesystem();
+ return std::make_pair(std::move(fs), normalized_path);
+ }
+ std::string fs_key = build_fs_cache_key(type, uri, default_fs_name_, normalized_path);
+ {
+ std::lock_guard lock(fs_lock_);
+ auto it = fs_cache_.find(fs_key);
+ if (it != fs_cache_.end()) {
+ return std::make_pair(it->second, normalized_path);
+ }
+ }
+ auto fs_result = create_fs(type, uri, normalized_path);
+ if (!fs_result.ok()) {
+ return fs_result.status();
+ }
+ doris::io::FileSystemSPtr fs = std::move(fs_result).value();
+ {
+ std::lock_guard lock(fs_lock_);
+ fs_cache_.emplace(std::move(fs_key), fs);
+ }
+ return std::make_pair(std::move(fs), std::move(normalized_path));
+ }
+
+ std::map<std::string, std::string> options_;
+ std::vector<doris::TNetworkAddress> broker_addresses_;
+ std::string default_fs_name_;
+ mutable std::mutex fs_lock_;
+ mutable std::unordered_map<std::string, doris::io::FileSystemSPtr> fs_cache_;
+};
+
+class DorisFileSystemFactory : public FileSystemFactory {
+public:
+ static const char IDENTIFIER[];
+
+ const char* Identifier() const override { return IDENTIFIER; }
+
+ Result<std::unique_ptr<FileSystem>> Create(
+ const std::string& path,
+ const std::map<std::string, std::string>& options) const override {
+ return std::make_unique<DorisFileSystem>(options);
+ }
+};
+
+const char DorisFileSystemFactory::IDENTIFIER[] = "doris";
+
+REGISTER_PAIMON_FACTORY(DorisFileSystemFactory);
+
+} // namespace paimon
+
+namespace doris::vectorized {
+
+void register_paimon_doris_file_system() {}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/paimon_doris_file_system.h b/be/src/vec/exec/format/table/paimon_doris_file_system.h
new file mode 100644
index 00000000000..22552c6eb6f
--- /dev/null
+++ b/be/src/vec/exec/format/table/paimon_doris_file_system.h
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+namespace doris::vectorized {
+
+// Force-link helper so the paimon-cpp file system factory registration is kept.
+void register_paimon_doris_file_system();
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/exec/format/table/paimon_predicate_converter.cpp b/be/src/vec/exec/format/table/paimon_predicate_converter.cpp
new file mode 100644
index 00000000000..e2c0c1c65d9
--- /dev/null
+++ b/be/src/vec/exec/format/table/paimon_predicate_converter.cpp
@@ -0,0 +1,654 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/paimon_predicate_converter.h"
+
+#include <algorithm>
+#include <cctype>
+#include <utility>
+
+#include "paimon/data/decimal.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/predicate/predicate_builder.h"
+#include "runtime/decimalv2_value.h"
+#include "runtime/descriptors.h"
+#include "runtime/runtime_state.h"
+#include "util/timezone_utils.h"
+#include "vec/columns/column_const.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/core/field.h"
+#include "vec/core/types.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/exprs/vcompound_pred.h"
+#include "vec/exprs/vdirect_in_predicate.h"
+#include "vec/exprs/vectorized_fn_call.h"
+#include "vec/exprs/vexpr.h"
+#include "vec/exprs/vin_predicate.h"
+#include "vec/exprs/vliteral.h"
+#include "vec/exprs/vslot_ref.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris::vectorized {
+
+PaimonPredicateConverter::PaimonPredicateConverter(
+ const std::vector<SlotDescriptor*>& file_slot_descs, RuntimeState* state)
+ : _state(state) {
+ _field_index_by_name.reserve(file_slot_descs.size());
+ for (size_t i = 0; i < file_slot_descs.size(); ++i) {
+ const auto& name = file_slot_descs[i]->col_name();
+ auto normalized = _normalize_name(name);
+ if (_field_index_by_name.find(normalized) == _field_index_by_name.end()) {
+ _field_index_by_name.emplace(std::move(normalized), static_cast<int32_t>(i));
+ }
+ }
+
+ if (!TimezoneUtils::find_cctz_time_zone("GMT", _gmt_tz)) {
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, _gmt_tz);
+ }
+}
+
+std::shared_ptr<paimon::Predicate> PaimonPredicateConverter::build(
+ const VExprContextSPtrs& conjuncts) {
+ std::vector<std::shared_ptr<paimon::Predicate>> predicates;
+ predicates.reserve(conjuncts.size());
+ for (const auto& conjunct : conjuncts) {
+ if (!conjunct || !conjunct->root()) {
+ continue;
+ }
+ auto root = conjunct->root();
+ if (root->is_rf_wrapper()) {
+ if (auto impl = root->get_impl()) {
+ root = impl;
+ }
+ }
+ auto predicate = _convert_expr(root);
+ if (predicate) {
+ predicates.emplace_back(std::move(predicate));
+ }
+ }
+
+ if (predicates.empty()) {
+ return nullptr;
+ }
+ if (predicates.size() == 1) {
+ return predicates.front();
+ }
+ auto and_result = paimon::PredicateBuilder::And(predicates);
+ if (!and_result.ok()) {
+ return nullptr;
+ }
+ return std::move(and_result).value();
+}
+
+std::shared_ptr<paimon::Predicate> PaimonPredicateConverter::_convert_expr(const VExprSPtr& expr) {
+ if (!expr) {
+ return nullptr;
+ }
+
+ auto uncast = VExpr::expr_without_cast(expr);
+
+ if (dynamic_cast<VDirectInPredicate*>(uncast.get()) != nullptr) {
+ // Older branches do not expose the direct-in literal set, so skip this pushdown.
+ return nullptr;
+ }
+
+ if (dynamic_cast<VInPredicate*>(uncast.get()) != nullptr) {
+ return _convert_in(uncast);
+ }
+
+ switch (uncast->op()) {
+ case TExprOpcode::COMPOUND_AND:
+ case TExprOpcode::COMPOUND_OR:
+ return _convert_compound(uncast);
+ case TExprOpcode::COMPOUND_NOT:
+ return nullptr;
+ case TExprOpcode::EQ:
+ case TExprOpcode::EQ_FOR_NULL:
+ case TExprOpcode::NE:
+ case TExprOpcode::GE:
+ case TExprOpcode::GT:
+ case TExprOpcode::LE:
+ case TExprOpcode::LT:
+ return _convert_binary(uncast);
+ default:
+ break;
+ }
+
+ if (auto* fn = dynamic_cast<VectorizedFnCall*>(uncast.get())) {
+ auto fn_name = _normalize_name(fn->fn().name.function_name);
+ if (fn_name == "is_null_pred" || fn_name == "is_not_null_pred") {
+ return _convert_is_null(uncast, fn_name);
+ }
+ if (fn_name == "like") {
+ return _convert_like_prefix(uncast);
+ }
+ }
+
+ return nullptr;
+}
+
+std::shared_ptr<paimon::Predicate> PaimonPredicateConverter::_convert_compound(
+ const VExprSPtr& expr) {
+ if (!expr || expr->get_num_children() != 2) {
+ return nullptr;
+ }
+ auto left = _convert_expr(expr->get_child(0));
+ if (!left) {
+ return nullptr;
+ }
+ auto right = _convert_expr(expr->get_child(1));
+ if (!right) {
+ return nullptr;
+ }
+
+ if (expr->op() == TExprOpcode::COMPOUND_AND) {
+ auto and_result = paimon::PredicateBuilder::And({left, right});
+ return and_result.ok() ? std::move(and_result).value() : nullptr;
+ }
+ if (expr->op() == TExprOpcode::COMPOUND_OR) {
+ auto or_result = paimon::PredicateBuilder::Or({left, right});
+ return or_result.ok() ? std::move(or_result).value() : nullptr;
+ }
+ return nullptr;
+}
+
+std::shared_ptr<paimon::Predicate> PaimonPredicateConverter::_convert_in(const VExprSPtr& expr) {
+ auto* in_pred = dynamic_cast<VInPredicate*>(expr.get());
+ if (!in_pred || expr->get_num_children() < 2) {
+ return nullptr;
+ }
+ auto field_meta = _resolve_field(expr->get_child(0));
+ if (!field_meta) {
+ return nullptr;
+ }
+
+ std::vector<paimon::Literal> literals;
+ literals.reserve(expr->get_num_children() - 1);
+ for (uint16_t i = 1; i < expr->get_num_children(); ++i) {
+ auto literal = _convert_literal(expr->get_child(i), *field_meta->slot_desc,
+ field_meta->field_type);
+ if (!literal) {
+ return nullptr;
+ }
+ literals.emplace_back(std::move(*literal));
+ }
+
+ if (literals.empty()) {
+ return nullptr;
+ }
+ if (in_pred->is_not_in()) {
+ return paimon::PredicateBuilder::NotIn(field_meta->index, field_meta->slot_desc->col_name(),
+ field_meta->field_type, literals);
+ }
+ return paimon::PredicateBuilder::In(field_meta->index, field_meta->slot_desc->col_name(),
+ field_meta->field_type, literals);
+}
+
+std::shared_ptr<paimon::Predicate> PaimonPredicateConverter::_convert_binary(
+ const VExprSPtr& expr) {
+ if (!expr || expr->get_num_children() != 2) {
+ return nullptr;
+ }
+ auto field_meta = _resolve_field(expr->get_child(0));
+ if (!field_meta) {
+ return nullptr;
+ }
+
+ if (expr->op() == TExprOpcode::EQ_FOR_NULL) {
+ return paimon::PredicateBuilder::IsNull(
+ field_meta->index, field_meta->slot_desc->col_name(), field_meta->field_type);
+ }
+
+ auto literal =
+ _convert_literal(expr->get_child(1), *field_meta->slot_desc, field_meta->field_type);
+ if (!literal) {
+ return nullptr;
+ }
+
+ switch (expr->op()) {
+ case TExprOpcode::EQ:
+ return paimon::PredicateBuilder::Equal(field_meta->index, field_meta->slot_desc->col_name(),
+ field_meta->field_type, *literal);
+ case TExprOpcode::NE:
+ return paimon::PredicateBuilder::NotEqual(field_meta->index,
+ field_meta->slot_desc->col_name(),
+ field_meta->field_type, *literal);
+ case TExprOpcode::GE:
+ return paimon::PredicateBuilder::GreaterOrEqual(field_meta->index,
+ field_meta->slot_desc->col_name(),
+ field_meta->field_type, *literal);
+ case TExprOpcode::GT:
+ return paimon::PredicateBuilder::GreaterThan(field_meta->index,
+ field_meta->slot_desc->col_name(),
+ field_meta->field_type, *literal);
+ case TExprOpcode::LE:
+ return paimon::PredicateBuilder::LessOrEqual(field_meta->index,
+ field_meta->slot_desc->col_name(),
+ field_meta->field_type, *literal);
+ case TExprOpcode::LT:
+ return paimon::PredicateBuilder::LessThan(field_meta->index,
+ field_meta->slot_desc->col_name(),
+ field_meta->field_type, *literal);
+ default:
+ break;
+ }
+ return nullptr;
+}
+
+std::shared_ptr<paimon::Predicate> PaimonPredicateConverter::_convert_is_null(
+ const VExprSPtr& expr, const std::string& fn_name) {
+ if (!expr || expr->get_num_children() != 1) {
+ return nullptr;
+ }
+ auto field_meta = _resolve_field(expr->get_child(0));
+ if (!field_meta) {
+ return nullptr;
+ }
+ if (fn_name == "is_not_null_pred") {
+ return paimon::PredicateBuilder::IsNotNull(
+ field_meta->index, field_meta->slot_desc->col_name(), field_meta->field_type);
+ }
+ return paimon::PredicateBuilder::IsNull(field_meta->index, field_meta->slot_desc->col_name(),
+ field_meta->field_type);
+}
+
+std::shared_ptr<paimon::Predicate> PaimonPredicateConverter::_convert_like_prefix(
+ const VExprSPtr& expr) {
+ if (!expr || expr->get_num_children() != 2) {
+ return nullptr;
+ }
+ auto field_meta = _resolve_field(expr->get_child(0));
+ if (!field_meta || field_meta->field_type != paimon::FieldType::STRING) {
+ return nullptr;
+ }
+
+ auto pattern_opt = _extract_string_literal(expr->get_child(1));
+ if (!pattern_opt) {
+ return nullptr;
+ }
+ const std::string& pattern = *pattern_opt;
+ if (!pattern.empty() && pattern.front() == '%') {
+ return nullptr;
+ }
+ if (pattern.empty() || pattern.back() != '%') {
+ return nullptr;
+ }
+
+ std::string prefix = pattern.substr(0, pattern.size() - 1);
+ paimon::Literal lower_literal(paimon::FieldType::STRING, prefix.data(), prefix.size());
+ auto lower_pred = paimon::PredicateBuilder::GreaterOrEqual(
+ field_meta->index, field_meta->slot_desc->col_name(), field_meta->field_type,
+ lower_literal);
+
+ auto upper_prefix = _next_prefix(prefix);
+ if (!upper_prefix) {
+ return lower_pred;
+ }
+
+ paimon::Literal upper_literal(paimon::FieldType::STRING, upper_prefix->data(),
+ upper_prefix->size());
+ auto upper_pred =
+ paimon::PredicateBuilder::LessThan(field_meta->index, field_meta->slot_desc->col_name(),
+ field_meta->field_type, upper_literal);
+ auto and_result = paimon::PredicateBuilder::And({lower_pred, upper_pred});
+ return and_result.ok() ? std::move(and_result).value() : nullptr;
+}
+
+std::optional<PaimonPredicateConverter::FieldMeta> PaimonPredicateConverter::_resolve_field(
+ const VExprSPtr& expr) const {
+ if (!_state || !expr) {
+ return std::nullopt;
+ }
+ auto slot_expr = VExpr::expr_without_cast(expr);
+ auto* slot_ref = dynamic_cast<VSlotRef*>(slot_expr.get());
+ if (!slot_ref) {
+ return std::nullopt;
+ }
+ auto* slot_desc = _state->desc_tbl().get_slot_descriptor(slot_ref->slot_id());
+ if (!slot_desc) {
+ return std::nullopt;
+ }
+ auto normalized = _normalize_name(slot_desc->col_name());
+ auto it = _field_index_by_name.find(normalized);
+ if (it == _field_index_by_name.end()) {
+ return std::nullopt;
+ }
+ const auto& slot_type = slot_desc->type();
+ auto field_type =
+ _to_paimon_field_type(slot_type.type, static_cast<uint32_t>(slot_type.precision));
+ if (!field_type) {
+ return std::nullopt;
+ }
+ return FieldMeta {it->second, *field_type, slot_desc};
+}
+
+std::optional<paimon::Literal> PaimonPredicateConverter::_convert_literal(
+ const VExprSPtr& expr, const SlotDescriptor& slot_desc,
+ paimon::FieldType field_type) const {
+ auto literal_expr = VExpr::expr_without_cast(expr);
+ auto* literal = dynamic_cast<VLiteral*>(literal_expr.get());
+ if (!literal) {
+ return std::nullopt;
+ }
+
+ const auto literal_type = literal_expr->type();
+ PrimitiveType literal_primitive = literal_type.type;
+ PrimitiveType slot_primitive = slot_desc.type().type;
+
+ ColumnPtr col = literal->get_column_ptr()->convert_to_full_column_if_const();
+ if (const auto* nullable = check_and_get_column<ColumnNullable>(*col)) {
+ if (nullable->is_null_at(0)) {
+ return std::nullopt;
+ }
+ col = nullable->get_nested_column_ptr();
+ }
+
+ Field field;
+ col->get(0, field);
+
+ switch (slot_primitive) {
+ case TYPE_BOOLEAN: {
+ if (literal_primitive != TYPE_BOOLEAN) {
+ return std::nullopt;
+ }
+ return paimon::Literal(static_cast<bool>(field.get<bool>()));
+ }
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT: {
+ if (!_is_integer_type(literal_primitive)) {
+ return std::nullopt;
+ }
+ int64_t value = 0;
+ switch (literal_primitive) {
+ case TYPE_TINYINT:
+ value = field.get<int8_t>();
+ break;
+ case TYPE_SMALLINT:
+ value = field.get<int16_t>();
+ break;
+ case TYPE_INT:
+ value = field.get<int32_t>();
+ break;
+ case TYPE_BIGINT:
+ value = field.get<int64_t>();
+ break;
+ default:
+ return std::nullopt;
+ }
+ if (slot_primitive == TYPE_TINYINT) {
+ return paimon::Literal(static_cast<int8_t>(value));
+ }
+ if (slot_primitive == TYPE_SMALLINT) {
+ return paimon::Literal(static_cast<int16_t>(value));
+ }
+ if (slot_primitive == TYPE_INT) {
+ return paimon::Literal(static_cast<int32_t>(value));
+ }
+ return paimon::Literal(static_cast<int64_t>(value));
+ }
+ case TYPE_DOUBLE: {
+ if (literal_primitive != TYPE_DOUBLE && literal_primitive != TYPE_FLOAT) {
+ return std::nullopt;
+ }
+ double value = 0;
+ if (literal_primitive == TYPE_FLOAT) {
+ value = static_cast<double>(field.get<float>());
+ } else {
+ value = field.get<double>();
+ }
+ return paimon::Literal(value);
+ }
+ case TYPE_DATE:
+ case TYPE_DATEV2: {
+ if (!_is_date_type(literal_primitive)) {
+ return std::nullopt;
+ }
+ int64_t seconds = 0;
+ if (literal_primitive == TYPE_DATE) {
+ Int64 v = field.get<Int64>();
+ auto* dt = reinterpret_cast<VecDateTimeValue*>(&v);
+ if (!dt->is_valid_date()) {
+ return std::nullopt;
+ }
+ dt->unix_timestamp(&seconds, _gmt_tz);
+ } else if (literal_primitive == TYPE_DATEV2) {
+ const auto& dt = field.get<PrimitiveTypeTraits<TYPE_DATEV2>::CppType>();
+ if (!dt.is_valid_date()) {
+ return std::nullopt;
+ }
+ dt.unix_timestamp(&seconds, _gmt_tz);
+ }
+ int32_t days = _seconds_to_days(seconds);
+ return paimon::Literal(paimon::FieldType::DATE, days);
+ }
+ case TYPE_DATETIME:
+ case TYPE_DATETIMEV2: {
+ if (!_is_datetime_type(literal_primitive)) {
+ return std::nullopt;
+ }
+ if (literal_primitive == TYPE_DATETIME) {
+ Int64 v = field.get<Int64>();
+ auto* dt = reinterpret_cast<VecDateTimeValue*>(&v);
+ if (!dt->is_valid_date()) {
+ return std::nullopt;
+ }
+ int64_t seconds = 0;
+ dt->unix_timestamp(&seconds, _gmt_tz);
+ return paimon::Literal(paimon::Timestamp::FromEpochMillis(seconds * 1000));
+ }
+ std::pair<int64_t, int64_t> ts;
+ const auto& dt = field.get<PrimitiveTypeTraits<TYPE_DATETIMEV2>::CppType>();
+ if (!dt.is_valid_date()) {
+ return std::nullopt;
+ }
+ dt.unix_timestamp(&ts, _gmt_tz);
+ int64_t millis = ts.first * 1000 + ts.second / 1000;
+ return paimon::Literal(paimon::Timestamp::FromEpochMillis(millis));
+ }
+ case TYPE_VARCHAR:
+ case TYPE_STRING: {
+ if (!_is_string_type(literal_primitive)) {
+ return std::nullopt;
+ }
+ const auto& value = field.get<String>();
+ return paimon::Literal(field_type, value.data(), value.size());
+ }
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256: {
+ if (!_is_decimal_type(literal_primitive)) {
+ return std::nullopt;
+ }
+ int32_t precision = literal_type.precision;
+ int32_t scale = literal_type.scale;
+ if (precision <= 0 || precision > paimon::Decimal::MAX_PRECISION) {
+ return std::nullopt;
+ }
+
+ paimon::Decimal::int128_t value = 0;
+ switch (literal_primitive) {
+ case TYPE_DECIMALV2: {
+ const auto& dec = field.get<DecimalField<Decimal128V2>>();
+ value = dec.get_value();
+ break;
+ }
+ case TYPE_DECIMAL32: {
+ const auto& dec = field.get<DecimalField<Decimal32>>();
+ value = dec.get_value();
+ break;
+ }
+ case TYPE_DECIMAL64: {
+ const auto& dec = field.get<DecimalField<Decimal64>>();
+ value = dec.get_value();
+ break;
+ }
+ case TYPE_DECIMAL128I: {
+ const auto& dec = field.get<DecimalField<Decimal128V3>>();
+ value = dec.get_value();
+ break;
+ }
+ default:
+ return std::nullopt;
+ }
+ return paimon::Literal(paimon::Decimal(precision, scale, value));
+ }
+ default:
+ break;
+ }
+ return std::nullopt;
+}
+
+std::optional<std::string> PaimonPredicateConverter::_extract_string_literal(
+ const VExprSPtr& expr) const {
+ auto literal_expr = VExpr::expr_without_cast(expr);
+ auto* literal = dynamic_cast<VLiteral*>(literal_expr.get());
+ if (!literal) {
+ return std::nullopt;
+ }
+ PrimitiveType literal_primitive = literal_expr->type().type;
+ if (!_is_string_type(literal_primitive)) {
+ return std::nullopt;
+ }
+
+ ColumnPtr col = literal->get_column_ptr()->convert_to_full_column_if_const();
+ if (const auto* nullable = check_and_get_column<ColumnNullable>(*col)) {
+ if (nullable->is_null_at(0)) {
+ return std::nullopt;
+ }
+ col = nullable->get_nested_column_ptr();
+ }
+ Field field;
+ col->get(0, field);
+ const auto& value = field.get<String>();
+ return value;
+}
+
+std::string PaimonPredicateConverter::_normalize_name(std::string_view name) {
+ std::string out(name);
+ std::transform(out.begin(), out.end(), out.begin(),
+ [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+ return out;
+}
+
+std::optional<std::string> PaimonPredicateConverter::_next_prefix(const std::string& prefix) {
+ if (prefix.empty()) {
+ return std::nullopt;
+ }
+ std::string upper = prefix;
+ for (int i = static_cast<int>(upper.size()) - 1; i >= 0; --i) {
+ auto c = static_cast<unsigned char>(upper[i]);
+ if (c != 0xFF) {
+ upper[i] = static_cast<char>(c + 1);
+ upper.resize(i + 1);
+ return upper;
+ }
+ }
+ return std::nullopt;
+}
+
+int32_t PaimonPredicateConverter::_seconds_to_days(int64_t seconds) {
+ static constexpr int64_t kSecondsPerDay = 24 * 60 * 60;
+ int64_t days = seconds / kSecondsPerDay;
+ if (seconds < 0 && seconds % kSecondsPerDay != 0) {
+ --days;
+ }
+ return static_cast<int32_t>(days);
+}
+
+bool PaimonPredicateConverter::_is_integer_type(PrimitiveType type) {
+ switch (type) {
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
+ case TYPE_INT:
+ case TYPE_BIGINT:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool PaimonPredicateConverter::_is_string_type(PrimitiveType type) {
+ return type == TYPE_CHAR || type == TYPE_VARCHAR || type == TYPE_STRING;
+}
+
+bool PaimonPredicateConverter::_is_decimal_type(PrimitiveType type) {
+ switch (type) {
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool PaimonPredicateConverter::_is_date_type(PrimitiveType type) {
+ return type == TYPE_DATE || type == TYPE_DATEV2;
+}
+
+bool PaimonPredicateConverter::_is_datetime_type(PrimitiveType type) {
+ return type == TYPE_DATETIME || type == TYPE_DATETIMEV2;
+}
+
+std::optional<paimon::FieldType> PaimonPredicateConverter::_to_paimon_field_type(
+ PrimitiveType type, uint32_t precision) {
+ switch (type) {
+ case TYPE_BOOLEAN:
+ return paimon::FieldType::BOOLEAN;
+ case TYPE_TINYINT:
+ return paimon::FieldType::TINYINT;
+ case TYPE_SMALLINT:
+ return paimon::FieldType::SMALLINT;
+ case TYPE_INT:
+ return paimon::FieldType::INT;
+ case TYPE_BIGINT:
+ return paimon::FieldType::BIGINT;
+ case TYPE_DOUBLE:
+ return paimon::FieldType::DOUBLE;
+ case TYPE_VARCHAR:
+ case TYPE_STRING:
+ return paimon::FieldType::STRING;
+ case TYPE_DATE:
+ case TYPE_DATEV2:
+ return paimon::FieldType::DATE;
+ case TYPE_DATETIME:
+ case TYPE_DATETIMEV2:
+ return paimon::FieldType::TIMESTAMP;
+ case TYPE_DECIMALV2:
+ case TYPE_DECIMAL32:
+ case TYPE_DECIMAL64:
+ case TYPE_DECIMAL128I:
+ case TYPE_DECIMAL256:
+ if (precision > 0 && precision > paimon::Decimal::MAX_PRECISION) {
+ return std::nullopt;
+ }
+ return paimon::FieldType::DECIMAL;
+ case TYPE_FLOAT:
+ case TYPE_CHAR:
+ default:
+ return std::nullopt;
+ }
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/paimon_predicate_converter.h b/be/src/vec/exec/format/table/paimon_predicate_converter.h
new file mode 100644
index 00000000000..9419d32c5e8
--- /dev/null
+++ b/be/src/vec/exec/format/table/paimon_predicate_converter.h
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "cctz/time_zone.h"
+#include "paimon/defs.h"
+#include "paimon/predicate/literal.h"
+#include "runtime/define_primitive_type.h"
+#include "vec/exprs/vexpr_fwd.h"
+
+namespace paimon {
+class Predicate;
+} // namespace paimon
+
+namespace doris {
+class RuntimeState;
+class SlotDescriptor;
+} // namespace doris
+
+namespace doris::vectorized {
+
+class PaimonPredicateConverter {
+public:
+ PaimonPredicateConverter(const std::vector<SlotDescriptor*>& file_slot_descs,
+ RuntimeState* state);
+
+ std::shared_ptr<paimon::Predicate> build(const VExprContextSPtrs& conjuncts);
+
+private:
+ struct FieldMeta {
+ int32_t index = -1;
+ paimon::FieldType field_type = paimon::FieldType::UNKNOWN;
+ const SlotDescriptor* slot_desc = nullptr;
+ };
+
+ std::shared_ptr<paimon::Predicate> _convert_expr(const VExprSPtr& expr);
+ std::shared_ptr<paimon::Predicate> _convert_compound(const VExprSPtr& expr);
+ std::shared_ptr<paimon::Predicate> _convert_in(const VExprSPtr& expr);
+ std::shared_ptr<paimon::Predicate> _convert_binary(const VExprSPtr& expr);
+ std::shared_ptr<paimon::Predicate> _convert_is_null(const VExprSPtr& expr,
+ const std::string& fn_name);
+ std::shared_ptr<paimon::Predicate> _convert_like_prefix(const VExprSPtr& expr);
+
+ std::optional<FieldMeta> _resolve_field(const VExprSPtr& expr) const;
+ std::optional<paimon::Literal> _convert_literal(const VExprSPtr& expr,
+ const SlotDescriptor& slot_desc,
+ paimon::FieldType field_type) const;
+ std::optional<std::string> _extract_string_literal(const VExprSPtr& expr) const;
+
+ static std::string _normalize_name(std::string_view name);
+ static std::optional<std::string> _next_prefix(const std::string& prefix);
+ static int32_t _seconds_to_days(int64_t seconds);
+ static bool _is_integer_type(PrimitiveType type);
+ static bool _is_string_type(PrimitiveType type);
+ static bool _is_decimal_type(PrimitiveType type);
+ static bool _is_date_type(PrimitiveType type);
+ static bool _is_datetime_type(PrimitiveType type);
+ static std::optional<paimon::FieldType> _to_paimon_field_type(PrimitiveType type,
+ uint32_t precision);
+
+ std::unordered_map<std::string, int32_t> _field_index_by_name;
+ RuntimeState* _state = nullptr;
+ cctz::time_zone _gmt_tz;
+};
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp
index dfd3397d270..7d651ce7da8 100644
--- a/be/src/vec/exec/scan/vfile_scanner.cpp
+++ b/be/src/vec/exec/scan/vfile_scanner.cpp
@@ -59,7 +59,9 @@
#include "vec/exec/format/table/hudi_jni_reader.h"
#include "vec/exec/format/table/iceberg_reader.h"
#include "vec/exec/format/table/max_compute_jni_reader.h"
+#include "vec/exec/format/table/paimon_cpp_reader.h"
#include "vec/exec/format/table/paimon_jni_reader.h"
+#include "vec/exec/format/table/paimon_predicate_converter.h"
#include "vec/exec/format/table/paimon_reader.h"
#include "vec/exec/format/table/transactional_hive_reader.h"
#include "vec/exec/format/wal/wal_reader.h"
@@ -813,10 +815,26 @@ Status VFileScanner::_get_next_reader() {
_cur_reader = std::move(mc_reader);
} else if (range.__isset.table_format_params &&
range.table_format_params.table_format_type == "paimon") {
- _cur_reader = PaimonJniReader::create_unique(_file_slot_descs, _state, _profile,
- range, _params);
- init_status = ((PaimonJniReader*)(_cur_reader.get()))
- ->init_reader(_colname_to_value_range);
+ if (_state->query_options().__isset.enable_paimon_cpp_reader &&
+ _state->query_options().enable_paimon_cpp_reader) {
+ auto cpp_reader = PaimonCppReader::create_unique(_file_slot_descs, _state,
+ _profile, range, _params);
+ cpp_reader->set_push_down_agg_type(_get_push_down_agg_type());
+ if (!_is_load && !_push_down_conjuncts.empty()) {
+ PaimonPredicateConverter predicate_converter(_file_slot_descs, _state);
+ auto predicate = predicate_converter.build(_push_down_conjuncts);
+ if (predicate) {
+ cpp_reader->set_predicate(std::move(predicate));
+ }
+ }
+ init_status = cpp_reader->init_reader();
+ _cur_reader = std::move(cpp_reader);
+ } else {
+ _cur_reader = PaimonJniReader::create_unique(_file_slot_descs, _state, _profile,
+ range, _params);
+ init_status = ((PaimonJniReader*)(_cur_reader.get()))
+ ->init_reader(_colname_to_value_range);
+ }
} else if (range.__isset.table_format_params &&
range.table_format_params.table_format_type == "hudi") {
_cur_reader = HudiJniReader::create_unique(*_params,
diff --git a/be/test/vec/exec/format/table/paimon_cpp_reader_test.cpp b/be/test/vec/exec/format/table/paimon_cpp_reader_test.cpp
new file mode 100644
index 00000000000..95fb22d9b65
--- /dev/null
+++ b/be/test/vec/exec/format/table/paimon_cpp_reader_test.cpp
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/paimon_cpp_reader.h"
+
+#include <gtest/gtest.h>
+
+#include <string>
+#include <vector>
+
+#include "runtime/runtime_state.h"
+#include "util/runtime_profile.h"
+#include "vec/core/block.h"
+
+namespace doris::vectorized {
+
+class PaimonCppReaderTest : public testing::Test {
+protected:
+ void SetUp() override {
+ _query_options.__set_batch_size(3);
+ _runtime_state = std::make_unique<RuntimeState>(_query_options, _query_globals);
+ }
+
+ TFileRangeDesc _build_range_with_table_level_row_count(int64_t row_count) {
+ TFileRangeDesc range;
+ range.__isset.table_format_params = true;
+ range.table_format_params.__isset.table_level_row_count = true;
+ range.table_format_params.table_level_row_count = row_count;
+ return range;
+ }
+
+ TQueryOptions _query_options;
+ TQueryGlobals _query_globals;
+ std::unique_ptr<RuntimeState> _runtime_state;
+ RuntimeProfile _profile {"paimon_cpp_reader_test"};
+ std::vector<SlotDescriptor*> _file_slot_descs;
+};
+
+TEST_F(PaimonCppReaderTest, CountPushDownUsesTableLevelRowCount) {
+ auto range = _build_range_with_table_level_row_count(5);
+ PaimonCppReader reader(_file_slot_descs, _runtime_state.get(), &_profile, range, nullptr);
+ reader.set_push_down_agg_type(TPushAggOp::type::COUNT);
+
+ auto init_status = reader.init_reader();
+ ASSERT_TRUE(init_status.ok()) << init_status;
+
+ Block block;
+ size_t read_rows = 0;
+ bool eof = false;
+
+ auto first_status = reader.get_next_block(&block, &read_rows, &eof);
+ ASSERT_TRUE(first_status.ok()) << first_status;
+ EXPECT_EQ(3, read_rows);
+ EXPECT_FALSE(eof);
+
+ auto second_status = reader.get_next_block(&block, &read_rows, &eof);
+ ASSERT_TRUE(second_status.ok()) << second_status;
+ EXPECT_EQ(2, read_rows);
+ EXPECT_TRUE(eof);
+
+ auto third_status = reader.get_next_block(&block, &read_rows, &eof);
+ ASSERT_TRUE(third_status.ok()) << third_status;
+ EXPECT_EQ(0, read_rows);
+ EXPECT_TRUE(eof);
+}
+
+TEST_F(PaimonCppReaderTest, InitReaderFailsWithoutPaimonSplit) {
+ TFileRangeDesc range;
+ range.__isset.table_format_params = true;
+ range.table_format_params.__isset.paimon_params = true;
+ range.table_format_params.paimon_params.__isset.paimon_table = true;
+ range.table_format_params.paimon_params.paimon_table = "s3://bucket/db.tbl";
+
+ PaimonCppReader reader(_file_slot_descs, _runtime_state.get(), &_profile, range, nullptr);
+ auto status = reader.init_reader();
+
+ ASSERT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("missing paimon_split"), std::string::npos);
+}
+
+} // namespace doris::vectorized
diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonUtils.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonUtils.java
index 44ffb298c98..d1d6c2f9bdb 100644
--- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonUtils.java
+++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonUtils.java
@@ -26,7 +26,8 @@ import java.util.List;
import java.util.stream.Collectors;
public class PaimonUtils {
- private static final Base64.Decoder DECODER = Base64.getUrlDecoder();
+ private static final Base64.Decoder URL_DECODER = Base64.getUrlDecoder();
+ private static final Base64.Decoder STD_DECODER = Base64.getDecoder();
public static List<String> getFieldNames(RowType rowType) {
return rowType.getFields().stream()
@@ -37,9 +38,14 @@ public class PaimonUtils {
public static <T> T deserialize(String encodedStr) {
try {
- return InstantiationUtil.deserializeObject(
- DECODER.decode(encodedStr.getBytes(java.nio.charset.StandardCharsets.UTF_8)),
- PaimonUtils.class.getClassLoader());
+ byte[] decoded;
+ try {
+ decoded = URL_DECODER.decode(encodedStr.getBytes(java.nio.charset.StandardCharsets.UTF_8));
+ } catch (IllegalArgumentException e) {
+ // Fallback to standard Base64 for splits encoded by native Paimon serialization.
+ decoded = STD_DECODER.decode(encodedStr.getBytes(java.nio.charset.StandardCharsets.UTF_8));
+ }
+ return InstantiationUtil.deserializeObject(decoded, PaimonUtils.class.getClassLoader());
} catch (Throwable e) {
throw new RuntimeException(e);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
index 8df214482cc..6b2ab053f6b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
@@ -46,6 +46,7 @@ import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.io.DataOutputViewStreamWrapper;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.table.source.DataSplit;
import org.apache.paimon.table.source.DeletionFile;
@@ -53,6 +54,7 @@ import org.apache.paimon.table.source.RawFile;
import org.apache.paimon.table.source.ReadBuilder;
import org.apache.paimon.utils.InstantiationUtil;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Base64;
@@ -147,6 +149,20 @@ public class PaimonScanNode extends FileQueryScanNode {
}
}
+ /**
+ * Serialize DataSplit using Paimon's native binary format for paimon-cpp reader.
+ */
+ public static String encodeDataSplitToString(DataSplit split) {
+ try {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos);
+ split.serialize(out);
+ return Base64.getEncoder().encodeToString(baos.toByteArray());
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to serialize DataSplit using Paimon native format", e);
+ }
+ }
+
@Override
protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
if (split instanceof PaimonSplit) {
@@ -167,9 +183,17 @@ public class PaimonScanNode extends FileQueryScanNode {
String fileFormat = getFileFormat(paimonSplit.getPathString());
if (split != null) {
- // use jni reader
+ // use jni reader or paimon-cpp reader
rangeDesc.setFormatType(TFileFormatType.FORMAT_JNI);
- fileDesc.setPaimonSplit(encodeObjectToString(split));
+ if (sessionVariable.isEnablePaimonCppReader() && split instanceof DataSplit) {
+ fileDesc.setPaimonSplit(encodeDataSplitToString((DataSplit) split));
+ } else {
+ fileDesc.setPaimonSplit(encodeObjectToString(split));
+ }
+ String tableLocation = source.getTableLocation();
+ if (tableLocation != null) {
+ fileDesc.setPaimonTable(tableLocation);
+ }
} else {
// use native reader
if (fileFormat.equals("orc")) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSource.java
index 1c6b88b16ec..38ffe002624 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSource.java
@@ -27,6 +27,7 @@ import org.apache.doris.datasource.property.constants.PaimonProperties;
import org.apache.doris.thrift.TFileAttributes;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.Table;
@@ -71,4 +72,12 @@ public class PaimonSource {
public String getFileFormatFromTableProperties() {
return originTable.options().getOrDefault(PaimonProperties.FILE_FORMAT, "parquet");
}
+
+ public String getTableLocation() {
+ if (originTable instanceof FileStoreTable) {
+ return ((FileStoreTable) originTable).location().toString();
+ }
+ // Fallback to path option
+ return originTable.options().get("path");
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index c33e26d0b06..91690fd064c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -612,6 +612,8 @@ public class SessionVariable implements Serializable, Writable {
public static final String FORCE_JNI_SCANNER = "force_jni_scanner";
+ public static final String ENABLE_PAIMON_CPP_READER = "enable_paimon_cpp_reader";
+
public static final String HUDI_JNI_SCANNER = "hudi_jni_scanner";
public static final String ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE = "enable_count_push_down_for_external_table";
@@ -956,13 +958,14 @@ public class SessionVariable implements Serializable, Writable {
public enum IgnoreSplitType {
NONE,
IGNORE_JNI,
- IGNORE_NATIVE
+ IGNORE_NATIVE,
+ IGNORE_PAIMON_CPP
}
public static final String IGNORE_SPLIT_TYPE = "ignore_split_type";
@VariableMgr.VarAttr(name = IGNORE_SPLIT_TYPE,
checker = "checkIgnoreSplitType",
- options = {"NONE", "IGNORE_JNI", "IGNORE_NATIVE"},
+ options = {"NONE", "IGNORE_JNI", "IGNORE_NATIVE", "IGNORE_PAIMON_CPP"},
description = {"忽略指定类型的split", "Ignore splits of the specified type"})
public String ignoreSplitType = IgnoreSplitType.NONE.toString();
@@ -2017,6 +2020,11 @@ public class SessionVariable implements Serializable, Writable {
description = {"强制使用jni方式读取外表", "Force the use of jni mode to read external table"})
private boolean forceJniScanner = false;
+ @VariableMgr.VarAttr(name = ENABLE_PAIMON_CPP_READER,
+ fuzzy = true,
+ description = {"Paimon 非原生文件读取使用 paimon-cpp", "Use paimon-cpp for non-native Paimon reads"})
+ private boolean enablePaimonCppReader = false;
+
@VariableMgr.VarAttr(name = HUDI_JNI_SCANNER, description = { "使用那种hudi jni scanner, 'hadoop' 或 'spark'",
"Which hudi jni scanner to use, 'hadoop' or 'spark'" })
private String hudiJniScanner = "spark";
@@ -3870,6 +3878,7 @@ public class SessionVariable implements Serializable, Writable {
tResult.setOrcTinyStripeThresholdBytes(orcTinyStripeThresholdBytes);
tResult.setOrcMaxMergeDistanceBytes(orcMaxMergeDistanceBytes);
tResult.setOrcOnceMaxReadBytes(orcOnceMaxReadBytes);
+ tResult.setEnablePaimonCppReader(enablePaimonCppReader);
tResult.setNewIsIpAddressInRange(newIsIpAddressInRange);
@@ -4473,6 +4482,10 @@ public class SessionVariable implements Serializable, Writable {
return hudiJniScanner;
}
+ public boolean isEnablePaimonCppReader() {
+ return enablePaimonCppReader;
+ }
+
public String getIgnoreSplitType() {
return ignoreSplitType;
}
@@ -4481,7 +4494,8 @@ public class SessionVariable implements Serializable, Writable {
try {
IgnoreSplitType.valueOf(value);
} catch (Exception e) {
- throw new UnsupportedOperationException("We only support `NONE`, `IGNORE_JNI` and `IGNORE_NATIVE`");
+ throw new UnsupportedOperationException(
+ "We only support `NONE`, `IGNORE_JNI`, `IGNORE_NATIVE` and `IGNORE_PAIMON_CPP`");
}
}
@@ -4497,6 +4511,10 @@ public class SessionVariable implements Serializable, Writable {
this.hudiJniScanner = hudiJniScanner;
}
+ public void setEnablePaimonCppReader(boolean enable) {
+ enablePaimonCppReader = enable;
+ }
+
public boolean isEnableCountPushDownForExternalTable() {
return enableCountPushDownForExternalTable;
}
diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift
index c612826836e..c0e6d5dc1d0 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -343,6 +343,8 @@ struct TQueryOptions {
// upgrade options. keep them same in every branch.
200: optional bool new_is_ip_address_in_range = false;
+ // Use paimon-cpp to read Paimon splits on BE
+ 201: optional bool enable_paimon_cpp_reader = false;
// For cloud, to control if the content would be written into file cache
1000: optional bool disable_file_cache = false
diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_cpp_reader.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_cpp_reader.groovy
new file mode 100644
index 00000000000..64ef323add6
--- /dev/null
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_cpp_reader.groovy
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_paimon_cpp_reader", "p0,external") {
+ String enabled = context.config.otherConfigs.get("enablePaimonTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disabled paimon test")
+ return
+ }
+
+ String catalogName = "test_paimon_cpp_reader"
+ String hdfsPort = context.config.otherConfigs.get("hive2HdfsPort")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+
+ try {
+ sql """drop catalog if exists ${catalogName}"""
+ sql """create catalog if not exists ${catalogName} properties (
+ "type" = "paimon",
+ "paimon.catalog.type" = "filesystem",
+ "warehouse" = "hdfs://${externalEnvIp}:${hdfsPort}/user/doris/paimon1"
+ );"""
+ sql """switch ${catalogName}"""
+ sql """use db1"""
+ // Do not force JNI; keep default selection behavior.
+ sql """set force_jni_scanner=false"""
+
+ def testQueries = [
+ """select c1 from complex_all order by c1""",
+ """select c1 from complex_all where c1 >= 2 order by c1""",
+ """select * from all_table order by c1""",
+ """select * from all_table_with_parquet where c13 like '13%' order by c1""",
+ """select * from complex_tab order by c1""",
+ """select c3['a_test'], c3['b_test'], c3['bbb'], c3['ccc'] from complex_tab order by c3['a_test'], c3['b_test']""",
+ """select array_max(c2) c from complex_tab order by c""",
+ """select c20[0] c from complex_all order by c""",
+ """select * from deletion_vector_orc""",
+ """select * from deletion_vector_parquet"""
+ ]
+
+ // Default path is JNI when enable_paimon_cpp_reader=false.
+ sql """set enable_paimon_cpp_reader=false"""
+ def jniResults = testQueries.collect { query -> sql(query) }
+
+ sql """set enable_paimon_cpp_reader=true"""
+ def cppResults = testQueries.collect { query -> sql(query) }
+
+ assertTrue(cppResults[0].size() > 0)
+ for (int i = 0; i < testQueries.size(); i++) {
+ assertEquals(jniResults[i].toString(), cppResults[i].toString())
+ }
+ } finally {
+ sql """set enable_paimon_cpp_reader=false"""
+ sql """set force_jni_scanner=false"""
+ sql """drop catalog if exists ${catalogName}"""
+ }
+}
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index abdad19a97c..364568f0a45 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -1032,6 +1032,10 @@ build_arrow() {
-DARROW_BUILD_STATIC=ON -DARROW_WITH_BROTLI=ON -DARROW_WITH_LZ4=ON -DARROW_USE_GLOG=ON \
-DARROW_WITH_SNAPPY=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DARROW_JSON=ON \
-DARROW_WITH_UTF8PROC=OFF -DARROW_WITH_RE2=ON -DARROW_ORC=ON \
+ -DARROW_COMPUTE=ON \
+ -DARROW_FILESYSTEM=ON \
+ -DARROW_DATASET=ON \
+ -DARROW_ACERO=ON \
-DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
-DCMAKE_INSTALL_LIBDIR=lib64 \
-DARROW_BOOST_USE_SHARED=OFF \
@@ -1079,6 +1083,8 @@ build_arrow() {
cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlicommon-static.a "${TP_INSTALL_DIR}/lib64/libbrotlicommon.a"
strip_lib libarrow.a
strip_lib libparquet.a
+ strip_lib libarrow_dataset.a
+ strip_lib libarrow_acero.a
}
# abseil
@@ -1821,6 +1827,107 @@ build_jindofs() {
cp -r ${TP_SOURCE_DIR}/${JINDOFS_SOURCE}/* "${TP_INSTALL_DIR}/jindofs_libs/"
}
+# pugixml
+build_pugixml() {
+ check_if_source_exist "${PUGIXML_SOURCE}"
+ cd "${TP_SOURCE_DIR}/${PUGIXML_SOURCE}"
+
+ rm -rf "${BUILD_DIR}"
+ mkdir -p "${BUILD_DIR}"
+ cd "${BUILD_DIR}"
+
+ "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
+ -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release ..
+ "${BUILD_SYSTEM}" -j "${PARALLEL}"
+ "${BUILD_SYSTEM}" install
+
+ cp "${TP_SOURCE_DIR}/${PUGIXML_SOURCE}/src/pugixml.hpp" "${TP_INSTALL_DIR}/include/"
+ cp "${TP_SOURCE_DIR}/${PUGIXML_SOURCE}/src/pugiconfig.hpp" "${TP_INSTALL_DIR}/include/"
+}
+
+# paimon-cpp
+build_paimon_cpp() {
+ check_if_source_exist "${PAIMON_CPP_SOURCE}"
+ cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}"
+
+ rm -rf "${BUILD_DIR}"
+ mkdir -p "${BUILD_DIR}"
+ cd "${BUILD_DIR}"
+
+ # Darwin doesn't build GNU libunwind in this script, so don't force -lunwind there.
+ local paimon_linker_flags="-L${TP_LIB_DIR} -lbrotlienc -lbrotlidec -lbrotlicommon -llzma"
+ if [[ "${KERNEL}" != 'Darwin' ]]; then
+ paimon_linker_flags="${paimon_linker_flags} -lunwind"
+ fi
+
+ CXXFLAGS="-Wno-nontrivial-memcall" \
+ "${CMAKE_CMD}" -C "${TP_DIR}/paimon-cpp-cache.cmake" \
+ -G "${GENERATOR}" \
+ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
+ -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
+ -DPAIMON_BUILD_SHARED=OFF \
+ -DPAIMON_BUILD_STATIC=ON \
+ -DPAIMON_BUILD_TESTS=OFF \
+ -DPAIMON_ENABLE_ORC=ON \
+ -DPAIMON_ENABLE_AVRO=OFF \
+ -DPAIMON_ENABLE_LANCE=OFF \
+ -DPAIMON_ENABLE_JINDO=OFF \
+ -DPAIMON_ENABLE_LUMINA=OFF \
+ -DPAIMON_ENABLE_LUCENE=OFF \
+ -DCMAKE_EXE_LINKER_FLAGS="${paimon_linker_flags}" \
+ -DCMAKE_SHARED_LINKER_FLAGS="${paimon_linker_flags}" \
+ ..
+ "${BUILD_SYSTEM}" -j "${PARALLEL}"
+ "${BUILD_SYSTEM}" install
+
+ # Install paimon-cpp internal dependencies with renamed versions
+ # These libraries are built but not installed by default
+ echo "Installing paimon-cpp internal dependencies..."
+
+ # Arrow deps: When PAIMON_USE_EXTERNAL_ARROW=ON (Plan B), paimon-cpp
+ # reuses Doris's Arrow and does NOT build arrow_ep, so the paimon_deps
+ # directory is not needed. When building its own Arrow (legacy), copy
+ # arrow artefacts into an isolated directory to avoid clashing with Doris.
+ local paimon_deps_dir="${TP_INSTALL_DIR}/paimon-cpp/lib64/paimon_deps"
+ if [ -d "arrow_ep-install/lib" ]; then
+ mkdir -p "${paimon_deps_dir}"
+ for paimon_arrow_dep in \
+ libarrow.a \
+ libarrow_filesystem.a \
+ libarrow_dataset.a \
+ libarrow_acero.a \
+ libparquet.a; do
+ if [ -f "arrow_ep-install/lib/${paimon_arrow_dep}" ]; then
+ cp -v "arrow_ep-install/lib/${paimon_arrow_dep}" "${paimon_deps_dir}/${paimon_arrow_dep}"
+ fi
+ done
+ else
+ echo " arrow_ep-install not found (PAIMON_USE_EXTERNAL_ARROW=ON?) – skipping paimon_deps Arrow copy"
+ fi
+
+ # Install roaring_bitmap, renamed to avoid conflict with Doris's croaringbitmap
+ if [ -f "release/libroaring_bitmap.a" ]; then
+ cp -v "release/libroaring_bitmap.a" "${TP_INSTALL_DIR}/lib64/libroaring_bitmap_paimon.a"
+ fi
+
+ # Install xxhash, renamed to avoid conflict with Doris's xxhash
+ if [ -f "release/libxxhash.a" ]; then
+ cp -v "release/libxxhash.a" "${TP_INSTALL_DIR}/lib64/libxxhash_paimon.a"
+ fi
+
+ # Install fmt v11 (from fmt_ep-install directory, renamed to avoid conflict with Doris's fmt v7)
+ if [ -f "fmt_ep-install/lib/libfmt.a" ]; then
+ cp -v "fmt_ep-install/lib/libfmt.a" "${TP_INSTALL_DIR}/lib64/libfmt_paimon.a"
+ fi
+
+ # Install tbb (from tbb_ep-install directory, renamed to avoid conflict with Doris's tbb)
+ if [ -f "tbb_ep-install/lib/libtbb.a" ]; then
+ cp -v "tbb_ep-install/lib/libtbb.a" "${TP_INSTALL_DIR}/lib64/libtbb_paimon.a"
+ fi
+
+ echo "Paimon-cpp internal dependencies installed successfully"
+}
+
if [[ "${#packages[@]}" -eq 0 ]]; then
packages=(
jindofs
@@ -1890,6 +1997,8 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
base64
brotli
icu
+ pugixml
+ paimon_cpp
)
if [[ "$(uname -s)" == 'Darwin' ]]; then
read -r -a packages <<<"binutils gettext ${packages[*]}"
@@ -1898,6 +2007,122 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
fi
fi
+# Map a package name to its source directory variable(s) and remove them to free disk space.
+# This is called after each package is built and installed successfully.
+cleanup_package_source() {
+ local pkg="$1"
+ local src_var
+ local src_dir
+
+ # Map package name to the uppercase *_SOURCE variable name
+ case "${pkg}" in
+ libevent) src_var="LIBEVENT_SOURCE" ;;
+ openssl) src_var="OPENSSL_SOURCE" ;;
+ thrift) src_var="THRIFT_SOURCE" ;;
+ protobuf) src_var="PROTOBUF_SOURCE" ;;
+ gflags) src_var="GFLAGS_SOURCE" ;;
+ glog) src_var="GLOG_SOURCE" ;;
+ gtest) src_var="GTEST_SOURCE" ;;
+ rapidjson) src_var="RAPIDJSON_SOURCE" ;;
+ snappy) src_var="SNAPPY_SOURCE" ;;
+ gperftools) src_var="GPERFTOOLS_SOURCE" ;;
+ zlib) src_var="ZLIB_SOURCE" ;;
+ crc32c) src_var="CRC32C_SOURCE" ;;
+ lz4) src_var="LZ4_SOURCE" ;;
+ bzip) src_var="BZIP_SOURCE" ;;
+ lzo2) src_var="LZO2_SOURCE" ;;
+ zstd) src_var="ZSTD_SOURCE" ;;
+ #boost) src_var="BOOST_SOURCE" ;; // boost is used for mysql later
+ abseil) src_var="ABSEIL_SOURCE" ;;
+ curl) src_var="CURL_SOURCE" ;;
+ re2) src_var="RE2_SOURCE" ;;
+ hyperscan)
+ # hyperscan also builds ragel, clean both
+ if [[ -n "${RAGEL_SOURCE}" && -d "${TP_SOURCE_DIR}/${RAGEL_SOURCE}" ]]; then
+ echo "Cleaning up source: ${RAGEL_SOURCE}"
+ rm -rf "${TP_SOURCE_DIR}/${RAGEL_SOURCE}"
+ fi
+ src_var="HYPERSCAN_SOURCE"
+ ;;
+ mysql) src_var="MYSQL_SOURCE" ;;
+ odbc) src_var="ODBC_SOURCE" ;;
+ leveldb) src_var="LEVELDB_SOURCE" ;;
+ brpc) src_var="BRPC_SOURCE" ;;
+ rocksdb) src_var="ROCKSDB_SOURCE" ;;
+ cyrus_sasl) src_var="CYRUS_SASL_SOURCE" ;;
+ librdkafka) src_var="LIBRDKAFKA_SOURCE" ;;
+ flatbuffers) src_var="FLATBUFFERS_SOURCE" ;;
+ arrow) src_var="ARROW_SOURCE" ;;
+ brotli) src_var="BROTLI_SOURCE" ;;
+ cares) src_var="CARES_SOURCE" ;;
+ grpc) src_var="GRPC_SOURCE" ;;
+ s2) src_var="S2_SOURCE" ;;
+ bitshuffle) src_var="BITSHUFFLE_SOURCE" ;;
+ croaringbitmap) src_var="CROARINGBITMAP_SOURCE" ;;
+ fmt) src_var="FMT_SOURCE" ;;
+ parallel_hashmap) src_var="PARALLEL_HASHMAP_SOURCE" ;;
+ orc) src_var="ORC_SOURCE" ;;
+ cctz) src_var="CCTZ_SOURCE" ;;
+ jemalloc_doris) src_var="JEMALLOC_DORIS_SOURCE" ;;
+ libunwind) src_var="LIBUNWIND_SOURCE" ;;
+ benchmark) src_var="BENCHMARK_SOURCE" ;;
+ simdjson) src_var="SIMDJSON_SOURCE" ;;
+ nlohmann_json) src_var="NLOHMANN_JSON_SOURCE" ;;
+ libbacktrace) src_var="LIBBACKTRACE_SOURCE" ;;
+ sse2neon) src_var="SSE2NEON_SOURCE" ;;
+ xxhash) src_var="XXHASH_SOURCE" ;;
+ concurrentqueue) src_var="CONCURRENTQUEUE_SOURCE" ;;
+ fast_float) src_var="FAST_FLOAT_SOURCE" ;;
+ hadoop_libs) src_var="HADOOP_LIBS_SOURCE" ;;
+ hadoop_libs_3_4) src_var="HADOOP_LIBS_3_4_SOURCE" ;;
+ avx2neon) src_var="AVX2NEON_SOURCE" ;;
+ libdeflate) src_var="LIBDEFLATE_SOURCE" ;;
+ streamvbyte) src_var="STREAMVBYTE_SOURCE" ;;
+ ali_sdk)
+ # ali_sdk internally builds jsoncpp and libuuid, clean all three
+ for dep_var in JSONCPP_SOURCE LIBUUID_SOURCE ALI_SDK_SOURCE; do
+ dep_dir="${!dep_var}"
+ if [[ -n "${dep_dir}" && -d "${TP_SOURCE_DIR}/${dep_dir}" ]]; then
+ echo "Cleaning up source: ${dep_dir}"
+ rm -rf "${TP_SOURCE_DIR}/${dep_dir}"
+ fi
+ done
+ return
+ ;;
+ base64) src_var="BASE64_SOURCE" ;;
+ azure) src_var="AZURE_SOURCE" ;;
+ dragonbox) src_var="DRAGONBOX_SOURCE" ;;
+ icu) src_var="ICU_SOURCE" ;;
+ jindofs) src_var="JINDOFS_SOURCE" ;;
+ pugixml) src_var="PUGIXML_SOURCE" ;;
+ paimon_cpp) src_var="PAIMON_CPP_SOURCE" ;;
+ aws_sdk) src_var="AWS_SDK_SOURCE" ;;
+ lzma) src_var="LZMA_SOURCE" ;;
+ xml2) src_var="XML2_SOURCE" ;;
+ idn) src_var="IDN_SOURCE" ;;
+ gsasl) src_var="GSASL_SOURCE" ;;
+ krb5) src_var="KRB5_SOURCE" ;;
+ hdfs3) src_var="HDFS3_SOURCE" ;;
+ libdivide) src_var="LIBDIVIDE_SOURCE" ;;
+ binutils) src_var="BINUTILS_SOURCE" ;;
+ gettext) src_var="GETTEXT_SOURCE" ;;
+ # Header-only files, skip cleanup
+ pdqsort|timsort|tsan_header|js_and_css)
+ return
+ ;;
+ *)
+ echo "Warning: no source mapping for package '${pkg}', skipping cleanup"
+ return
+ ;;
+ esac
+
+ src_dir="${!src_var}"
+ if [[ -n "${src_dir}" && -d "${TP_SOURCE_DIR}/${src_dir}" ]]; then
+ echo "Cleaning up source: ${src_dir}"
+ rm -rf "${TP_SOURCE_DIR}/${src_dir}"
+ fi
+}
+
for package in "${packages[@]}"; do
if [[ "${package}" == "${start_package}" ]]; then
PACKAGE_FOUND=1
@@ -1905,6 +2130,8 @@ for package in "${packages[@]}"; do
if [[ "${CONTINUE}" -eq 0 ]] || [[ "${PACKAGE_FOUND}" -eq 1 ]]; then
command="build_${package}"
${command}
+ cd "${TP_DIR}"
+ cleanup_package_source "${package}"
fi
done
diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
index c7f2eea4c37..baba7863b69 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -75,6 +75,18 @@ md5sum_func() {
return 0
}
+is_git_package() {
+ local TP_ARCH="$1"
+ local GIT_URL_VAR="${TP_ARCH}_GIT_URL"
+ [[ -n "${!GIT_URL_VAR}" ]]
+}
+
+git_url_for() {
+ local TP_ARCH="$1"
+ local GIT_URL_VAR="${TP_ARCH}_GIT_URL"
+ echo "${!GIT_URL_VAR}"
+}
+
# return 0 if download succeed.
# return 1 if not.
download_func() {
@@ -131,6 +143,10 @@ download_func() {
# download thirdparty archives
echo "===== Downloading thirdparty archives..."
for TP_ARCH in "${TP_ARCHIVES[@]}"; do
+ if is_git_package "${TP_ARCH}"; then
+ echo "Skip downloading ${TP_ARCH} (git repo: $(git_url_for "${TP_ARCH}"))"
+ continue
+ fi
NAME="${TP_ARCH}_NAME"
MD5SUM="${TP_ARCH}_MD5SUM"
if [[ -z "${REPOSITORY_URL}" ]]; then
@@ -156,6 +172,9 @@ echo "===== Downloading thirdparty archives...done"
# check if all tp archives exists
echo "===== Checking all thirdpart archives..."
for TP_ARCH in "${TP_ARCHIVES[@]}"; do
+ if is_git_package "${TP_ARCH}"; then
+ continue
+ fi
NAME="${TP_ARCH}_NAME"
if [[ ! -r "${TP_SOURCE_DIR}/${!NAME}" ]]; then
echo "Failed to fetch ${!NAME}"
@@ -173,6 +192,9 @@ SUFFIX_XZ="\.tar\.xz$"
SUFFIX_ZIP="\.zip$"
SUFFIX_BZ2="\.tar\.bz2$"
for TP_ARCH in "${TP_ARCHIVES[@]}"; do
+ if is_git_package "${TP_ARCH}"; then
+ continue
+ fi
NAME="${TP_ARCH}_NAME"
SOURCE="${TP_ARCH}_SOURCE"
@@ -212,6 +234,57 @@ for TP_ARCH in "${TP_ARCHIVES[@]}"; do
done
echo "===== Unpacking all thirdparty archives...done"
+# Clone and checkout git repositories
+echo "===== Cloning git repositories..."
+for TP_ARCH in "${TP_ARCHIVES[@]}"; do
+ if ! is_git_package "${TP_ARCH}"; then
+ continue
+ fi
+
+ GIT_URL_VAR="${TP_ARCH}_GIT_URL"
+ GIT_TAG_VAR="${TP_ARCH}_GIT_TAG"
+ SOURCE_VAR="${TP_ARCH}_SOURCE"
+
+ GIT_URL="${!GIT_URL_VAR}"
+ GIT_TAG="${!GIT_TAG_VAR}"
+ SOURCE_DIR="${TP_SOURCE_DIR}/${!SOURCE_VAR}"
+
+ if [[ -z "${GIT_URL}" ]] || [[ -z "${GIT_TAG}" ]] || [[ -z "${!SOURCE_VAR}" ]]; then
+ echo "Warning: ${TP_ARCH} git configuration incomplete, skipping"
+ continue
+ fi
+
+ if [[ ! -d "${SOURCE_DIR}" ]]; then
+ echo "Cloning ${TP_ARCH} from ${GIT_URL}..."
+ cd "${TP_SOURCE_DIR}"
+ if ! git clone "${GIT_URL}" "${!SOURCE_VAR}"; then
+ echo "Failed to clone ${TP_ARCH}"
+ exit 1
+ fi
+ else
+ echo "${TP_ARCH} repository already exists, updating..."
+ cd "${SOURCE_DIR}"
+ git fetch origin || true
+ fi
+
+ cd "${SOURCE_DIR}"
+ if ! git checkout "${GIT_TAG}" 2>/dev/null; then
+ echo "Tag ${GIT_TAG} not found, trying to fetch..."
+ is_shallow="$(git rev-parse --is-shallow-repository 2>/dev/null || echo false)"
+ if [[ "${is_shallow}" == "true" ]]; then
+ git fetch --unshallow origin || git fetch --depth=2147483647 origin
+ else
+ git fetch origin
+ fi
+ if ! git checkout "${GIT_TAG}"; then
+ echo "Failed to checkout ${GIT_TAG} for ${TP_ARCH}"
+ exit 1
+ fi
+ fi
+ echo "Successfully checked out ${GIT_TAG} for ${TP_ARCH}"
+done
+echo "===== Cloning git repositories...done"
+
echo "===== Patching thirdparty archives..."
###################################################################################
@@ -326,6 +399,16 @@ if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-13.0.0" ]]; then
fi
cd -
fi
+if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-17.0.0" ]]; then
+ cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}"
+ if [[ ! -f "${PATCHED_MARK}" ]]; then
+ # Paimon-cpp parquet patches: row-group-aware batch reader, max_row_group_size,
+ # GetBufferedSize(), int96 NANO guard, and Thrift_VERSION empty fix.
+ patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-paimon.patch"
+ touch "${PATCHED_MARK}"
+ fi
+ cd -
+fi
echo "Finished patching ${ARROW_SOURCE}"
# patch librdkafka to avoid crash
@@ -471,5 +554,59 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " THRIFT " ]]; then
echo "Finished patching ${THRIFT_SOURCE}"
fi
+# patch re2
+if [[ " ${TP_ARCHIVES[*]} " =~ " RE2 " ]]; then
+ if [[ "${RE2_SOURCE}" == 're2-2021-02-02' ]]; then
+ cd "${TP_SOURCE_DIR}/${RE2_SOURCE}"
+ if [[ ! -f "${PATCHED_MARK}" ]]; then
+ for patch_file in "${TP_PATCH_DIR}"/re2-*; do
+ echo "patch ${patch_file}"
+ patch -p1 --ignore-whitespace <"${patch_file}"
+ done
+ touch "${PATCHED_MARK}"
+ fi
+ cd -
+ fi
+ echo "Finished patching ${RE2_SOURCE}"
+fi
+
+# patch azure
+if [[ " ${TP_ARCHIVES[*]} " =~ " AZURE " ]]; then
+ cd "${TP_SOURCE_DIR}/${AZURE_SOURCE}"
+ if [[ ! -f "${PATCHED_MARK}" ]]; then
+ patch -p1 <"${TP_PATCH_DIR}/azure-sdk-for-cpp-azure-core_1.16.0.patch"
+ touch "${PATCHED_MARK}"
+ fi
+ cd -
+ echo "Finished patching ${AZURE_SOURCE}"
+fi
+
+# patch paimon-cpp
+if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then
+ cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}"
+ if [[ ! -f "${PATCHED_MARK}" ]]; then
+ if patch -p1 -N --batch --dry-run <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch" >/dev/null 2>&1; then
+ patch -p1 -N --batch <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch"
+ else
+ echo "Skip paimon-cpp patch: already applied or not applicable for current source"
+ fi
+ touch "${PATCHED_MARK}"
+ fi
+ cd -
+ echo "Finished patching ${PAIMON_CPP_SOURCE}"
+fi
+
+if [[ " ${TP_ARCHIVES[*]} " =~ " CCTZ " ]] ; then
+ cd $TP_SOURCE_DIR/$CCTZ_SOURCE
+ if [[ ! -f "$PATCHED_MARK" ]] ; then
+ for patch_file in "${TP_PATCH_DIR}"/cctz-*; do
+ echo "patch ${patch_file}"
+ patch -p1 --ignore-whitespace <"${patch_file}"
+ done
+ touch "$PATCHED_MARK"
+ fi
+ cd -
+ echo "Finished patching ${CCTZ_SOURCE}"
+fi
# vim: ts=4 sw=4 ts=4 tw=100:
diff --git a/thirdparty/paimon-cpp-cache.cmake b/thirdparty/paimon-cpp-cache.cmake
new file mode 100644
index 00000000000..dbebd94a0cc
--- /dev/null
+++ b/thirdparty/paimon-cpp-cache.cmake
@@ -0,0 +1,136 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# CMake Initial Cache for paimon-cpp
+# Configures paimon-cpp to reuse selected Doris third-party libraries
+# Usage: cmake -C paimon-cpp-cache.cmake ...
+
+# Get the Doris thirdparty installation directory from environment
+set(DORIS_THIRDPARTY_DIR "$ENV{TP_INSTALL_DIR}" CACHE PATH "Doris thirdparty install directory")
+
+if(NOT DORIS_THIRDPARTY_DIR)
+ message(FATAL_ERROR "TP_INSTALL_DIR environment variable must be set")
+endif()
+
+message(STATUS "Using Doris thirdparty libraries from: ${DORIS_THIRDPARTY_DIR}")
+
+# Set CMAKE_PREFIX_PATH to help find_package locate our libraries
+set(CMAKE_PREFIX_PATH "${DORIS_THIRDPARTY_DIR};${CMAKE_PREFIX_PATH}" CACHE STRING "Search path for find_package")
+
+# Library and include paths
+set(DORIS_LIB_DIR "${DORIS_THIRDPARTY_DIR}/lib" CACHE PATH "Doris library directory")
+set(DORIS_INCLUDE_DIR "${DORIS_THIRDPARTY_DIR}/include" CACHE PATH "Doris include directory")
+
+# ============================================================================
+# ZLIB - Reuse from Doris (version 1.3.1)
+# ============================================================================
+set(ZLIB_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "ZLIB root directory")
+set(ZLIB_LIBRARY "${DORIS_LIB_DIR}/libz.a" CACHE FILEPATH "ZLIB library")
+set(ZLIB_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "ZLIB include directory")
+
+# ============================================================================
+# ZSTD - Reuse from Doris (version 1.5.7)
+# ============================================================================
+set(ZSTD_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "ZSTD root directory")
+set(ZSTD_LIBRARY "${DORIS_LIB_DIR}/libzstd.a" CACHE FILEPATH "ZSTD library")
+set(ZSTD_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "ZSTD include directory")
+
+# ============================================================================
+# LZ4 - Reuse from Doris (version 1.9.4)
+# ============================================================================
+set(LZ4_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "LZ4 root directory")
+set(LZ4_LIBRARY "${DORIS_LIB_DIR}/liblz4.a" CACHE FILEPATH "LZ4 library")
+set(LZ4_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "LZ4 include directory")
+
+# ============================================================================
+# glog - NOT reused from Doris
+# paimon-cpp's build_glog() unconditionally calls externalproject_add() to
+# build glog 0.7.1. Any GLOG_ROOT/GLOG_LIBRARY/GLOG_INCLUDE_DIR set here
+# would be overwritten by that macro, so we skip them entirely.
+# ============================================================================
+
+# ============================================================================
+# Arrow - Reuse from Doris (Doris Arrow now includes COMPUTE/DATASET/ACERO/FILESYSTEM)
+# Doris's Arrow 17.0.0 is built with the full module set that paimon-cpp
+# needs, so we skip paimon-cpp's internal externalproject_add(arrow_ep ...).
+# ============================================================================
+set(PAIMON_USE_EXTERNAL_ARROW ON CACHE BOOL "Use pre-built Arrow from Doris instead of building from source")
+
+set(DORIS_LIB64_DIR "${DORIS_THIRDPARTY_DIR}/lib64" CACHE PATH "Doris lib64 directory")
+
+set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "Arrow include directory")
+set(PAIMON_EXTERNAL_ARROW_LIB "${DORIS_LIB64_DIR}/libarrow.a" CACHE FILEPATH "Arrow core library")
+set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "${DORIS_LIB64_DIR}/libarrow_dataset.a" CACHE FILEPATH "Arrow Dataset library")
+set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "${DORIS_LIB64_DIR}/libarrow_acero.a" CACHE FILEPATH "Arrow Acero library")
+set(PAIMON_EXTERNAL_PARQUET_LIB "${DORIS_LIB64_DIR}/libparquet.a" CACHE FILEPATH "Parquet library")
+set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB "${DORIS_LIB64_DIR}/libarrow_bundled_dependencies.a" CACHE FILEPATH "Arrow bundled dependencies library")
+
+# Protobuf, Thrift - still built separately by paimon-cpp
+
+# ============================================================================
+# Snappy - Reuse from Doris
+# ============================================================================
+set(Snappy_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "Snappy root directory")
+set(SNAPPY_ROOT "${DORIS_THIRDPARTY_DIR}" CACHE PATH "Snappy root directory (legacy)")
+set(SNAPPY_LIBRARY "${DORIS_LIB_DIR}/libsnappy.a" CACHE FILEPATH "Snappy library")
+set(SNAPPY_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "Snappy include directory")
+
+# ============================================================================
+# Build configuration
+# ============================================================================
+set(CMAKE_POSITION_INDEPENDENT_CODE ON CACHE BOOL "Build with -fPIC")
+set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build type")
+
+# Symbol visibility control to prevent conflicts with Doris
+# paimon-cpp builds Arrow/ORC/etc with hidden symbols to avoid conflicts
+set(CMAKE_CXX_VISIBILITY_PRESET "hidden" CACHE STRING "Hide C++ symbols by default")
+set(CMAKE_C_VISIBILITY_PRESET "hidden" CACHE STRING "Hide C symbols by default")
+set(CMAKE_VISIBILITY_INLINES_HIDDEN ON CACHE BOOL "Hide inline function symbols")
+
+# Verify that required libraries exist
+if(NOT EXISTS "${ZLIB_LIBRARY}")
+ message(FATAL_ERROR "ZLIB library not found: ${ZLIB_LIBRARY}")
+endif()
+if(NOT EXISTS "${ZSTD_LIBRARY}")
+ message(FATAL_ERROR "ZSTD library not found: ${ZSTD_LIBRARY}")
+endif()
+if(NOT EXISTS "${LZ4_LIBRARY}")
+ message(FATAL_ERROR "LZ4 library not found: ${LZ4_LIBRARY}")
+endif()
+if(NOT EXISTS "${SNAPPY_LIBRARY}")
+ message(FATAL_ERROR "Snappy library not found: ${SNAPPY_LIBRARY}")
+endif()
+
+message(STATUS "========================================")
+message(STATUS "Paimon-cpp Library Reuse Configuration")
+message(STATUS "========================================")
+message(STATUS "Reusing from Doris:")
+message(STATUS " ✓ ZLIB, ZSTD, LZ4, Snappy")
+if(PAIMON_USE_EXTERNAL_ARROW)
+ message(STATUS " ✓ Arrow, Parquet, Arrow Dataset, Arrow Acero (Plan B)")
+else()
+ message(STATUS " ✗ Arrow (building separately, symbol visibility=hidden)")
+endif()
+message(STATUS "")
+message(STATUS "Building separately:")
+if(NOT PAIMON_USE_EXTERNAL_ARROW)
+ message(STATUS " - Arrow, Protobuf, Thrift, ORC")
+else()
+ message(STATUS " - Protobuf, Thrift, ORC")
+endif()
+message(STATUS " - glog, RapidJSON, TBB")
+message(STATUS "========================================")
diff --git a/thirdparty/patches/apache-arrow-17.0.0-paimon.patch b/thirdparty/patches/apache-arrow-17.0.0-paimon.patch
new file mode 100644
index 00000000000..4e53117b79b
--- /dev/null
+++ b/thirdparty/patches/apache-arrow-17.0.0-paimon.patch
@@ -0,0 +1,224 @@
+diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc
+index ec3890a41f..943f69bb6c 100644
+--- a/cpp/src/parquet/arrow/schema.cc
++++ b/cpp/src/parquet/arrow/schema.cc
+@@ -178,7 +178,7 @@ static Status GetTimestampMetadata(const ::arrow::TimestampType& type,
+
+ // The user is explicitly asking for Impala int96 encoding, there is no
+ // logical type.
+- if (arrow_properties.support_deprecated_int96_timestamps()) {
++ if (arrow_properties.support_deprecated_int96_timestamps() && target_unit == ::arrow::TimeUnit::NANO) {
+ *physical_type = ParquetType::INT96;
+ return Status::OK();
+ }
+
+diff --git a/cpp/src/parquet/arrow/reader.cc b/cpp/src/parquet/arrow/reader.cc
+index 285e2a5973..aa6f92f077 100644
+--- a/cpp/src/parquet/arrow/reader.cc
++++ b/cpp/src/parquet/arrow/reader.cc
+@@ -1013,25 +1013,32 @@ Status FileReaderImpl::GetRecordBatchReader(const std::vector<int>& row_groups,
+ return Status::OK();
+ }
+
+- int64_t num_rows = 0;
++ std::vector<int64_t> num_rows;
+ for (int row_group : row_groups) {
+- num_rows += parquet_reader()->metadata()->RowGroup(row_group)->num_rows();
++ num_rows.push_back(parquet_reader()->metadata()->RowGroup(row_group)->num_rows());
+ }
+
+ using ::arrow::RecordBatchIterator;
++ int row_group_idx = 0;
+
+ // NB: This lambda will be invoked outside the scope of this call to
+ // `GetRecordBatchReader()`, so it must capture `readers` and `batch_schema` by value.
+ // `this` is a non-owning pointer so we are relying on the parent FileReader outliving
+ // this RecordBatchReader.
+ ::arrow::Iterator<RecordBatchIterator> batches = ::arrow::MakeFunctionIterator(
+- [readers, batch_schema, num_rows,
++ [readers, batch_schema, num_rows, row_group_idx,
+ this]() mutable -> ::arrow::Result<RecordBatchIterator> {
+ ::arrow::ChunkedArrayVector columns(readers.size());
+
+- // don't reserve more rows than necessary
+- int64_t batch_size = std::min(properties().batch_size(), num_rows);
+- num_rows -= batch_size;
++ int64_t batch_size = 0;
++ if (!num_rows.empty()) {
++ // don't reserve more rows than necessary
++ batch_size = std::min(properties().batch_size(), num_rows[row_group_idx]);
++ num_rows[row_group_idx] -= batch_size;
++ if (num_rows[row_group_idx] == 0 && (num_rows.size() - 1) != row_group_idx) {
++ row_group_idx++;
++ }
++ }
+
+ RETURN_NOT_OK(::arrow::internal::OptionalParallelFor(
+ reader_properties_.use_threads(), static_cast<int>(readers.size()),
+diff --git a/cpp/src/parquet/arrow/writer.cc b/cpp/src/parquet/arrow/writer.cc
+index 4fd7ef1b47..87326a54f1 100644
+--- a/cpp/src/parquet/arrow/writer.cc
++++ b/cpp/src/parquet/arrow/writer.cc
+@@ -314,6 +314,14 @@ class FileWriterImpl : public FileWriter {
+ return Status::OK();
+ }
+
++ int64_t GetBufferedSize() override {
++ if (row_group_writer_ == nullptr) {
++ return 0;
++ }
++ return row_group_writer_->total_compressed_bytes() +
++ row_group_writer_->total_compressed_bytes_written();
++ }
++
+ Status Close() override {
+ if (!closed_) {
+ // Make idempotent
+@@ -418,10 +426,13 @@ class FileWriterImpl : public FileWriter {
+
+ // Max number of rows allowed in a row group.
+ const int64_t max_row_group_length = this->properties().max_row_group_length();
++ const int64_t max_row_group_size = this->properties().max_row_group_size();
+
+ // Initialize a new buffered row group writer if necessary.
+ if (row_group_writer_ == nullptr || !row_group_writer_->buffered() ||
+- row_group_writer_->num_rows() >= max_row_group_length) {
++ row_group_writer_->num_rows() >= max_row_group_length ||
++ (row_group_writer_->total_compressed_bytes_written() +
++ row_group_writer_->total_compressed_bytes() >= max_row_group_size)) {
+ RETURN_NOT_OK(NewBufferedRowGroup());
+ }
+
+diff --git a/cpp/src/parquet/arrow/writer.h b/cpp/src/parquet/arrow/writer.h
+index 4a1a033a7b..0f13d05e44 100644
+--- a/cpp/src/parquet/arrow/writer.h
++++ b/cpp/src/parquet/arrow/writer.h
+@@ -138,6 +138,9 @@ class PARQUET_EXPORT FileWriter {
+ /// option in this case.
+ virtual ::arrow::Status WriteRecordBatch(const ::arrow::RecordBatch& batch) = 0;
+
++ /// \brief Return the buffered size in bytes.
++ virtual int64_t GetBufferedSize() = 0;
++
+ /// \brief Write the footer and close the file.
+ virtual ::arrow::Status Close() = 0;
+ virtual ~FileWriter();
+diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h
+index 4d3acb491e..3906ff3c59 100644
+--- a/cpp/src/parquet/properties.h
++++ b/cpp/src/parquet/properties.h
+@@ -139,6 +139,7 @@ static constexpr bool DEFAULT_IS_DICTIONARY_ENABLED = true;
+ static constexpr int64_t DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT = kDefaultDataPageSize;
+ static constexpr int64_t DEFAULT_WRITE_BATCH_SIZE = 1024;
+ static constexpr int64_t DEFAULT_MAX_ROW_GROUP_LENGTH = 1024 * 1024;
++static constexpr int64_t DEFAULT_MAX_ROW_GROUP_SIZE = 128 * 1024 * 1024;
+ static constexpr bool DEFAULT_ARE_STATISTICS_ENABLED = true;
+ static constexpr int64_t DEFAULT_MAX_STATISTICS_SIZE = 4096;
+ static constexpr Encoding::type DEFAULT_ENCODING = Encoding::UNKNOWN;
+@@ -232,6 +233,7 @@ class PARQUET_EXPORT WriterProperties {
+ dictionary_pagesize_limit_(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT),
+ write_batch_size_(DEFAULT_WRITE_BATCH_SIZE),
+ max_row_group_length_(DEFAULT_MAX_ROW_GROUP_LENGTH),
++ max_row_group_size_(DEFAULT_MAX_ROW_GROUP_SIZE),
+ pagesize_(kDefaultDataPageSize),
+ version_(ParquetVersion::PARQUET_2_6),
+ data_page_version_(ParquetDataPageVersion::V1),
+@@ -244,6 +246,7 @@ class PARQUET_EXPORT WriterProperties {
+ dictionary_pagesize_limit_(properties.dictionary_pagesize_limit()),
+ write_batch_size_(properties.write_batch_size()),
+ max_row_group_length_(properties.max_row_group_length()),
++ max_row_group_size_(properties.max_row_group_size()),
+ pagesize_(properties.data_pagesize()),
+ version_(properties.version()),
+ data_page_version_(properties.data_page_version()),
+@@ -321,6 +324,13 @@ class PARQUET_EXPORT WriterProperties {
+ return this;
+ }
+
++ /// Specify the max bytes size to put in a single row group.
++ /// Default 128 M.
++ Builder* max_row_group_size(int64_t max_row_group_size) {
++ max_row_group_size_ = max_row_group_size;
++ return this;
++ }
++
+ /// Specify the data page size.
+ /// Default 1MB.
+ Builder* data_pagesize(int64_t pg_size) {
+@@ -664,7 +674,7 @@ class PARQUET_EXPORT WriterProperties {
+
+ return std::shared_ptr<WriterProperties>(new WriterProperties(
+ pool_, dictionary_pagesize_limit_, write_batch_size_, max_row_group_length_,
+- pagesize_, version_, created_by_, page_checksum_enabled_,
++ max_row_group_size_, pagesize_, version_, created_by_, page_checksum_enabled_,
+ std::move(file_encryption_properties_), default_column_properties_,
+ column_properties, data_page_version_, store_decimal_as_integer_,
+ std::move(sorting_columns_)));
+@@ -675,6 +685,7 @@ class PARQUET_EXPORT WriterProperties {
+ int64_t dictionary_pagesize_limit_;
+ int64_t write_batch_size_;
+ int64_t max_row_group_length_;
++ int64_t max_row_group_size_;
+ int64_t pagesize_;
+ ParquetVersion::type version_;
+ ParquetDataPageVersion data_page_version_;
+@@ -705,6 +716,8 @@ class PARQUET_EXPORT WriterProperties {
+
+ inline int64_t max_row_group_length() const { return max_row_group_length_; }
+
++ inline int64_t max_row_group_size() const { return max_row_group_size_; }
++
+ inline int64_t data_pagesize() const { return pagesize_; }
+
+ inline ParquetDataPageVersion data_page_version() const {
+@@ -810,7 +823,7 @@ class PARQUET_EXPORT WriterProperties {
+ private:
+ explicit WriterProperties(
+ MemoryPool* pool, int64_t dictionary_pagesize_limit, int64_t write_batch_size,
+- int64_t max_row_group_length, int64_t pagesize, ParquetVersion::type version,
++ int64_t max_row_group_length, int64_t max_row_group_size, int64_t pagesize, ParquetVersion::type version,
+ const std::string& created_by, bool page_write_checksum_enabled,
+ std::shared_ptr<FileEncryptionProperties> file_encryption_properties,
+ const ColumnProperties& default_column_properties,
+@@ -821,6 +834,7 @@ class PARQUET_EXPORT WriterProperties {
+ dictionary_pagesize_limit_(dictionary_pagesize_limit),
+ write_batch_size_(write_batch_size),
+ max_row_group_length_(max_row_group_length),
++ max_row_group_size_(max_row_group_size),
+ pagesize_(pagesize),
+ parquet_data_page_version_(data_page_version),
+ parquet_version_(version),
+@@ -836,6 +850,7 @@ class PARQUET_EXPORT WriterProperties {
+ int64_t dictionary_pagesize_limit_;
+ int64_t write_batch_size_;
+ int64_t max_row_group_length_;
++ int64_t max_row_group_size_;
+ int64_t pagesize_;
+ ParquetDataPageVersion parquet_data_page_version_;
+ ParquetVersion::type parquet_version_;
+diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
+index 9df922afa2..5c8b3d4d07 100644
+--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
+@@ -1789,7 +1789,20 @@ if(ARROW_WITH_THRIFT)
+ REQUIRED_VERSION
+ 0.11.0)
+
+- string(REPLACE "." ";" Thrift_VERSION_LIST ${Thrift_VERSION})
++ if(NOT Thrift_VERSION)
++ if(DEFINED thrift_PC_VERSION AND thrift_PC_VERSION)
++ set(Thrift_VERSION "${thrift_PC_VERSION}")
++ elseif(DEFINED ThriftAlt_VERSION AND ThriftAlt_VERSION)
++ set(Thrift_VERSION "${ThriftAlt_VERSION}")
++ elseif(DEFINED THRIFT_VERSION AND THRIFT_VERSION)
++ set(Thrift_VERSION "${THRIFT_VERSION}")
++ endif()
++ endif()
++ if(NOT Thrift_VERSION)
++ message(FATAL_ERROR "Thrift_VERSION is empty after resolving Thrift dependency")
++ endif()
++
++ string(REPLACE "." ";" Thrift_VERSION_LIST "${Thrift_VERSION}")
+ list(GET Thrift_VERSION_LIST 0 Thrift_VERSION_MAJOR)
+ list(GET Thrift_VERSION_LIST 1 Thrift_VERSION_MINOR)
+ list(GET Thrift_VERSION_LIST 2 Thrift_VERSION_PATCH)
diff --git a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch b/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch
new file mode 100644
index 00000000000..31af1db7f0f
--- /dev/null
+++ b/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch
@@ -0,0 +1,331 @@
+diff --git a/cmake_modules/BuildUtils.cmake b/cmake_modules/BuildUtils.cmake
+index 74654a4..4065297 100644
+--- a/cmake_modules/BuildUtils.cmake
++++ b/cmake_modules/BuildUtils.cmake
+@@ -55,12 +55,18 @@ function(add_paimon_lib LIB_NAME)
+ # Necessary to make static linking into other shared libraries work properly
+ set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1)
+ if(ARG_DEPENDENCIES)
+- # Avoid add_dependencies on non-existent targets (e.g. when building static only).
++ # In static-only builds, some dependency names are still declared as
++ # *_shared. Map them to *_static when the shared target is unavailable.
+ set(_paimon_objlib_deps)
+ foreach(_paimon_dep IN LISTS ARG_DEPENDENCIES)
+- if(TARGET ${_paimon_dep})
+- list(APPEND _paimon_objlib_deps ${_paimon_dep})
++ set(_paimon_mapped_dep "${_paimon_dep}")
++ if(NOT TARGET ${_paimon_mapped_dep} AND _paimon_dep MATCHES "_shared$")
++ string(REGEX REPLACE "_shared$" "_static" _paimon_mapped_dep "${_paimon_dep}")
+ endif()
++ if(TARGET ${_paimon_mapped_dep})
++ list(APPEND _paimon_objlib_deps ${_paimon_mapped_dep})
++ endif()
++ unset(_paimon_mapped_dep)
+ endforeach()
+ if(_paimon_objlib_deps)
+ add_dependencies(${LIB_NAME}_objlib ${_paimon_objlib_deps})
+
+diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
+--- a/cmake_modules/ThirdpartyToolchain.cmake
++++ b/cmake_modules/ThirdpartyToolchain.cmake
+@@ -909,9 +909,6 @@ macro(build_orc)
+ "-DCMAKE_CXX_FLAGS=${ORC_CMAKE_CXX_FLAGS}"
+ "-DCMAKE_C_FLAGS=${ORC_CMAKE_C_FLAGS}"
+ "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${ORC_CMAKE_CXX_FLAGS}"
+- "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}"
+- "-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}"
+- "-DCMAKE_MODULE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}"
+ "-DSNAPPY_HOME=${ORC_SNAPPY_ROOT}"
+ "-DLZ4_HOME=${ORC_LZ4_ROOT}"
+ "-DZSTD_HOME=${ORC_ZSTD_ROOT}"
+@@ -923,6 +920,13 @@ macro(build_orc)
+ -DBUILD_TOOLS=OFF
+ -DBUILD_CPP_ENABLE_METRICS=ON)
+
++ if(ORC_RPATH)
++ list(APPEND ORC_CMAKE_ARGS
++ "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath,${ORC_RPATH}"
++ "-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-rpath,${ORC_RPATH}"
++ "-DCMAKE_MODULE_LINKER_FLAGS=-Wl,-rpath,${ORC_RPATH}")
++ endif()
++
+ set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/orc.diff")
+ externalproject_add(orc_ep
+ URL ${ORC_SOURCE_URL}
+
+diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
+--- a/cmake_modules/ThirdpartyToolchain.cmake
++++ b/cmake_modules/ThirdpartyToolchain.cmake
+@@ -1024,6 +1024,14 @@ macro(build_arrow)
+ "-DCMAKE_C_FLAGS=${ARROW_CMAKE_C_FLAGS}"
+ "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${ARROW_CMAKE_CXX_FLAGS}"
+ -DARROW_DEPENDENCY_USE_SHARED=OFF
++ # Avoid forcing CONDA dependency mode when CONDA_PREFIX is present.
++ # AUTO keeps the normal "find system first, fallback to bundled"
++ # behavior and prevents accidental pickup of conda's thrift/zstd.
++ -DARROW_DEPENDENCY_SOURCE=AUTO
++ # Isolate from user/system CMake package registries to improve
++ # reproducibility in CI and local mixed environments.
++ -DCMAKE_FIND_USE_PACKAGE_REGISTRY=OFF
++ -DCMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=OFF
+ -DARROW_BUILD_SHARED=OFF
+ -DARROW_BUILD_STATIC=ON
+ -DARROW_BUILD_TESTS=OFF
+
+diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
+--- a/cmake_modules/ThirdpartyToolchain.cmake
++++ b/cmake_modules/ThirdpartyToolchain.cmake
+@@ -761,6 +761,9 @@ macro(build_protobuf)
+ get_target_property(THIRDPARTY_ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES)
+ get_filename_component(THIRDPARTY_ZLIB_ROOT "${THIRDPARTY_ZLIB_INCLUDE_DIR}"
+ DIRECTORY)
++ set(THIRDPARTY_ZLIB_STATIC_LIB
++ "${THIRDPARTY_ZLIB_ROOT}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}z${CMAKE_STATIC_LIBRARY_SUFFIX}"
++ )
+
+ # Strip lto flags (which may be added by dh_auto_configure)
+ # See https://github.com/protocolbuffers/protobuf/issues/7092
+@@ -778,6 +781,10 @@ macro(build_protobuf)
+ "-DCMAKE_CXX_FLAGS=${PROTOBUF_CXX_FLAGS}"
+ "-DCMAKE_C_FLAGS=${PROTOBUF_C_FLAGS}"
+ "-DZLIB_ROOT=${THIRDPARTY_ZLIB_ROOT}"
++ "-DZLIB_INCLUDE_DIR=${THIRDPARTY_ZLIB_INCLUDE_DIR}"
++ "-DZLIB_LIBRARY=${THIRDPARTY_ZLIB_STATIC_LIB}"
++ "-DZLIB_LIBRARY_RELEASE=${THIRDPARTY_ZLIB_STATIC_LIB}"
++ "-DZLIB_LIBRARY_DEBUG=${THIRDPARTY_ZLIB_STATIC_LIB}"
+ -Dprotobuf_BUILD_TESTS=OFF
+ -Dprotobuf_DEBUG_POSTFIX=)
+ set(PROTOBUF_CONFIGURE SOURCE_SUBDIR "cmake" CMAKE_ARGS ${PROTOBUF_CMAKE_ARGS})
+
+diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
+--- a/cmake_modules/ThirdpartyToolchain.cmake
++++ b/cmake_modules/ThirdpartyToolchain.cmake
+@@ -34,6 +34,16 @@ set(EP_COMMON_TOOLCHAIN "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
+ "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}")
+
++option(PAIMON_USE_EXTERNAL_ARROW "Reuse external Arrow/Parquet instead of building arrow_ep" OFF)
++set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "" CACHE PATH
++ "Include directory for external Arrow/Parquet headers")
++set(PAIMON_EXTERNAL_ARROW_LIB "" CACHE FILEPATH "Path to external libarrow.a")
++set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "" CACHE FILEPATH "Path to external libarrow_dataset.a")
++set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "" CACHE FILEPATH "Path to external libarrow_acero.a")
++set(PAIMON_EXTERNAL_PARQUET_LIB "" CACHE FILEPATH "Path to external libparquet.a")
++set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB "" CACHE FILEPATH
++ "Path to external libarrow_bundled_dependencies.a")
++
+ macro(set_urls URLS)
+ set(${URLS} ${ARGN})
+ endmacro()
+
+diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
+--- a/cmake_modules/ThirdpartyToolchain.cmake
++++ b/cmake_modules/ThirdpartyToolchain.cmake
+@@ -961,5 +961,95 @@ macro(build_orc)
+ endmacro()
+
+ macro(build_arrow)
+- message(STATUS "Building Arrow from source")
++ if(PAIMON_USE_EXTERNAL_ARROW)
++ set(ARROW_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/doris_external_arrow_include")
++ file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}")
++ if(NOT EXISTS "${ARROW_INCLUDE_DIR}/arrow")
++ execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink
++ "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}/arrow"
++ "${ARROW_INCLUDE_DIR}/arrow")
++ endif()
++ if(EXISTS "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}/parquet"
++ AND NOT EXISTS "${ARROW_INCLUDE_DIR}/parquet")
++ execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink
++ "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}/parquet"
++ "${ARROW_INCLUDE_DIR}/parquet")
++ endif()
++
++ if(NOT PAIMON_EXTERNAL_ARROW_INCLUDE_DIR)
++ message(FATAL_ERROR
++ "PAIMON_EXTERNAL_ARROW_INCLUDE_DIR must be set when PAIMON_USE_EXTERNAL_ARROW=ON"
++ )
++ endif()
++ if(NOT EXISTS "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}")
++ message(FATAL_ERROR
++ "PAIMON_EXTERNAL_ARROW_INCLUDE_DIR not found: ${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}"
++ )
++ endif()
++
++ foreach(_paimon_external_lib
++ IN ITEMS PAIMON_EXTERNAL_ARROW_LIB
++ PAIMON_EXTERNAL_ARROW_DATASET_LIB
++ PAIMON_EXTERNAL_ARROW_ACERO_LIB
++ PAIMON_EXTERNAL_PARQUET_LIB
++ PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB)
++ if(NOT ${_paimon_external_lib})
++ message(FATAL_ERROR
++ "${_paimon_external_lib} must be set when PAIMON_USE_EXTERNAL_ARROW=ON")
++ endif()
++ if(NOT EXISTS "${${_paimon_external_lib}}")
++ message(FATAL_ERROR
++ "${_paimon_external_lib} not found: ${${_paimon_external_lib}}")
++ endif()
++ endforeach()
++
++ add_library(arrow STATIC IMPORTED)
++ set_target_properties(arrow
++ PROPERTIES IMPORTED_LOCATION "${PAIMON_EXTERNAL_ARROW_LIB}"
++ INTERFACE_INCLUDE_DIRECTORIES
++ "${ARROW_INCLUDE_DIR}")
++
++ add_library(arrow_dataset STATIC IMPORTED)
++ set_target_properties(arrow_dataset
++ PROPERTIES IMPORTED_LOCATION
++ "${PAIMON_EXTERNAL_ARROW_DATASET_LIB}"
++ INTERFACE_INCLUDE_DIRECTORIES
++ "${ARROW_INCLUDE_DIR}")
++
++ add_library(arrow_acero STATIC IMPORTED)
++ set_target_properties(arrow_acero
++ PROPERTIES IMPORTED_LOCATION
++ "${PAIMON_EXTERNAL_ARROW_ACERO_LIB}"
++ INTERFACE_INCLUDE_DIRECTORIES
++ "${ARROW_INCLUDE_DIR}")
++
++ add_library(parquet STATIC IMPORTED)
++ set_target_properties(parquet
++ PROPERTIES IMPORTED_LOCATION "${PAIMON_EXTERNAL_PARQUET_LIB}"
++ INTERFACE_INCLUDE_DIRECTORIES
++ "${ARROW_INCLUDE_DIR}")
++
++ add_library(arrow_bundled_dependencies STATIC IMPORTED)
++ set_target_properties(arrow_bundled_dependencies
++ PROPERTIES IMPORTED_LOCATION
++ "${PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB}"
++ INTERFACE_INCLUDE_DIRECTORIES
++ "${ARROW_INCLUDE_DIR}")
++
++ target_link_libraries(arrow_acero INTERFACE arrow)
++
++ target_link_libraries(arrow_dataset INTERFACE arrow_acero)
++
++ target_link_libraries(arrow
++ INTERFACE zstd
++ snappy
++ lz4
++ zlib
++ arrow_bundled_dependencies)
++
++ target_link_libraries(parquet
++ INTERFACE zstd snappy lz4 zlib arrow_bundled_dependencies
++ arrow_dataset)
++ else()
++ message(STATUS "Building Arrow from source")
+
+ get_target_property(ARROW_SNAPPY_INCLUDE_DIR snappy INTERFACE_INCLUDE_DIRECTORIES)
+ get_filename_component(ARROW_SNAPPY_ROOT "${ARROW_SNAPPY_INCLUDE_DIR}" DIRECTORY)
+
+diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
+--- a/cmake_modules/ThirdpartyToolchain.cmake
++++ b/cmake_modules/ThirdpartyToolchain.cmake
+@@ -1121,6 +1121,7 @@ macro(build_arrow)
+ zlib
+ arrow_bundled_dependencies
+ arrow_dataset)
++ endif()
+
+ endmacro(build_arrow)
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -326,10 +326,10 @@ if(PAIMON_ENABLE_LUMINA)
+ include_directories("${CMAKE_SOURCE_DIR}/third_party/lumina/include")
+ endif()
+
++include_directories(SYSTEM ${GLOG_INCLUDE_DIR})
+ include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
+ include_directories(SYSTEM ${TBB_INCLUDE_DIR})
+
+-include_directories(SYSTEM ${GLOG_INCLUDE_DIR})
+ add_compile_definitions("GLOG_USE_GLOG_EXPORT")
+
+ set(THREADS_PREFER_PTHREAD_FLAG ON)
+
+diff --git a/src/paimon/common/logging/logging.cpp b/src/paimon/common/logging/logging.cpp
+--- a/src/paimon/common/logging/logging.cpp
++++ b/src/paimon/common/logging/logging.cpp
+@@ -83,7 +83,7 @@ std::unique_ptr<Logger> Logger::GetLogger(const std::string& path) {
+ }
+ std::unique_lock<std::shared_mutex> ulock(getRegistryLock());
+ if (!google::IsGoogleLoggingInitialized()) {
+- google::InitGoogleLogging(program_invocation_name);
++ google::InitGoogleLogging("paimon-cpp");
+ }
+ return std::make_unique<GlogAdaptor>();
+ }
+
+diff --git a/src/paimon/common/memory/memory_pool.cpp b/src/paimon/common/memory/memory_pool.cpp
+--- a/src/paimon/common/memory/memory_pool.cpp
++++ b/src/paimon/common/memory/memory_pool.cpp
+@@ -55,7 +55,7 @@ void* MemoryPoolImpl::Malloc(uint64_t size, uint64_t alignment) {
+ return memptr;
+ }
+
+-void* MemoryPoolImpl::Realloc(void* p, size_t old_size, size_t new_size, size_t alignment) {
++void* MemoryPoolImpl::Realloc(void* p, size_t old_size, size_t new_size, uint64_t alignment) {
+ if (alignment == 0) {
+ void* memptr = ::realloc(p, new_size);
+ total_allocated_size.fetch_add(new_size - old_size);
+
+diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/blob/blob_format_writer.cpp
+--- a/src/paimon/format/blob/blob_format_writer.cpp
++++ b/src/paimon/format/blob/blob_format_writer.cpp
+@@ -138,7 +138,8 @@ Status BlobFormatWriter::WriteBlob(std::string_view blob_data) {
+ }
+ PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, in->Length());
+ uint64_t total_read_length = 0;
+- uint32_t read_len = std::min(file_length, tmp_buffer_->size());
++ uint32_t read_len =
++ static_cast<uint32_t>(std::min<uint64_t>(file_length, tmp_buffer_->size()));
+ while (read_len > 0) {
+ PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_len, in->Read(tmp_buffer_->data(), read_len));
+ if (static_cast<uint32_t>(actual_read_len) != read_len) {
+@@ -149,7 +150,8 @@ Status BlobFormatWriter::WriteBlob(std::string_view blob_data) {
+ }
+ PAIMON_RETURN_NOT_OK(WriteWithCrc32(tmp_buffer_->data(), actual_read_len));
+ total_read_length += actual_read_len;
+- read_len = std::min(file_length - total_read_length, tmp_buffer_->size());
++ read_len = static_cast<uint32_t>(
++ std::min<uint64_t>(file_length - total_read_length, tmp_buffer_->size()));
+ }
+
+ // write bin length
+
+--- a/cmake_modules/arrow.diff
++++ b/cmake_modules/arrow.diff
+@@ -196,3 +196,29 @@
+ int64_t pagesize_;
+ ParquetDataPageVersion parquet_data_page_version_;
+ ParquetVersion::type parquet_version_;
++diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
++index 9df922afa2..5c8b3d4d07 100644
++--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
++@@ -1789,7 +1789,20 @@ if(ARROW_WITH_THRIFT)
++ REQUIRED_VERSION
++ 0.11.0)
++
++- string(REPLACE "." ";" Thrift_VERSION_LIST ${Thrift_VERSION})
+++ if(NOT Thrift_VERSION)
+++ if(DEFINED thrift_PC_VERSION AND thrift_PC_VERSION)
+++ set(Thrift_VERSION "${thrift_PC_VERSION}")
+++ elseif(DEFINED ThriftAlt_VERSION AND ThriftAlt_VERSION)
+++ set(Thrift_VERSION "${ThriftAlt_VERSION}")
+++ elseif(DEFINED THRIFT_VERSION AND THRIFT_VERSION)
+++ set(Thrift_VERSION "${THRIFT_VERSION}")
+++ endif()
+++ endif()
+++ if(NOT Thrift_VERSION)
+++ message(FATAL_ERROR "Thrift_VERSION is empty after resolving Thrift dependency")
+++ endif()
+++
+++ string(REPLACE "." ";" Thrift_VERSION_LIST "${Thrift_VERSION}")
++ list(GET Thrift_VERSION_LIST 0 Thrift_VERSION_MAJOR)
++ list(GET Thrift_VERSION_LIST 1 Thrift_VERSION_MINOR)
++ list(GET Thrift_VERSION_LIST 2 Thrift_VERSION_PATCH)
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 7bc66c4677f..1c81f4865f6 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -134,10 +134,11 @@ LZO2_SOURCE=lzo-2.10
LZO2_MD5SUM="39d3f3f9c55c87b1e5d6888e1420f4b5"
# rapidjson
-RAPIDJSON_DOWNLOAD="https://github.com/Tencent/rapidjson/archive/1a803826f1197b5e30703afe4b9c0e7dd48074f5.zip"
-RAPIDJSON_NAME=rapidjson-1a803826f1197b5e30703afe4b9c0e7dd48074f5.zip
-RAPIDJSON_SOURCE=rapidjson-1a803826f1197b5e30703afe4b9c0e7dd48074f5
-RAPIDJSON_MD5SUM="f2212a77e055a15501477f1e390007ea"
+# Updated to match paimon-cpp version (commit 232389d4f1012dddec4ef84861face2d2ba85709)
+RAPIDJSON_DOWNLOAD="https://github.com/miloyip/rapidjson/archive/232389d4f1012dddec4ef84861face2d2ba85709.tar.gz"
+RAPIDJSON_NAME=rapidjson-232389d4f1012dddec4ef84861face2d2ba85709.tar.gz
+RAPIDJSON_SOURCE=rapidjson-232389d4f1012dddec4ef84861face2d2ba85709
+RAPIDJSON_MD5SUM="577d3495a07b66fcd4a2866c93831bc4"
# curl
CURL_DOWNLOAD="https://curl.se/download/curl-8.2.1.tar.gz"
@@ -522,6 +523,20 @@ JINDOFS_NAME=jindofs-6.8.2-libs-0.1.tar.gz
JINDOFS_SOURCE=jindofs-6.8.2-libs-0.1
JINDOFS_MD5SUM="0e5b0f71e636b8ed3f09e0bf16208fd1"
+# pugixml
+PUGIXML_DOWNLOAD="https://github.com/zeux/pugixml/releases/download/v1.15/pugixml-1.15.tar.gz"
+PUGIXML_NAME=pugixml-1.15.tar.gz
+PUGIXML_SOURCE=pugixml-1.15
+PUGIXML_MD5SUM="3b894c29455eb33a40b165c6e2de5895"
+
+# paimon-cpp
+# Using git clone since there's no official release yet
+# We'll use a specific commit or tag for reproducibility
+PAIMON_CPP_GIT_URL="https://github.com/alibaba/paimon-cpp.git"
+PAIMON_CPP_GIT_TAG="0a4f4e2e7967fdb0be180711bbe581a18eeeb2dd"
+PAIMON_CPP_NAME=paimon-cpp
+PAIMON_CPP_SOURCE=paimon-cpp
+
# all thirdparties which need to be downloaded is set in array TP_ARCHIVES
export TP_ARCHIVES=(
'LIBEVENT'
@@ -599,6 +614,8 @@ export TP_ARCHIVES=(
'BASE64'
'ICU'
'JINDOFS'
+ 'PUGIXML'
+ 'PAIMON_CPP'
)
if [[ "$(uname -s)" == 'Darwin' ]]; then
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org