You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by ni...@apache.org on 2026/09/03 03:47:29 UTC
(paimon-cpp) branch main updated: fix: align format options and listagg semantics with Java (#263)
This is an automated email from the ASF dual-hosted git repository.
SteNicholas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 2e6935f4 fix: align format options and listagg semantics with Java (#263)
2e6935f4 is described below
commit 2e6935f493f2b6a3a937fb2362fac895308a9662
Author: lxy <38...@users.noreply.github.com>
AuthorDate: Thu Sep 3 11:47:24 2026 +0800
fix: align format options and listagg semantics with Java (#263)
---
examples/read_write_demo.cpp | 3 +-
include/paimon/defs.h | 4 -
...map_shared_shredding_read_plan_factory_test.cpp | 1 -
.../variant_shredding_write_plan_factory_test.cpp | 2 -
src/paimon/common/defs.cpp | 1 -
.../utils/binary_row_partition_computer_test.cpp | 17 +-
src/paimon/common/utils/string_utils.cpp | 60 ++++-
src/paimon/common/utils/string_utils.h | 8 +
src/paimon/common/utils/string_utils_test.cpp | 29 +++
src/paimon/core/append/append_only_writer_test.cpp | 41 +---
src/paimon/core/core_options.cpp | 4 +-
src/paimon/core/core_options_test.cpp | 9 +-
src/paimon/core/index/index_file_handler_test.cpp | 22 +-
.../index/pk/primary_key_index_definitions.cpp | 11 +-
.../core/io/infer_shredding_file_writer_test.cpp | 4 +-
src/paimon/core/io/single_file_writer_test.cpp | 15 +-
src/paimon/core/manifest/index_manifest_file.cpp | 8 +-
src/paimon/core/manifest/index_manifest_file.h | 3 +-
.../manifest/index_manifest_file_handler_test.cpp | 19 +-
src/paimon/core/manifest/manifest_file.cpp | 9 +-
src/paimon/core/manifest/manifest_file.h | 8 +-
src/paimon/core/manifest/manifest_file_test.cpp | 58 +++--
src/paimon/core/manifest/manifest_list.cpp | 7 +-
src/paimon/core/manifest/manifest_list.h | 3 +-
src/paimon/core/manifest/manifest_list_test.cpp | 15 +-
.../compact/aggregate/field_listagg_agg.h | 21 +-
.../compact/aggregate/field_listagg_agg_test.cpp | 40 +++-
.../operation/append_only_file_store_scan_test.cpp | 3 -
.../operation/append_only_file_store_write.cpp | 4 +-
.../core/operation/append_only_file_store_write.h | 2 +-
.../core/operation/commit/commit_scanner_test.cpp | 2 +-
.../core/operation/expire_snapshots_test.cpp | 6 +-
src/paimon/core/operation/file_store_commit.cpp | 1 -
.../core/operation/file_store_commit_impl_test.cpp | 232 +++++++------------
.../core/operation/file_store_commit_test.cpp | 6 +-
.../operation/key_value_file_store_scan_test.cpp | 7 +-
.../operation/key_value_file_store_write_test.cpp | 2 +-
.../core/operation/manifest_file_merger_test.cpp | 5 +-
src/paimon/core/table/sink/commit_message_test.cpp | 2 -
.../core/table/system/global_system_tables.cpp | 1 -
.../core/utils/file_store_path_factory_test.cpp | 1 -
src/paimon/core/utils/objects_file.h | 13 ++
.../format/avro/avro_stats_extractor_test.cpp | 3 +-
src/paimon/format/parquet/CMakeLists.txt | 1 +
src/paimon/format/parquet/parquet_format_defs.h | 2 +-
.../format/parquet/parquet_format_writer_test.cpp | 11 +-
.../format/parquet/parquet_writer_builder_test.cpp | 32 ++-
src/paimon/format/parquet/variant_parquet_test.cpp | 1 -
src/paimon/rest/rest_catalog_test.cpp | 1 -
test/inte/blob_table_inte_test.cpp | 252 ++++++++++-----------
test/inte/clean_inte_test.cpp | 56 ++---
test/inte/data_evolution_table_test.cpp | 26 +--
test/inte/global_index_test.cpp | 55 ++---
test/inte/nested_column_pruning_inte_test.cpp | 22 +-
test/inte/pk_compaction_inte_test.cpp | 2 +-
test/inte/read_inte_test.cpp | 104 ++++-----
test/inte/realtime_write_inte_test.cpp | 14 +-
test/inte/scan_and_read_inte_test.cpp | 3 +-
test/inte/variant_table_inte_test.cpp | 9 -
test/inte/write_and_read_inte_test.cpp | 244 ++++++++++----------
test/inte/write_inte_test.cpp | 163 ++++++-------
...manifest-ed03e3fd-3ff4-4d88-9d5d-683700f78967-0 | Bin 2720 -> 2290 bytes
...manifest-f2299c3d-c3f1-400f-ad3d-124e3a342389-0 | Bin 2747 -> 2285 bytes
...est-list-12b37c17-d02f-4409-8993-ac01a56210bf-0 | Bin 1182 -> 1112 bytes
...est-list-12b37c17-d02f-4409-8993-ac01a56210bf-1 | Bin 1197 -> 1117 bytes
...est-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-0 | Bin 250 -> 1006 bytes
...est-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-1 | Bin 1182 -> 1112 bytes
.../append_table_with_alter_table/schema/schema-0 | 4 +-
.../append_table_with_alter_table/schema/schema-1 | 4 +-
.../branch/branch-rt/schema/schema-0 | 4 +-
.../branch/branch-rt/schema/schema-1 | 4 +-
...manifest-6090bcc3-a5a6-4c51-bc5e-7af6498e5b99-0 | Bin 2659 -> 2218 bytes
...manifest-b204b605-41df-407f-85d9-722bef2754f4-0 | Bin 3159 -> 2387 bytes
...est-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-0 | Bin 370 -> 1006 bytes
...est-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-1 | Bin 1458 -> 1123 bytes
...est-list-779da483-08f4-4271-8a62-d6576f56bc80-0 | Bin 370 -> 1006 bytes
...est-list-779da483-08f4-4271-8a62-d6576f56bc80-1 | Bin 1443 -> 1125 bytes
.../append_table_with_rt_branch/schema/schema-0 | 4 +-
.../append_table_with_rt_branch/schema/schema-1 | 4 +-
...manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-0 | Bin 2886 -> 2235 bytes
...manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-1 | Bin 2887 -> 2240 bytes
...manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-0 | Bin 2890 -> 2245 bytes
...manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-1 | Bin 3284 -> 2295 bytes
...est-list-32868327-8690-4675-8593-f8c6ade64177-0 | Bin 1574 -> 1121 bytes
...est-list-32868327-8690-4675-8593-f8c6ade64177-1 | Bin 1527 -> 1113 bytes
...est-list-32868327-8690-4675-8593-f8c6ade64177-2 | Bin 1677 -> 1152 bytes
...est-list-32868327-8690-4675-8593-f8c6ade64177-3 | Bin 1529 -> 1106 bytes
...est-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-0 | Bin 392 -> 1006 bytes
...est-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-1 | Bin 1527 -> 1106 bytes
...est-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-2 | Bin 1527 -> 1106 bytes
...est-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-3 | Bin 392 -> 1006 bytes
...est-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-4 | Bin 1527 -> 1106 bytes
...est-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-5 | Bin 1531 -> 1113 bytes
.../pk_compact_lookup/schema/schema-0 | 4 +-
...manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-0 | Bin 2987 -> 2402 bytes
...manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-1 | Bin 3024 -> 2420 bytes
...manifest-daf4fd15-fe55-4216-8668-bac0b6a93781-0 | Bin 2697 -> 2219 bytes
...est-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-0 | Bin 316 -> 1006 bytes
...est-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-1 | Bin 1261 -> 1114 bytes
...est-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-2 | Bin 1261 -> 1114 bytes
...est-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-3 | Bin 1257 -> 1114 bytes
...est-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-0 | Bin 1322 -> 1122 bytes
...est-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-1 | Bin 1239 -> 1108 bytes
.../pk_table_with_mor/schema/schema-0 | 4 +-
.../pk_table_with_mor/schema/schema-1 | 4 +-
.../pk_table_with_mor/schema/schema-2 | 4 +-
.../pk_table_with_mor/schema/schema-3 | 4 +-
.../pk_table_with_mor/schema/schema-4 | 4 +-
...manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-0 | Bin 2382 -> 2207 bytes
...manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-1 | Bin 2841 -> 2447 bytes
...manifest-97ddb443-4cd9-42e9-9871-4d51d4fa1b49-0 | Bin 2688 -> 2339 bytes
...manifest-eb34bdd2-2c23-49af-b5aa-537a596d8fe3-0 | Bin 2385 -> 2209 bytes
...manifest-ed67f349-7c6f-4f51-8723-274a060042d1-0 | Bin 2638 -> 2295 bytes
...est-list-2663284f-7bac-441a-bfa1-d11bb24af95d-0 | Bin 1173 -> 1121 bytes
...est-list-2663284f-7bac-441a-bfa1-d11bb24af95d-1 | Bin 1171 -> 1121 bytes
...est-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-0 | Bin 250 -> 1006 bytes
...est-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1 | Bin 1173 -> 1121 bytes
...est-list-739ea106-ac05-4953-a13f-c7260482dbf1-0 | Bin 1370 -> 1182 bytes
...est-list-739ea106-ac05-4953-a13f-c7260482dbf1-1 | Bin 1177 -> 1118 bytes
...est-list-739ea106-ac05-4953-a13f-c7260482dbf1-2 | Bin 1428 -> 1210 bytes
...est-list-739ea106-ac05-4953-a13f-c7260482dbf1-3 | Bin 1189 -> 1118 bytes
...est-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-0 | Bin 1334 -> 1153 bytes
...est-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-1 | Bin 1179 -> 1118 bytes
.../parquet/append_09.db/append_09/schema/schema-0 | 2 +-
124 files changed, 874 insertions(+), 886 deletions(-)
diff --git a/examples/read_write_demo.cpp b/examples/read_write_demo.cpp
index 946b3503..bcf7db1e 100644
--- a/examples/read_write_demo.cpp
+++ b/examples/read_write_demo.cpp
@@ -56,8 +56,7 @@ arrow::Result<std::shared_ptr<arrow::StructArray>> PrepareData(const arrow::Fiel
paimon::Status Run(const std::string& root_path, const std::string& db_name,
const std::string& table_name) {
- std::map<std::string, std::string> options = {{paimon::Options::MANIFEST_FORMAT, "orc"},
- {paimon::Options::FILE_FORMAT, "parquet"},
+ std::map<std::string, std::string> options = {{paimon::Options::FILE_FORMAT, "parquet"},
{paimon::Options::FILE_SYSTEM, "local"}};
// create table
diff --git a/include/paimon/defs.h b/include/paimon/defs.h
index 0069ded7..e819c85d 100644
--- a/include/paimon/defs.h
+++ b/include/paimon/defs.h
@@ -164,10 +164,6 @@ struct PAIMON_EXPORT Options {
/// Default value is 8MB.
static const char MANIFEST_TARGET_FILE_SIZE[];
- /// "manifest.format" - Specify the message format of manifest files.
- /// Default value is avro.
- static const char MANIFEST_FORMAT[];
-
/// "manifest.compression" - File compression for manifest, default value is zstd.
static const char MANIFEST_COMPRESSION[];
diff --git a/src/paimon/common/data/shredding/map_shared_shredding_read_plan_factory_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_read_plan_factory_test.cpp
index de8ce58a..1763d6a2 100644
--- a/src/paimon/common/data/shredding/map_shared_shredding_read_plan_factory_test.cpp
+++ b/src/paimon/common/data/shredding/map_shared_shredding_read_plan_factory_test.cpp
@@ -232,7 +232,6 @@ class MapSharedShreddingReadPlanFactoryTest : public ::testing::Test {
std::map<std::string, std::string> options_ = {
{Options::FILE_SYSTEM, "local"},
{Options::FILE_FORMAT, "mock_format"},
- {Options::MANIFEST_FORMAT, "mock_format"},
{"fields.tags.map.storage-layout", "shared-shredding"},
{"fields.tags.map.shared-shredding.max-columns", "2"},
{"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
diff --git a/src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp b/src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp
index 6dbb8d79..1546acad 100644
--- a/src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp
+++ b/src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp
@@ -47,8 +47,6 @@ class VariantShreddingWritePlanFactoryTest : public ::testing::Test {
}
Result<CoreOptions> MakeOptions(std::map<std::string, std::string> options) const {
- // Keep the manifest format resolvable in test binaries without the avro plugin.
- options.emplace("manifest.format", "parquet");
return CoreOptions::FromMap(options);
}
diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp
index f0b0f561..962216f9 100644
--- a/src/paimon/common/defs.cpp
+++ b/src/paimon/common/defs.cpp
@@ -48,7 +48,6 @@ const char Options::FILE_COMPRESSION[] = "file.compression";
const char Options::FILE_COMPRESSION_ZSTD_LEVEL[] = "file.compression.zstd-level";
const char Options::FILE_BLOCK_SIZE[] = "file.block-size";
const char Options::MANIFEST_TARGET_FILE_SIZE[] = "manifest.target-file-size";
-const char Options::MANIFEST_FORMAT[] = "manifest.format";
const char Options::MANIFEST_COMPRESSION[] = "manifest.compression";
const char Options::MANIFEST_MERGE_MIN_COUNT[] = "manifest.merge-min-count";
const char Options::MANIFEST_FULL_COMPACTION_FILE_SIZE[] =
diff --git a/src/paimon/common/utils/binary_row_partition_computer_test.cpp b/src/paimon/common/utils/binary_row_partition_computer_test.cpp
index 18c42178..2a9d6b2d 100644
--- a/src/paimon/common/utils/binary_row_partition_computer_test.cpp
+++ b/src/paimon/common/utils/binary_row_partition_computer_test.cpp
@@ -251,20 +251,27 @@ TEST(BinaryRowPartitionComputerTest, TestNullOrWhitespaceOnlyStr) {
arrow::field("f0", arrow::utf8()),
arrow::field("f1", arrow::utf8()),
arrow::field("f2", arrow::utf8()),
+ arrow::field("f3", arrow::utf8()),
};
auto schema = arrow::schema(fields);
- std::vector<std::string> partition_keys = {"f0", "f1", "f2"};
+ std::vector<std::string> partition_keys = {"f0", "f1", "f2", "f3"};
ASSERT_OK_AND_ASSIGN(
std::unique_ptr<BinaryRowPartitionComputer> computer,
BinaryRowPartitionComputer::Create(partition_keys, schema, "__DEFAULT_PARTITION__",
/*legacy_partition_name_enabled=*/true, pool));
- ASSERT_OK_AND_ASSIGN(auto partition_key_values,
- computer->GeneratePartitionVector(BinaryRowGenerator::GenerateRow(
- {std::string(" "), std::string(""), std::string("ab ")}, pool.get())));
+ ASSERT_OK_AND_ASSIGN(
+ auto partition_key_values,
+ computer->GeneratePartitionVector(BinaryRowGenerator::GenerateRow(
+ {std::string(" "), std::string(""), std::string("ab "), std::string(u8"\u3000\u2000")},
+ pool.get())));
std::vector<std::pair<std::string, std::string>> expected = {
- {"f0", "__DEFAULT_PARTITION__"}, {"f1", "__DEFAULT_PARTITION__"}, {"f2", "ab "}};
+ {"f0", "__DEFAULT_PARTITION__"},
+ {"f1", "__DEFAULT_PARTITION__"},
+ {"f2", "ab "},
+ {"f3", "__DEFAULT_PARTITION__"},
+ };
ASSERT_EQ(partition_key_values, expected);
}
diff --git a/src/paimon/common/utils/string_utils.cpp b/src/paimon/common/utils/string_utils.cpp
index 869e184d..7b3ed96b 100644
--- a/src/paimon/common/utils/string_utils.cpp
+++ b/src/paimon/common/utils/string_utils.cpp
@@ -37,6 +37,14 @@ bool IsTrimCharacter(unsigned char c) {
return c <= 0x20;
}
+bool IsJavaWhitespace(uint32_t code_point) {
+ return (code_point >= 0x0009 && code_point <= 0x000d) ||
+ (code_point >= 0x001c && code_point <= 0x0020) || code_point == 0x1680 ||
+ (code_point >= 0x2000 && code_point <= 0x2006) ||
+ (code_point >= 0x2008 && code_point <= 0x200a) || code_point == 0x2028 ||
+ code_point == 0x2029 || code_point == 0x205f || code_point == 0x3000;
+}
+
char ToAsciiLower(unsigned char c) {
return c >= 'A' && c <= 'Z' ? static_cast<char>(c + ('a' - 'A')) : static_cast<char>(c);
}
@@ -85,18 +93,58 @@ bool StringUtils::EndsWith(const std::string& str, const std::string& suffix) {
size_t s2 = suffix.size();
return (s1 >= s2) && (str.compare(s1 - s2, s2, suffix) == 0);
}
-bool StringUtils::IsNullOrWhitespaceOnly(const std::string& str) {
- if (str.empty()) {
- return true;
- }
- for (char c : str) {
- if (!std::isspace(static_cast<unsigned char>(c))) {
+
+bool StringUtils::IsBlank(std::string_view str) {
+ size_t offset = 0;
+ while (offset < str.size()) {
+ const auto first = static_cast<uint8_t>(str[offset]);
+ uint32_t code_point = 0;
+ size_t length = 0;
+ if (first <= 0x7f) {
+ code_point = first;
+ length = 1;
+ } else if (first >= 0xc2 && first <= 0xdf) {
+ code_point = first & 0x1f;
+ length = 2;
+ } else if (first >= 0xe0 && first <= 0xef) {
+ code_point = first & 0x0f;
+ length = 3;
+ } else if (first >= 0xf0 && first <= 0xf4) {
+ code_point = first & 0x07;
+ length = 4;
+ } else {
+ return false;
+ }
+ if (offset + length > str.size()) {
+ return false;
+ }
+ for (size_t i = 1; i < length; ++i) {
+ const auto continuation = static_cast<uint8_t>(str[offset + i]);
+ if ((continuation & 0xc0) != 0x80) {
+ return false;
+ }
+ code_point = (code_point << 6) | (continuation & 0x3f);
+ }
+ if ((length == 3 && code_point < 0x800) || (length == 4 && code_point < 0x10000) ||
+ (code_point >= 0xd800 && code_point <= 0xdfff) || code_point > 0x10ffff) {
return false;
}
+ if (!IsJavaWhitespace(code_point)) {
+ return false;
+ }
+ offset += length;
}
return true;
}
+bool StringUtils::IsNullOrWhitespaceOnly(const std::string& str) {
+ return IsBlank(str);
+}
+
+bool StringUtils::IsEmptyAfterTrim(std::string_view str) {
+ return std::all_of(str.begin(), str.end(), [](unsigned char c) { return IsTrimCharacter(c); });
+}
+
void StringUtils::Trim(std::string* str) {
auto first = std::find_if_not(str->begin(), str->end(),
[](unsigned char c) { return IsTrimCharacter(c); });
diff --git a/src/paimon/common/utils/string_utils.h b/src/paimon/common/utils/string_utils.h
index 7681a8d9..49c0aff2 100644
--- a/src/paimon/common/utils/string_utils.h
+++ b/src/paimon/common/utils/string_utils.h
@@ -31,6 +31,7 @@
#include <set>
#include <sstream>
#include <string>
+#include <string_view>
#include <system_error>
#include <vector>
@@ -96,8 +97,15 @@ class PAIMON_EXPORT StringUtils {
static bool EndsWith(const std::string& str, const std::string& suffix);
+ /// Returns true if the string is empty or contains only characters recognized by Java
+ /// Character.isWhitespace.
+ static bool IsBlank(std::string_view str);
+
static bool IsNullOrWhitespaceOnly(const std::string& str);
+ /// Returns true if Java String::trim would produce an empty string.
+ static bool IsEmptyAfterTrim(std::string_view str);
+
static void Trim(std::string* str);
static std::string ToLowerCase(const std::string& str);
diff --git a/src/paimon/common/utils/string_utils_test.cpp b/src/paimon/common/utils/string_utils_test.cpp
index a4f23078..89b0334f 100644
--- a/src/paimon/common/utils/string_utils_test.cpp
+++ b/src/paimon/common/utils/string_utils_test.cpp
@@ -21,6 +21,7 @@
#include <limits>
#include <memory>
+#include <vector>
#include "gtest/gtest.h"
#include "paimon/status.h"
@@ -209,6 +210,34 @@ TEST_F(StringUtilsTest, TestIsNullOrWhitespaceOnly) {
auto ret = StringUtils::IsNullOrWhitespaceOnly(str);
ASSERT_TRUE(ret);
}
+ ASSERT_TRUE(StringUtils::IsNullOrWhitespaceOnly(u8"\u3000\u2000"));
+}
+
+TEST_F(StringUtilsTest, TestIsBlank) {
+ const std::vector<std::string> blank_strings = {
+ "", " ", " ", "\t", "\n", "\r",
+ "\r\n", " \t\n\r ", u8"\u1680", u8"\u2000", u8"\u3000", u8" \t\u3000\u2000\n"};
+ for (const std::string& blank : blank_strings) {
+ ASSERT_TRUE(StringUtils::IsBlank(blank)) << blank;
+ }
+
+ ASSERT_FALSE(StringUtils::IsBlank("user1"));
+ ASSERT_FALSE(StringUtils::IsBlank(" user1 "));
+ ASSERT_FALSE(StringUtils::IsBlank(u8"\u00a0"));
+ ASSERT_FALSE(StringUtils::IsBlank(std::string("\xc0\x80", 2)));
+
+ // Non-breaking or otherwise excluded by Character.isWhitespace.
+ ASSERT_FALSE(StringUtils::IsBlank(u8"\u2007")); // FIGURE SPACE
+ ASSERT_FALSE(StringUtils::IsBlank(u8"\u202f")); // NARROW NO-BREAK SPACE
+ ASSERT_FALSE(StringUtils::IsBlank(u8"\u0085")); // NEL
+ ASSERT_FALSE(StringUtils::IsBlank(u8"\u180e")); // Not whitespace since Java 8
+}
+
+TEST_F(StringUtilsTest, TestIsEmptyAfterTrim) {
+ ASSERT_TRUE(StringUtils::IsEmptyAfterTrim(""));
+ ASSERT_TRUE(StringUtils::IsEmptyAfterTrim(" \t\x1c"));
+ ASSERT_FALSE(StringUtils::IsEmptyAfterTrim(" a "));
+ ASSERT_FALSE(StringUtils::IsEmptyAfterTrim(u8"\u3000"));
}
TEST_F(StringUtilsTest, TestToLowerCase) {
diff --git a/src/paimon/core/append/append_only_writer_test.cpp b/src/paimon/core/append/append_only_writer_test.cpp
index bf64420e..4a4d5dc6 100644
--- a/src/paimon/core/append/append_only_writer_test.cpp
+++ b/src/paimon/core/append/append_only_writer_test.cpp
@@ -190,7 +190,6 @@ class AppendOnlyWriterTest : public testing::Test {
std::map<std::string, std::string> raw_options = {
{Options::FILE_SYSTEM, "local"},
{Options::FILE_FORMAT, "mock_format"},
- {Options::MANIFEST_FORMAT, "mock_format"},
};
for (const auto& [key, value] : overrides) {
raw_options[key] = value;
@@ -334,7 +333,6 @@ TEST_F(AppendOnlyWriterTest, TestEmptyCommits) {
std::map<std::string, std::string> raw_options;
raw_options[Options::FILE_FORMAT] = "mock_format";
raw_options[Options::FILE_SYSTEM] = "local";
- raw_options[Options::MANIFEST_FORMAT] = "mock_format";
ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(raw_options));
arrow::FieldVector fields = {
@@ -364,7 +362,6 @@ TEST_F(AppendOnlyWriterTest, TestWriteAndPrepareCommit) {
std::map<std::string, std::string> raw_options;
raw_options[Options::FILE_FORMAT] = "mock_format";
raw_options[Options::FILE_SYSTEM] = "local";
- raw_options[Options::MANIFEST_FORMAT] = "mock_format";
ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(raw_options));
arrow::FieldVector fields = {
@@ -446,7 +443,6 @@ TEST_F(AppendOnlyWriterTest, TestWriteAndClose) {
std::map<std::string, std::string> raw_options;
raw_options[Options::FILE_FORMAT] = "orc";
raw_options[Options::FILE_SYSTEM] = "local";
- raw_options[Options::MANIFEST_FORMAT] = "orc";
ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(raw_options));
arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())};
@@ -491,7 +487,6 @@ TEST_F(AppendOnlyWriterTest, TestInvalidRowKind) {
std::map<std::string, std::string> raw_options;
raw_options[Options::FILE_FORMAT] = "orc";
raw_options[Options::FILE_SYSTEM] = "local";
- raw_options[Options::MANIFEST_FORMAT] = "orc";
ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(raw_options));
arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())};
@@ -632,8 +627,7 @@ TEST_F(AppendOnlyWriterTest,
}
TEST_F(AppendOnlyWriterTest, TestCloseDeletesCompactAfterFiles) {
- auto options =
- CreateOptions({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}});
+ auto options = CreateOptions({{Options::FILE_FORMAT, "orc"}});
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
auto path_factory = CreatePathFactory(dir->Str(), "orc", options);
@@ -740,9 +734,8 @@ TEST_F(AppendOnlyWriterTest, TestCompactPassesFullCompactionFlag) {
}
TEST_F(AppendOnlyWriterTest, TestWriteWithSingleBlobField) {
- auto options = CreateOptions({{Options::FILE_FORMAT, "orc"},
- {Options::MANIFEST_FORMAT, "orc"},
- {Options::TARGET_FILE_ROW_NUM, "1"}});
+ auto options =
+ CreateOptions({{Options::FILE_FORMAT, "orc"}, {Options::TARGET_FILE_ROW_NUM, "1"}});
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
auto path_factory = CreatePathFactory(dir->Str(), "orc", options);
@@ -780,8 +773,7 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithSingleBlobField) {
}
TEST_F(AppendOnlyWriterTest, TestWriteWithOnlyBlobField) {
- auto options =
- CreateOptions({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}});
+ auto options = CreateOptions({{Options::FILE_FORMAT, "orc"}});
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
auto path_factory = CreatePathFactory(dir->Str(), "orc", options);
@@ -828,8 +820,7 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithOnlyBlobField) {
}
TEST_F(AppendOnlyWriterTest, TestWriteWithMultipleBlobFields) {
- auto options =
- CreateOptions({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}});
+ auto options = CreateOptions({{Options::FILE_FORMAT, "orc"}});
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
auto path_factory = CreatePathFactory(dir->Str(), "orc", options);
@@ -896,9 +887,8 @@ TEST_F(AppendOnlyWriterTest, TestMultiplePrepareCommitSequenceContinuity) {
}
TEST_F(AppendOnlyWriterTest, TestWriteValidBlobViewField) {
- auto options = CreateOptions({{Options::FILE_FORMAT, "orc"},
- {Options::MANIFEST_FORMAT, "orc"},
- {Options::BLOB_VIEW_FIELD, "view"}});
+ auto options =
+ CreateOptions({{Options::FILE_FORMAT, "orc"}, {Options::BLOB_VIEW_FIELD, "view"}});
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
auto path_factory = CreatePathFactory(dir->Str(), "orc", options);
@@ -933,9 +923,8 @@ TEST_F(AppendOnlyWriterTest, TestWriteValidBlobViewField) {
}
TEST_F(AppendOnlyWriterTest, TestWriteInvalidBlobViewFieldRejected) {
- auto options = CreateOptions({{Options::FILE_FORMAT, "orc"},
- {Options::MANIFEST_FORMAT, "orc"},
- {Options::BLOB_VIEW_FIELD, "view"}});
+ auto options =
+ CreateOptions({{Options::FILE_FORMAT, "orc"}, {Options::BLOB_VIEW_FIELD, "view"}});
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
auto path_factory = CreatePathFactory(dir->Str(), "orc", options);
@@ -977,7 +966,7 @@ INSTANTIATE_TEST_SUITE_P(FileFormats, AppendOnlyWriterShreddingTest,
TEST_F(AppendOnlyWriterTest, TestSharedShreddingMapRejectsAvroFormatOnCommit) {
auto options = CreateOptions({
{Options::FILE_FORMAT, "avro"},
- {Options::MANIFEST_FORMAT, "avro"},
+
{"fields.tags.map.storage-layout", "shared-shredding"},
{"fields.tags.map.shared-shredding.max-columns", "3"},
{"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
@@ -1013,7 +1002,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapFieldContent) {
// Configure with shared-shredding map on "tags" field, K=3.
auto options = CreateOptions({
{Options::FILE_FORMAT, format},
- {Options::MANIFEST_FORMAT, format},
{Options::TARGET_FILE_ROW_NUM, "2"},
{"fields.tags.map.storage-layout", "shared-shredding"},
{"fields.tags.map.shared-shredding.max-columns", "3"},
@@ -1087,7 +1075,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllEmptyFirstFile) {
std::string format = GetFormat();
auto options = CreateOptions({
{Options::FILE_FORMAT, format},
- {Options::MANIFEST_FORMAT, format},
{"fields.tags.map.storage-layout", "shared-shredding"},
{"fields.tags.map.shared-shredding.max-columns", "3"},
{"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
@@ -1145,7 +1132,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllNullThenAllEmptyF
std::string format = GetFormat();
auto options = CreateOptions({
{Options::FILE_FORMAT, format},
- {Options::MANIFEST_FORMAT, format},
{Options::TARGET_FILE_ROW_NUM, "2"},
{"fields.tags.map.storage-layout", "shared-shredding"},
{"fields.tags.map.shared-shredding.max-columns", "3"},
@@ -1258,7 +1244,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapWithOverflow) {
// K=2, write rows with 3+ keys to trigger overflow.
auto options = CreateOptions({
{Options::FILE_FORMAT, format},
- {Options::MANIFEST_FORMAT, format},
{"fields.tags.map.storage-layout", "shared-shredding"},
{"fields.tags.map.shared-shredding.max-columns", "2"},
{"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
@@ -1330,7 +1315,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapWithLruPlacemen
std::string format = GetFormat();
auto options = CreateOptions({
{Options::FILE_FORMAT, format},
- {Options::MANIFEST_FORMAT, format},
{"fields.tags.map.storage-layout", "shared-shredding"},
{"fields.tags.map.shared-shredding.max-columns", "3"},
{"fields.tags.map.shared-shredding.column-placement-policy", "lru"},
@@ -1397,7 +1381,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossRol
std::string format = GetFormat();
auto options = CreateOptions({
{Options::FILE_FORMAT, format},
- {Options::MANIFEST_FORMAT, format},
{Options::TARGET_FILE_ROW_NUM, "1"},
{"fields.tags.map.storage-layout", "shared-shredding"},
{"fields.tags.map.shared-shredding.max-columns", "10"},
@@ -1504,7 +1487,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapFirstFileUsesConfigu
std::string format = GetFormat();
auto options = CreateOptions({
{Options::FILE_FORMAT, format},
- {Options::MANIFEST_FORMAT, format},
{"fields.tags.map.storage-layout", "shared-shredding"},
{"fields.tags.map.shared-shredding.max-columns", "10"},
{"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
@@ -1563,7 +1545,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsAdaptA
// Two shared-shredding MAP fields with different initial K: tags(K=8), attrs(K=4).
auto options = CreateOptions({
{Options::FILE_FORMAT, format},
- {Options::MANIFEST_FORMAT, format},
{Options::TARGET_FILE_ROW_NUM, "1"},
{"fields.tags.map.storage-layout", "shared-shredding"},
{"fields.tags.map.shared-shredding.max-columns", "8"},
@@ -1682,7 +1663,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapDataFileMetaInfo) {
// Verify PrepareCommit returns correct DataFileMeta for shared-shredding map files.
auto options = CreateOptions({
{Options::FILE_FORMAT, format},
- {Options::MANIFEST_FORMAT, format},
{"fields.tags.map.storage-layout", "shared-shredding"},
{"fields.tags.map.shared-shredding.max-columns", "3"},
{"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
@@ -1762,7 +1742,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapWithBlobSeparation)
// This tests that blob separation + shredding works correctly together.
auto options = CreateOptions({
{Options::FILE_FORMAT, format},
- {Options::MANIFEST_FORMAT, format},
{"fields.tags.map.storage-layout", "shared-shredding"},
{"fields.tags.map.shared-shredding.max-columns", "3"},
{"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp
index 385b2300..aaa5b645 100644
--- a/src/paimon/core/core_options.cpp
+++ b/src/paimon/core/core_options.cpp
@@ -610,9 +610,9 @@ struct CoreOptions::Impl {
// Parse manifest file configurations: format, compression, merge, and compaction thresholds.
Status ParseManifestOptions(const ConfigParser& parser) {
- // Parse manifest.format - manifest file format, default "avro"
+ // Parse the legacy manifest.format option for reading existing tables.
PAIMON_RETURN_NOT_OK(parser.ParseObject<FileFormatFactory>(
- Options::MANIFEST_FORMAT, /*default_identifier=*/"avro", &manifest_file_format));
+ "manifest.format", /*default_identifier=*/"avro", &manifest_file_format));
// Parse manifest.compression - manifest file compression, default "zstd"
PAIMON_RETURN_NOT_OK(parser.Parse(Options::MANIFEST_COMPRESSION, &manifest_compression));
// Parse manifest.target-file-size - suggested manifest file size, default 8MB
diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp
index 56ab466d..7bd5b5e7 100644
--- a/src/paimon/core/core_options_test.cpp
+++ b/src/paimon/core/core_options_test.cpp
@@ -37,7 +37,8 @@ namespace paimon::test {
TEST(CoreOptionsTest, TestDefaultValue) {
ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({}));
- ASSERT_EQ(core_options.GetManifestFormat()->Identifier(), "avro");
+ std::shared_ptr<FileFormat> manifest_format = core_options.GetManifestFormat();
+ ASSERT_EQ(manifest_format->Identifier(), "avro");
ASSERT_EQ(core_options.GetFileFormat()->Identifier(), "parquet");
ASSERT_EQ(nullptr, core_options.GetChangelogFileFormat());
ASSERT_EQ(core_options.GetWriteFileFormat(0)->Identifier(), "parquet");
@@ -198,7 +199,7 @@ TEST(CoreOptionsTest, TestFromMap) {
{Options::FILE_SYSTEM, "Local"},
{Options::FILE_FORMAT, "ORC"},
{Options::CHANGELOG_FILE_FORMAT, "avro"},
- {Options::MANIFEST_FORMAT, "avRo"},
+ {"manifest.format", "ORC"},
{Options::BUCKET, "3"},
{Options::PAGE_SIZE, "128 kb"},
{Options::TARGET_FILE_SIZE, "512MB"},
@@ -339,8 +340,8 @@ TEST(CoreOptionsTest, TestFromMap) {
ASSERT_EQ(core_options.GetWriteFileFormat(1)->Identifier(), "orc");
ASSERT_EQ(core_options.GetWriteFileFormat(3)->Identifier(), "parquet");
- auto manifest_format = core_options.GetManifestFormat();
- ASSERT_EQ(manifest_format->Identifier(), "avro");
+ std::shared_ptr<FileFormat> manifest_format = core_options.GetManifestFormat();
+ ASSERT_EQ(manifest_format->Identifier(), "orc");
ASSERT_EQ(3, core_options.GetBucket());
ASSERT_EQ(128 * 1024L, core_options.GetPageSize());
diff --git a/src/paimon/core/index/index_file_handler_test.cpp b/src/paimon/core/index/index_file_handler_test.cpp
index a23591f9..a4b5295f 100644
--- a/src/paimon/core/index/index_file_handler_test.cpp
+++ b/src/paimon/core/index/index_file_handler_test.cpp
@@ -50,10 +50,16 @@ class IndexFileHandlerTest : public testing::Test {
}
Result<std::unique_ptr<IndexFileHandler>> CreateIndexFileHandler(
- const std::string& table_path, const CoreOptions& core_options) const {
- SchemaManager schema_manager(core_options.GetFileSystem(), table_path);
+ const std::string& table_path, const CoreOptions& override_options) const {
+ SchemaManager schema_manager(override_options.GetFileSystem(), table_path);
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<TableSchema> table_schema,
schema_manager.ReadSchema(/*schema_id=*/0));
+ std::map<std::string, std::string> options = table_schema->Options();
+ for (const auto& [key, value] : override_options.ToMap()) {
+ options[key] = value;
+ }
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options,
+ CoreOptions::FromMap(options, override_options.GetFileSystem()));
auto schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields());
PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> external_paths,
core_options.CreateExternalPaths());
@@ -151,8 +157,7 @@ TEST_F(IndexFileHandlerTest, TestScan) {
std::string table_path = paimon::test::GetDataDir() +
"/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality/";
- ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
- CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}}));
+ ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({}));
ASSERT_OK_AND_ASSIGN(std::unique_ptr<IndexFileHandler> index_file_handler,
CreateIndexFileHandler(table_path, core_options));
@@ -203,8 +208,7 @@ TEST_F(IndexFileHandlerTest, TestScan) {
TEST_F(IndexFileHandlerTest, Test09VersionScan) {
std::string table_path = paimon::test::GetDataDir() + "/orc/pk_09.db/pk_09/";
- ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
- CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}}));
+ ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({}));
ASSERT_OK_AND_ASSIGN(std::unique_ptr<IndexFileHandler> index_file_handler,
CreateIndexFileHandler(table_path, core_options));
@@ -254,8 +258,7 @@ TEST_F(IndexFileHandlerTest, Test09VersionScan) {
TEST_F(IndexFileHandlerTest, TestScanWithNoIndexManifest) {
std::string table_path = paimon::test::GetDataDir() + "/orc/pk_09.db/pk_09/";
- ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
- CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}}));
+ ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({}));
ASSERT_OK_AND_ASSIGN(std::unique_ptr<IndexFileHandler> index_file_handler,
CreateIndexFileHandler(table_path, core_options));
@@ -292,8 +295,7 @@ TEST_F(IndexFileHandlerTest, TestScanByPartitionBucketAndReadAllDeletionVectors)
std::string table_path = paimon::test::GetDataDir() +
"/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality/";
- ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
- CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}}));
+ ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({}));
ASSERT_OK_AND_ASSIGN(std::unique_ptr<IndexFileHandler> index_file_handler,
CreateIndexFileHandler(table_path, core_options));
diff --git a/src/paimon/core/index/pk/primary_key_index_definitions.cpp b/src/paimon/core/index/pk/primary_key_index_definitions.cpp
index 214338f4..cc10d56c 100644
--- a/src/paimon/core/index/pk/primary_key_index_definitions.cpp
+++ b/src/paimon/core/index/pk/primary_key_index_definitions.cpp
@@ -102,7 +102,10 @@ Result<std::map<std::string, std::string>> SortedIndexOptions(
std::string option_key =
fmt::format("{}{}.{}.index.options", kFieldScopedPrefix, column, option_family);
auto iter = table_options.find(option_key);
- if (iter == table_options.end() || StringUtils::IsNullOrWhitespaceOnly(iter->second)) {
+ if (iter == table_options.end()) {
+ return resolved;
+ }
+ if (StringUtils::IsEmptyAfterTrim(iter->second)) {
return resolved;
}
@@ -113,11 +116,13 @@ Result<std::map<std::string, std::string>> SortedIndexOptions(
fmt::format("{} must be a JSON object of option key-value pairs.", option_key));
}
for (auto member = document.MemberBegin(); member != document.MemberEnd(); ++member) {
- if (!member->name.IsString() ||
- StringUtils::IsNullOrWhitespaceOnly(member->name.GetString())) {
+ if (!member->name.IsString()) {
return Status::Invalid(fmt::format("{} contains an empty option key.", option_key));
}
std::string key = member->name.GetString();
+ if (StringUtils::IsEmptyAfterTrim(key)) {
+ return Status::Invalid(fmt::format("{} contains an empty option key.", option_key));
+ }
if (member->value.IsNull()) {
return Status::Invalid(
fmt::format("{} value for key {} must not be null.", option_key, key));
diff --git a/src/paimon/core/io/infer_shredding_file_writer_test.cpp b/src/paimon/core/io/infer_shredding_file_writer_test.cpp
index b0e82756..ff1cc405 100644
--- a/src/paimon/core/io/infer_shredding_file_writer_test.cpp
+++ b/src/paimon/core/io/infer_shredding_file_writer_test.cpp
@@ -128,9 +128,7 @@ class InferShreddingFileWriterTest : public ::testing::Test {
MakeWriter(int32_t buffer_rows) {
std::map<std::string, std::string> option_map = {
{"variant.inferShreddingSchema", "true"},
- {"variant.shredding.maxInferBufferRow", std::to_string(buffer_rows)},
- // Keep the manifest format resolvable in test binaries without the avro plugin.
- {"manifest.format", "parquet"}};
+ {"variant.shredding.maxInferBufferRow", std::to_string(buffer_rows)}};
EXPECT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map));
auto plan_factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_);
auto create_inner = [this](const std::shared_ptr<ShreddingBatchConverter>& converter)
diff --git a/src/paimon/core/io/single_file_writer_test.cpp b/src/paimon/core/io/single_file_writer_test.cpp
index 78fce54c..121343bc 100644
--- a/src/paimon/core/io/single_file_writer_test.cpp
+++ b/src/paimon/core/io/single_file_writer_test.cpp
@@ -58,9 +58,8 @@ TEST(SingleFileWriterTest, TestSimple) {
return Status::OK();
};
SimpleSingleFileWriter writer("zstd", converter);
- ASSERT_OK_AND_ASSIGN(
- CoreOptions options,
- CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "orc"}}));
+ ASSERT_OK_AND_ASSIGN(CoreOptions options,
+ CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
auto file_format = options.GetWriteFileFormat(/*level=*/0);
auto file_system = options.GetFileSystem();
ArrowSchema arrow_schema;
@@ -90,9 +89,8 @@ TEST(SingleFileWriterTest, TestInvalidConvert) {
return Status::Invalid("");
};
SimpleSingleFileWriter writer("zstd", converter);
- ASSERT_OK_AND_ASSIGN(
- CoreOptions options,
- CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "orc"}}));
+ ASSERT_OK_AND_ASSIGN(CoreOptions options,
+ CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
auto file_format = options.GetWriteFileFormat(/*level=*/0);
auto file_system = options.GetFileSystem();
ArrowSchema arrow_schema;
@@ -119,9 +117,8 @@ TEST(SingleFileWriterTest, CompletionCallbackFailureIsTerminal) {
return Status::OK();
};
SimpleSingleFileWriter writer("zstd", converter);
- ASSERT_OK_AND_ASSIGN(
- CoreOptions options,
- CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "orc"}}));
+ ASSERT_OK_AND_ASSIGN(CoreOptions options,
+ CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
auto file_format = options.GetWriteFileFormat(/*level=*/0);
auto file_system = options.GetFileSystem();
ArrowSchema arrow_schema;
diff --git a/src/paimon/core/manifest/index_manifest_file.cpp b/src/paimon/core/manifest/index_manifest_file.cpp
index 648c7302..ab8ed8c6 100644
--- a/src/paimon/core/manifest/index_manifest_file.cpp
+++ b/src/paimon/core/manifest/index_manifest_file.cpp
@@ -61,19 +61,21 @@ Result<std::unique_ptr<IndexManifestFile>> IndexManifestFile::Create(
std::shared_ptr<PathFactory> index_manifest_file_factory =
path_factory->CreateIndexManifestFileFactory();
- return std::unique_ptr<IndexManifestFile>(
- new IndexManifestFile(file_system, reader_builder, writer_builder, compression,
- index_manifest_file_factory, bucket_mode, options.GetCache(), pool));
+ return std::unique_ptr<IndexManifestFile>(new IndexManifestFile(
+ file_system, reader_builder, writer_builder, file_format->Identifier(), compression,
+ index_manifest_file_factory, bucket_mode, options.GetCache(), pool));
}
IndexManifestFile::IndexManifestFile(const std::shared_ptr<FileSystem>& file_system,
const std::shared_ptr<ReaderBuilder>& reader_builder,
const std::shared_ptr<WriterBuilder>& writer_builder,
+ const std::string& file_format_identifier,
const std::string& compression,
const std::shared_ptr<PathFactory>& path_factory,
int32_t bucket_mode, const std::shared_ptr<Cache>& cache,
const std::shared_ptr<MemoryPool>& pool)
: ObjectsFile<IndexManifestEntry>(file_system, reader_builder, writer_builder,
+ file_format_identifier,
std::make_unique<IndexManifestEntrySerializer>(pool),
compression, path_factory, cache, pool),
bucket_mode_(bucket_mode) {}
diff --git a/src/paimon/core/manifest/index_manifest_file.h b/src/paimon/core/manifest/index_manifest_file.h
index f6fd46f3..99122327 100644
--- a/src/paimon/core/manifest/index_manifest_file.h
+++ b/src/paimon/core/manifest/index_manifest_file.h
@@ -61,10 +61,9 @@ class IndexManifestFile : public ObjectsFile<IndexManifestEntry> {
IndexManifestFile(const std::shared_ptr<FileSystem>& file_system,
const std::shared_ptr<ReaderBuilder>& reader_builder,
const std::shared_ptr<WriterBuilder>& writer_builder,
- const std::string& compression,
+ const std::string& file_format_identifier, const std::string& compression,
const std::shared_ptr<PathFactory>& path_factory, int32_t bucket_mode,
const std::shared_ptr<Cache>& cache, const std::shared_ptr<MemoryPool>& pool);
-
const int32_t bucket_mode_;
};
} // namespace paimon
diff --git a/src/paimon/core/manifest/index_manifest_file_handler_test.cpp b/src/paimon/core/manifest/index_manifest_file_handler_test.cpp
index 8d20bc05..9661697c 100644
--- a/src/paimon/core/manifest/index_manifest_file_handler_test.cpp
+++ b/src/paimon/core/manifest/index_manifest_file_handler_test.cpp
@@ -49,9 +49,10 @@ class IndexManifestFileHandlerTest : public testing::Test {
ASSERT_TRUE(dir_ != nullptr);
}
- Result<std::unique_ptr<IndexManifestFile>> CreateManifestFile(int32_t bucket_mode) const {
+ Result<std::unique_ptr<IndexManifestFile>> CreateManifestFile(
+ int32_t bucket_mode, const std::string& file_format_identifier = "avro") const {
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileFormat> file_format,
- FileFormatFactory::Get("orc", {}));
+ FileFormatFactory::Get(file_format_identifier, {}));
auto schema = arrow::schema({arrow::field("f0", arrow::int32())});
PAIMON_ASSIGN_OR_RAISE(
std::shared_ptr<FileStorePathFactory> path_factory,
@@ -61,8 +62,7 @@ class IndexManifestFileHandlerTest : public testing::Test {
/*legacy_partition_name_enabled=*/true, /*external_paths=*/{},
/*global_index_external_path=*/std::nullopt,
/*index_file_in_data_file_dir=*/false, pool_));
- PAIMON_ASSIGN_OR_RAISE(CoreOptions options,
- CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}}));
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap({}));
return IndexManifestFile::Create(dir_->GetFileSystem(), file_format, "zstd", path_factory,
bucket_mode, pool_, options);
}
@@ -219,6 +219,17 @@ TEST_F(IndexManifestFileHandlerTest, GlobalCombinerOverwritesDuplicateAddedEntri
ASSERT_EQ(written_entries[0].index_file->RowCount(), 20);
}
+TEST_F(IndexManifestFileHandlerTest, LegacyManifestFormatIsReadOnly) {
+ ASSERT_OK_AND_ASSIGN(auto index_manifest_file,
+ CreateManifestFile(/*bucket_mode=*/4, /*file_format_identifier=*/"orc"));
+ auto partition = BinaryRow::EmptyRow();
+ std::vector<IndexManifestEntry> entries = {
+ MakeEntry(FileKind::Add(), partition, /*bucket=*/0, /*index_type=*/"BTREE", "global-0", 1)};
+
+ ASSERT_NOK_WITH_MSG(index_manifest_file->WriteIndexFiles(std::nullopt, entries),
+ "manifest.format 'orc' is read-only");
+}
+
TEST_F(IndexManifestFileHandlerTest, GlobalDvCombinerReplacesIndexFileInBucketUnawareMode) {
ASSERT_OK_AND_ASSIGN(auto index_manifest_file, CreateManifestFile(/*bucket_mode=*/-1));
diff --git a/src/paimon/core/manifest/manifest_file.cpp b/src/paimon/core/manifest/manifest_file.cpp
index a556b82e..0fdb78d7 100644
--- a/src/paimon/core/manifest/manifest_file.cpp
+++ b/src/paimon/core/manifest/manifest_file.cpp
@@ -51,12 +51,14 @@ class MemoryPool;
ManifestFile::ManifestFile(const std::shared_ptr<FileSystem>& file_system,
const std::shared_ptr<ReaderBuilder>& reader_builder,
const std::shared_ptr<WriterBuilder>& writer_builder,
+ const std::string& file_format_identifier,
const std::string& compression,
const std::shared_ptr<PathFactory>& path_factory,
int64_t target_file_size, const std::shared_ptr<MemoryPool>& pool,
const CoreOptions& options,
const std::shared_ptr<arrow::Schema>& partition_type)
: ObjectsFile<ManifestEntry>(file_system, reader_builder, writer_builder,
+ file_format_identifier,
std::make_unique<ManifestEntrySerializer>(pool), compression,
path_factory, options.GetCache(), pool),
target_file_size_(target_file_size),
@@ -82,9 +84,9 @@ Result<std::unique_ptr<ManifestFile>> ManifestFile::Create(
writer_builder->WithMemoryPool(pool);
std::shared_ptr<PathFactory> manifest_file_factory = path_factory->CreateManifestFileFactory();
- return std::unique_ptr<ManifestFile>(
- new ManifestFile(file_system, reader_builder, writer_builder, compression,
- manifest_file_factory, target_file_size, pool, options, partition_type));
+ return std::unique_ptr<ManifestFile>(new ManifestFile(
+ file_system, reader_builder, writer_builder, file_format->Identifier(), compression,
+ manifest_file_factory, target_file_size, pool, options, partition_type));
}
Status ManifestFile::ReadBucketEntries(const std::string& file_name, int32_t bucket,
@@ -112,6 +114,7 @@ Result<std::vector<ManifestFileMeta>> ManifestFile::Write(
if (entries.empty()) {
return std::vector<ManifestFileMeta>();
}
+ PAIMON_RETURN_NOT_OK(ValidateWrite());
auto converter = [this](ManifestEntry entry, ::ArrowArray* dest) -> Status {
if (!to_array_converter_) {
PAIMON_ASSIGN_OR_RAISE(to_array_converter_, MetaToArrowArrayConverter::Create(
diff --git a/src/paimon/core/manifest/manifest_file.h b/src/paimon/core/manifest/manifest_file.h
index 0211e14d..528bb7bf 100644
--- a/src/paimon/core/manifest/manifest_file.h
+++ b/src/paimon/core/manifest/manifest_file.h
@@ -70,11 +70,11 @@ class ManifestFile : public ObjectsFile<ManifestEntry> {
ManifestFile(const std::shared_ptr<FileSystem>& file_system,
const std::shared_ptr<ReaderBuilder>& reader_builder,
const std::shared_ptr<WriterBuilder>& writer_builder,
- const std::string& compression, const std::shared_ptr<PathFactory>& path_factory,
- int64_t target_file_size, const std::shared_ptr<MemoryPool>& pool,
- const CoreOptions& options, const std::shared_ptr<arrow::Schema>& partition_type);
+ const std::string& file_format_identifier, const std::string& compression,
+ const std::shared_ptr<PathFactory>& path_factory, int64_t target_file_size,
+ const std::shared_ptr<MemoryPool>& pool, const CoreOptions& options,
+ const std::shared_ptr<arrow::Schema>& partition_type);
- private:
int64_t target_file_size_;
CoreOptions options_;
std::shared_ptr<arrow::Schema> partition_type_;
diff --git a/src/paimon/core/manifest/manifest_file_test.cpp b/src/paimon/core/manifest/manifest_file_test.cpp
index df78fc00..5cd264aa 100644
--- a/src/paimon/core/manifest/manifest_file_test.cpp
+++ b/src/paimon/core/manifest/manifest_file_test.cpp
@@ -32,6 +32,7 @@
#include "paimon/core/manifest/file_kind.h"
#include "paimon/core/manifest/file_source.h"
#include "paimon/core/manifest/manifest_entry.h"
+#include "paimon/core/manifest/manifest_file_meta.h"
#include "paimon/core/stats/simple_stats.h"
#include "paimon/core/utils/file_store_path_factory.h"
#include "paimon/data/decimal.h"
@@ -116,8 +117,7 @@ class ManifestFileTest : public testing::Test {
/*global_index_external_path=*/std::nullopt,
/*index_file_in_data_file_dir=*/false, pool));
EXPECT_OK_AND_ASSIGN(CoreOptions options,
- CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"},
- {Options::MANIFEST_FORMAT, file_format_str}}));
+ CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
EXPECT_OK_AND_ASSIGN(
std::unique_ptr<ManifestFile> manifest_file,
ManifestFile::Create(file_system, file_format, "zstd", path_factory,
@@ -254,9 +254,8 @@ TEST_F(ManifestFileTest, TestManifestCacheIsDisabledWithoutInjectedCache) {
/*legacy_partition_name_enabled=*/true, /*external_paths=*/{},
/*global_index_external_path=*/std::nullopt,
/*index_file_in_data_file_dir=*/false, pool));
- ASSERT_OK_AND_ASSIGN(
- CoreOptions options,
- CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}}));
+ ASSERT_OK_AND_ASSIGN(CoreOptions options,
+ CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
ASSERT_OK_AND_ASSIGN(
std::unique_ptr<ManifestFile> manifest_file,
ManifestFile::Create(counting_file_system, file_format, "zstd", path_factory,
@@ -296,9 +295,8 @@ TEST_F(ManifestFileTest, TestManifestCacheReusesCachedBytes) {
/*legacy_partition_name_enabled=*/true, /*external_paths=*/{},
/*global_index_external_path=*/std::nullopt,
/*index_file_in_data_file_dir=*/false, pool));
- ASSERT_OK_AND_ASSIGN(
- CoreOptions options,
- CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}}));
+ ASSERT_OK_AND_ASSIGN(CoreOptions options,
+ CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
options.WithCache(manifest_cache);
ASSERT_OK_AND_ASSIGN(
std::unique_ptr<ManifestFile> manifest_file,
@@ -337,9 +335,8 @@ TEST_F(ManifestFileTest, TestReadBucketEntriesMaterializesOnlySelectedBucket) {
/*legacy_partition_name_enabled=*/true, /*external_paths=*/{},
/*global_index_external_path=*/std::nullopt,
/*index_file_in_data_file_dir=*/false, pool));
- ASSERT_OK_AND_ASSIGN(
- CoreOptions options,
- CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}}));
+ ASSERT_OK_AND_ASSIGN(CoreOptions options,
+ CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
options.WithCache(manifest_cache);
ASSERT_OK_AND_ASSIGN(
std::unique_ptr<ManifestFile> manifest_file,
@@ -381,7 +378,7 @@ TEST_F(ManifestFileTest, TestReadBucketEntriesSkipsDeserializingOtherBuckets) {
std::shared_ptr<FileSystem> file_system = test_dir->GetFileSystem();
ASSERT_OK(file_system->Mkdirs(FileStorePathFactory::ManifestPath(test_dir->Str())));
ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileFormat> file_format,
- FileFormatFactory::Get("orc", {}));
+ FileFormatFactory::Get("avro", {}));
auto unused_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", arrow::utf8())}));
ASSERT_OK_AND_ASSIGN(
std::shared_ptr<FileStorePathFactory> path_factory,
@@ -391,9 +388,8 @@ TEST_F(ManifestFileTest, TestReadBucketEntriesSkipsDeserializingOtherBuckets) {
/*legacy_partition_name_enabled=*/true, /*external_paths=*/{},
/*global_index_external_path=*/std::nullopt,
/*index_file_in_data_file_dir=*/false, pool));
- ASSERT_OK_AND_ASSIGN(
- CoreOptions options,
- CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}}));
+ ASSERT_OK_AND_ASSIGN(CoreOptions options,
+ CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
ASSERT_OK_AND_ASSIGN(
std::unique_ptr<ManifestFile> manifest_file,
ManifestFile::Create(file_system, file_format, "zstd", path_factory,
@@ -418,6 +414,38 @@ TEST_F(ManifestFileTest, TestReadBucketEntriesSkipsDeserializingOtherBuckets) {
ASSERT_EQ(std::vector<ManifestEntry>({valid_target_bucket}), bucket_entries);
}
+TEST_F(ManifestFileTest, TestLegacyManifestFormatIsReadOnly) {
+ auto pool = GetDefaultPool();
+ std::vector<ManifestEntry> entries =
+ ReadManifestEntry("orc", paimon::test::GetDataDir() + "/orc/append_09.db/append_09",
+ "manifest-3a44a0da-1008-463c-914e-28d271375e24-0", pool);
+ ASSERT_FALSE(entries.empty());
+
+ auto dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileFormat> file_format,
+ FileFormatFactory::Get("orc", {}));
+ auto unused_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", arrow::utf8())}));
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<FileStorePathFactory> path_factory,
+ FileStorePathFactory::Create(dir->Str(), unused_schema, /*partition_keys=*/{},
+ /*default_part_value=*/"", file_format->Identifier(),
+ /*data_file_prefix=*/"data-",
+ /*legacy_partition_name_enabled=*/true, /*external_paths=*/{},
+ /*global_index_external_path=*/std::nullopt,
+ /*index_file_in_data_file_dir=*/false, pool));
+ ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({}));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<ManifestFile> manifest_file,
+ ManifestFile::Create(dir->GetFileSystem(), file_format, "zstd", path_factory,
+ /*target_file_size=*/1024, pool, options, unused_schema));
+
+ ASSERT_NOK_WITH_MSG(manifest_file->Write({entries.front()}),
+ "manifest.format 'orc' is read-only");
+ ASSERT_NOK_WITH_MSG(manifest_file->WriteWithoutRolling({entries.front()}),
+ "manifest.format 'orc' is read-only");
+}
+
TEST_F(ManifestFileTest, TestWithNullCount) {
auto pool = GetDefaultPool();
auto manifest_entries =
diff --git a/src/paimon/core/manifest/manifest_list.cpp b/src/paimon/core/manifest/manifest_list.cpp
index 887b8845..83025a91 100644
--- a/src/paimon/core/manifest/manifest_list.cpp
+++ b/src/paimon/core/manifest/manifest_list.cpp
@@ -41,11 +41,13 @@ class MemoryPool;
ManifestList::ManifestList(const std::shared_ptr<FileSystem>& file_system,
const std::shared_ptr<ReaderBuilder>& reader_builder,
const std::shared_ptr<WriterBuilder>& writer_builder,
+ const std::string& file_format_identifier,
const std::string& compression,
const std::shared_ptr<PathFactory>& path_factory,
const std::shared_ptr<Cache>& cache,
const std::shared_ptr<MemoryPool>& pool)
: ObjectsFile<ManifestFileMeta>(file_system, reader_builder, writer_builder,
+ file_format_identifier,
std::make_unique<ManifestFileMetaSerializer>(pool), compression,
std::move(path_factory), cache, pool) {}
@@ -71,8 +73,9 @@ Result<std::unique_ptr<ManifestList>> ManifestList::Create(
// create manifest list
std::shared_ptr<PathFactory> manifest_list_path_factory =
path_factory->CreateManifestListFactory();
- return std::unique_ptr<ManifestList>(new ManifestList(
- fs, reader_builder, writer_builder, compression, manifest_list_path_factory, cache, pool));
+ return std::unique_ptr<ManifestList>(new ManifestList(fs, reader_builder, writer_builder,
+ file_format->Identifier(), compression,
+ manifest_list_path_factory, cache, pool));
}
Result<std::pair<std::string, int64_t>> ManifestList::Write(
diff --git a/src/paimon/core/manifest/manifest_list.h b/src/paimon/core/manifest/manifest_list.h
index ff67e808..6bf64e8e 100644
--- a/src/paimon/core/manifest/manifest_list.h
+++ b/src/paimon/core/manifest/manifest_list.h
@@ -123,7 +123,8 @@ class ManifestList : public ObjectsFile<ManifestFileMeta> {
ManifestList(const std::shared_ptr<FileSystem>& file_system,
const std::shared_ptr<ReaderBuilder>& reader_builder,
const std::shared_ptr<WriterBuilder>& writer_builder,
- const std::string& compression, const std::shared_ptr<PathFactory>& path_factory,
+ const std::string& file_format_identifier, const std::string& compression,
+ const std::shared_ptr<PathFactory>& path_factory,
const std::shared_ptr<Cache>& cache, const std::shared_ptr<MemoryPool>& pool);
};
diff --git a/src/paimon/core/manifest/manifest_list_test.cpp b/src/paimon/core/manifest/manifest_list_test.cpp
index 23949bfd..97b420cf 100644
--- a/src/paimon/core/manifest/manifest_list_test.cpp
+++ b/src/paimon/core/manifest/manifest_list_test.cpp
@@ -142,11 +142,20 @@ TEST_F(ManifestListTest, TestEmptyManifestList) {
ASSERT_EQ(manifest_file_metas.size(), 0);
}
-TEST_F(ManifestListTest, TestReadChangelogManifests) {
+TEST_F(ManifestListTest, TestLegacyManifestFormatIsReadOnly) {
auto pool = GetDefaultPool();
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
auto manifest_list = CreateManifestList("orc", dir->Str(), pool);
+
+ ASSERT_NOK_WITH_MSG(manifest_list->Write({}), "manifest.format 'orc' is read-only");
+}
+
+TEST_F(ManifestListTest, TestReadChangelogManifests) {
+ auto pool = GetDefaultPool();
+ auto dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ auto manifest_list = CreateManifestList("avro", dir->Str(), pool);
ManifestFileMeta expected_meta(
"changelog-manifest", /*file_size=*/100, /*num_added_files=*/1,
/*num_deleted_files=*/0, SimpleStats::EmptyStats(), /*schema_id=*/0,
@@ -224,11 +233,11 @@ TEST_F(ManifestListTest, TestReadWithMinAndMaxRowId) {
// test write meta
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
- auto manifest_list = CreateManifestList("orc", dir->Str(), pool);
+ auto manifest_list = CreateManifestList("avro", dir->Str(), pool);
std::pair<std::string, int64_t> file_meta;
ASSERT_OK_AND_ASSIGN(file_meta, manifest_list->Write({expected_meta}));
// test read meta from C++
- auto manifest_file_metas2 = ReadManifestFileMeta("orc", dir->Str(), file_meta.first, pool);
+ auto manifest_file_metas2 = ReadManifestFileMeta("avro", dir->Str(), file_meta.first, pool);
ASSERT_EQ(manifest_file_metas2.size(), 1);
ASSERT_EQ(manifest_file_metas2, expected_manifest_file_metas);
}
diff --git a/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg.h b/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg.h
index a6d7ed5c..a6dc04b7 100644
--- a/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg.h
+++ b/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg.h
@@ -20,10 +20,12 @@
#include <memory>
#include <string>
+#include <string_view>
#include <unordered_set>
#include <utility>
#include "paimon/common/data/data_define.h"
+#include "paimon/common/utils/string_utils.h"
#include "paimon/core/core_options.h"
#include "paimon/core/mergetree/compact/aggregate/field_aggregator.h"
@@ -60,15 +62,18 @@ class FieldListaggAgg : public FieldAggregator {
const VariantType& input_field) override {
bool accumulator_null = DataDefine::IsVariantNull(accumulator);
bool input_null = DataDefine::IsVariantNull(input_field);
- if (accumulator_null || input_null) {
- return accumulator_null ? input_field : accumulator;
+ if (input_null) {
+ return accumulator;
}
- std::string_view acc_str = DataDefine::GetStringView(accumulator);
std::string_view in_str = DataDefine::GetStringView(input_field);
- if (in_str.empty()) {
+ if (StringUtils::IsBlank(in_str)) {
return accumulator;
}
- if (acc_str.empty()) {
+ if (accumulator_null) {
+ return input_field;
+ }
+ std::string_view acc_str = DataDefine::GetStringView(accumulator);
+ if (StringUtils::IsBlank(acc_str)) {
return input_field;
}
@@ -95,9 +100,7 @@ class FieldListaggAgg : public FieldAggregator {
size_t pos = remaining.find(delimiter_);
std::string_view token =
(pos == std::string_view::npos) ? remaining : remaining.substr(0, pos);
- if (!token.empty()) {
- seen.insert(token);
- }
+ seen.insert(token);
if (pos == std::string_view::npos) {
break;
}
@@ -113,7 +116,7 @@ class FieldListaggAgg : public FieldAggregator {
size_t pos = remaining.find(delimiter_);
std::string_view token =
(pos == std::string_view::npos) ? remaining : remaining.substr(0, pos);
- if (!token.empty() && seen.insert(token).second) {
+ if (!StringUtils::IsBlank(token) && seen.insert(token).second) {
result.append(delimiter_);
result.append(token);
}
diff --git a/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp b/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp
index 3578cdaa..1efac34e 100644
--- a/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp
+++ b/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp
@@ -21,6 +21,7 @@
#include <map>
#include <string>
#include <string_view>
+#include <vector>
#include "arrow/type_fwd.h"
#include "gtest/gtest.h"
@@ -88,13 +89,41 @@ TEST_F(FieldListaggAggTest, TestEmptyString) {
auto ret = agg->Agg(std::string_view(""), std::string_view("world")).value();
ASSERT_EQ(DataDefine::GetVariantValue<std::string_view>(ret), "world");
}
- // both empty -> return input (which is empty)
+ // blank input -> return accumulator (which is empty)
{
auto ret = agg->Agg(std::string_view(""), std::string_view("")).value();
ASSERT_EQ(DataDefine::GetVariantValue<std::string_view>(ret), "");
}
}
+TEST_F(FieldListaggAggTest, TestBlankStrings) {
+ ASSERT_OK_AND_ASSIGN(auto agg, MakeAgg());
+
+ const std::vector<std::string> blank_strings = {"",
+ " ",
+ " ",
+ "\t",
+ "\n",
+ "\r",
+ "\r\n",
+ " \t\n\r ",
+ u8"\u3000",
+ u8"\u2000",
+ u8" \t\u3000\u2000\n"};
+ for (const std::string& blank : blank_strings) {
+ auto ret = agg->Agg(std::string_view("user1"), std::string_view(blank)).value();
+ ASSERT_EQ(DataDefine::GetVariantValue<std::string_view>(ret), "user1");
+ }
+
+ // A blank accumulator must not add a leading delimiter.
+ auto ret = agg->Agg(std::string_view(u8"\u3000\t"), std::string_view("user1")).value();
+ ASSERT_EQ(DataDefine::GetVariantValue<std::string_view>(ret), "user1");
+
+ // A blank input must not turn a null accumulator into a non-null value.
+ ret = agg->Agg(NullType(), std::string_view(u8" \t\u3000")).value();
+ ASSERT_TRUE(DataDefine::IsVariantNull(ret));
+}
+
TEST_F(FieldListaggAggTest, TestMultipleAccumulation) {
ASSERT_OK_AND_ASSIGN(auto agg, MakeAgg());
@@ -113,6 +142,15 @@ TEST_F(FieldListaggAggTest, TestDistinct) {
ASSERT_EQ(DataDefine::GetVariantValue<std::string_view>(ret), "a;b;c");
}
+TEST_F(FieldListaggAggTest, TestDistinctIgnoresBlankTokens) {
+ ASSERT_OK_AND_ASSIGN(auto agg, MakeAgg(",", true));
+
+ auto ret =
+ agg->Agg(std::string_view("user1"), std::string_view(u8" ,user2,\t,\u3000,user1,\u2000"))
+ .value();
+ ASSERT_EQ(DataDefine::GetVariantValue<std::string_view>(ret), "user1,user2");
+}
+
TEST_F(FieldListaggAggTest, TestDistinctNoDuplicates) {
ASSERT_OK_AND_ASSIGN(auto agg, MakeAgg(" ", true));
diff --git a/src/paimon/core/operation/append_only_file_store_scan_test.cpp b/src/paimon/core/operation/append_only_file_store_scan_test.cpp
index d72dcaf0..fd16d47f 100644
--- a/src/paimon/core/operation/append_only_file_store_scan_test.cpp
+++ b/src/paimon/core/operation/append_only_file_store_scan_test.cpp
@@ -101,7 +101,6 @@ TEST(AppendOnlyFileStoreScanTest, TestReadPartitionEntries) {
std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/";
ScanContextBuilder context_builder(table_path);
context_builder.AddOption(Options::FILE_FORMAT, "orc")
- .AddOption(Options::MANIFEST_FORMAT, "orc")
.AddOption(Options::SCAN_SNAPSHOT_ID, "5");
ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish());
@@ -148,7 +147,6 @@ TEST(AppendOnlyFileStoreScanTest, TestScanDurationMetric) {
std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/";
ScanContextBuilder context_builder(table_path);
context_builder.AddOption(Options::FILE_FORMAT, "orc")
- .AddOption(Options::MANIFEST_FORMAT, "orc")
.AddOption(Options::SCAN_SNAPSHOT_ID, "5");
ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish());
@@ -204,7 +202,6 @@ std::shared_ptr<FileStoreScan> BuildScan(const std::string& table_path,
bool manifest_entry_lazy_decode_enabled = true) {
ScanContextBuilder context_builder(table_path);
context_builder.AddOption(Options::FILE_FORMAT, "orc")
- .AddOption(Options::MANIFEST_FORMAT, "orc")
.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "8")
.AddOption(Options::SCAN_MANIFEST_ENTRY_LAZY_DECODE_ENABLED,
manifest_entry_lazy_decode_enabled ? "true" : "false")
diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp
index 85d5c565..f91e0f60 100644
--- a/src/paimon/core/operation/append_only_file_store_write.cpp
+++ b/src/paimon/core/operation/append_only_file_store_write.cpp
@@ -78,7 +78,7 @@ namespace {
// the table does not set them - ParquetWriterBuilder for the first, ParquetFileBatchReader for the
// second.
constexpr char kParquetFormat[] = "parquet";
-constexpr char kParquetEnableDictionary[] = "parquet.enable-dictionary";
+constexpr char kParquetEnableDictionary[] = "parquet.enable.dictionary";
constexpr char kParquetReadEnableDictionaryPassthrough[] =
"parquet.read.enable-dictionary-passthrough";
constexpr bool kDefaultParquetEnableDictionary = true;
@@ -328,7 +328,7 @@ Result<std::optional<std::string>> AppendOnlyFileStoreWrite::GetDictionaryPassth
options_.ToMap(), kParquetEnableDictionary,
kDefaultParquetEnableDictionary));
if (!enable_dictionary) {
- return std::optional<std::string>("parquet.enable-dictionary is false");
+ return std::optional<std::string>("parquet.enable.dictionary is false");
}
if (plan_factory != nullptr) {
return std::optional<std::string>("the table is written through a shredding writer");
diff --git a/src/paimon/core/operation/append_only_file_store_write.h b/src/paimon/core/operation/append_only_file_store_write.h
index 132118a7..d030a07c 100644
--- a/src/paimon/core/operation/append_only_file_store_write.h
+++ b/src/paimon/core/operation/append_only_file_store_write.h
@@ -146,7 +146,7 @@ class AppendOnlyFileStoreWrite : public AbstractFileStoreWrite {
/// The veto stands unless all three of:
///
/// - a Parquet output file, since no other writer takes a dictionary-encoded batch;
- /// - `parquet.enable-dictionary`, or the writer densifies what the reader just handed it and
+ /// - `parquet.enable.dictionary`, or the writer densifies what the reader just handed it and
/// the encoding is carried across the rewrite for nothing;
/// - a rewrite that stays a passthrough, since a shredding writer reshapes each batch against
/// a fixed physical schema and cannot take a dictionary-encoded one.
diff --git a/src/paimon/core/operation/commit/commit_scanner_test.cpp b/src/paimon/core/operation/commit/commit_scanner_test.cpp
index 3d5c6270..1e02849d 100644
--- a/src/paimon/core/operation/commit/commit_scanner_test.cpp
+++ b/src/paimon/core/operation/commit/commit_scanner_test.cpp
@@ -119,7 +119,7 @@ class CommitScannerTest : public testing::Test {
Result<std::shared_ptr<IndexManifestFile>> CreateIndexManifestFile() const {
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileFormat> file_format,
- FileFormatFactory::Get("orc", {}));
+ FileFormatFactory::Get("avro", {}));
PAIMON_ASSIGN_OR_RAISE(
std::shared_ptr<FileStorePathFactory> path_factory,
FileStorePathFactory::Create(
diff --git a/src/paimon/core/operation/expire_snapshots_test.cpp b/src/paimon/core/operation/expire_snapshots_test.cpp
index ffdd30fe..f3472d5b 100644
--- a/src/paimon/core/operation/expire_snapshots_test.cpp
+++ b/src/paimon/core/operation/expire_snapshots_test.cpp
@@ -75,14 +75,15 @@ class ExpireSnapshotsTest : public testing::Test {
test_data_path_ = "tmp";
path_factory_ = CreateFactory(test_data_path_);
+ std::shared_ptr<FileFormat> manifest_format = options.GetManifestFormat();
ASSERT_OK_AND_ASSIGN(
manifest_list_,
- ManifestList::Create(fs_, options.GetManifestFormat(), options.GetManifestCompression(),
+ ManifestList::Create(fs_, manifest_format, options.GetManifestCompression(),
path_factory_, options.GetCache(), mem_pool_));
ASSERT_OK_AND_ASSIGN(
manifest_file_,
- ManifestFile::Create(fs_, options.GetManifestFormat(), options.GetManifestCompression(),
+ ManifestFile::Create(fs_, manifest_format, options.GetManifestCompression(),
path_factory_, options.GetManifestTargetFileSize(), mem_pool_,
options, partition_schema_));
}
@@ -113,7 +114,6 @@ class ExpireSnapshotsTest : public testing::Test {
std::shared_ptr<FileStorePathFactory> CreateFactory(const std::string& root) const {
std::map<std::string, std::string> raw_options;
raw_options[Options::FILE_FORMAT] = "orc";
- raw_options[Options::MANIFEST_FORMAT] = "orc";
raw_options[Options::FILE_SYSTEM] = "local";
EXPECT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(raw_options));
EXPECT_OK_AND_ASSIGN(std::vector<std::string> external_paths,
diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp
index ad0942ad..6d900560 100644
--- a/src/paimon/core/operation/file_store_commit.cpp
+++ b/src/paimon/core/operation/file_store_commit.cpp
@@ -127,7 +127,6 @@ Result<std::unique_ptr<FileStoreCommit>> FileStoreCommit::Create(
assert(options.GetFileSystem());
assert(options.GetFileFormat());
PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options));
-
PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path));
if (is_object_store && !ctx->UseRESTCatalogCommit() &&
opts.find("enable-object-store-commit-in-inte-test") == opts.end()) {
diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp
index b1b8677f..a5079d7d 100644
--- a/src/paimon/core/operation/file_store_commit_impl_test.cpp
+++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp
@@ -379,8 +379,7 @@ class FileStoreCommitImplTest : public testing::Test {
TEST_F(FileStoreCommitImplTest, TestCommit) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -408,8 +407,7 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) {
TimezoneGuard guard("Asia/Shanghai");
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.UseRESTCatalogCommit(true)
.Finish());
@@ -466,8 +464,7 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) {
TEST_F(FileStoreCommitImplTest, TestSnapshotSequenceMaxPropertyMergedOnCommit) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot")
.Finish());
@@ -514,8 +511,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryTenTimes)
FileSystemFactory::Get("gmock_fs", table_path, {}));
CommitContextBuilder context_builder(table_path, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::COMMIT_MAX_RETRIES, "10")
.AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms")
.AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms")
@@ -548,7 +544,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryTenTimes)
}
TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) {
- std::string test_data_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/";
+ std::string test_data_path = paimon::test::GetDataDir() + "/parquet/append_09.db/append_09/";
auto dir = UniqueTestDirectory::Create();
std::string table_path = dir->Str();
ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path));
@@ -556,8 +552,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) {
FileSystemFactory::Get("gmock_fs", table_path, {}));
CommitContextBuilder context_builder(table_path, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms")
.AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms")
.WithFileSystem(fs)
@@ -591,7 +586,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) {
std::vector<std::shared_ptr<CommitMessage>> msgs =
GetCommitMessages(paimon::test::GetDataDir() +
- "/orc/append_09.db/append_09/commit_messages/commit_messages-01",
+ "/parquet/append_09.db/append_09/commit_messages/commit_messages-01",
/*version=*/3);
ASSERT_GT(msgs.size(), 0);
ASSERT_OK(commit->Commit(msgs));
@@ -606,7 +601,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) {
}
TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActuallySucceed) {
- std::string test_data_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/";
+ std::string test_data_path = paimon::test::GetDataDir() + "/parquet/append_09.db/append_09/";
auto dir = UniqueTestDirectory::Create();
std::string table_path = dir->Str();
ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path));
@@ -614,8 +609,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua
FileSystemFactory::Get("gmock_fs", table_path, {}));
CommitContextBuilder context_builder(table_path, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.WithFileSystem(fs)
.Finish());
@@ -632,7 +626,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua
std::vector<std::shared_ptr<CommitMessage>> msgs =
GetCommitMessages(paimon::test::GetDataDir() +
- "/orc/append_09.db/append_09/commit_messages/commit_messages-01",
+ "/parquet/append_09.db/append_09/commit_messages/commit_messages-01",
/*version=*/3);
ASSERT_GT(msgs.size(), 0);
ASSERT_NOK(commit->Commit(msgs, /*commit_identifier=*/1));
@@ -642,8 +636,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua
CommitContextBuilder context_builder_2(table_path, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context_2,
- context_builder_2.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder_2.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.WithFileSystem(fs)
.Finish());
@@ -657,7 +650,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua
}));
std::vector<std::shared_ptr<CommitMessage>> msgs_2 =
GetCommitMessages(paimon::test::GetDataDir() +
- "/orc/append_09.db/append_09/commit_messages/commit_messages-02",
+ "/parquet/append_09.db/append_09/commit_messages/commit_messages-02",
/*version=*/3);
ASSERT_OK(commit_2->Commit(msgs_2, /*commit_identifier=*/2));
ASSERT_OK_AND_ASSIGN(exist, file_system_->Exists(new_snapshot_7));
@@ -667,8 +660,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua
TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "5kb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "5kb")
.AddOption(Options::MANIFEST_MERGE_MIN_COUNT, "2")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -731,8 +723,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) {
TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -803,8 +794,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) {
TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatest) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
@@ -861,8 +851,7 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatest) {
TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNoLatestSnapshotReturnsError) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
@@ -873,8 +862,7 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNoLatestSnapshotReturnsErr
TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestTargetNotExistReturnsError) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
@@ -908,8 +896,7 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestDeletionVectorOnlyChange)
CommitContextBuilder context_builder(dv_table_path, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
@@ -978,8 +965,7 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestConcurrentConflictReturnsF
FileSystemFactory::Get("gmock_fs", table_path_, {}));
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.WithFileSystem(fs)
.Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
@@ -1021,8 +1007,7 @@ TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestConcurrentConflictReturnsF
TEST_F(FileStoreCommitImplTest, TestCommitAndOverwriteWithNoPartitionKey) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
@@ -1067,8 +1052,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitAndOverwriteWithNoPartitionKey) {
TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
@@ -1125,8 +1109,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) {
TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -1243,8 +1226,7 @@ TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) {
TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.IgnoreEmptyCommit(true)
.Finish());
@@ -1262,8 +1244,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) {
TEST_F(FileStoreCommitImplTest, TestTryOverwrite) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.IgnoreEmptyCommit(true)
.Finish());
@@ -1293,8 +1274,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwrite) {
TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.IgnoreEmptyCommit(true)
.Finish());
@@ -1328,8 +1308,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) {
TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.IgnoreEmptyCommit(true)
.Finish());
@@ -1356,8 +1335,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) {
TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.IgnoreEmptyCommit(true)
.Finish());
@@ -1408,8 +1386,7 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) {
TEST_F(FileStoreCommitImplTest, TestDropPartitionAndExpireSnapshot) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1")
.AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1")
@@ -1456,8 +1433,7 @@ TEST_F(FileStoreCommitImplTest, TestDropPartitionAndExpireSnapshot) {
TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.AddOption(Options::SNAPSHOT_NUM_RETAINED_MIN, "1")
.AddOption(Options::SNAPSHOT_NUM_RETAINED_MAX, "1")
@@ -1504,8 +1480,7 @@ TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) {
TEST_F(FileStoreCommitImplTest, TestTruncateTable) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.IgnoreEmptyCommit(true)
.Finish());
@@ -1538,9 +1513,7 @@ TEST_F(FileStoreCommitImplTest, TestTruncateTable) {
TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::FILE_SYSTEM, "local")
- .Finish());
+ context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
auto commit_impl = std::dynamic_pointer_cast<FileStoreCommitImpl>(
std::shared_ptr<FileStoreCommit>(std::move(commit)));
@@ -1587,9 +1560,7 @@ TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) {
TEST_F(FileStoreCommitImplTest, AbortIgnoresMissingFilesAndFailsForNonImplMessage) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::FILE_SYSTEM, "local")
- .Finish());
+ context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
auto commit_impl = std::dynamic_pointer_cast<FileStoreCommitImpl>(
std::shared_ptr<FileStoreCommit>(std::move(commit)));
@@ -1613,9 +1584,7 @@ TEST_F(FileStoreCommitImplTest, AbortIgnoresDeleteFailures) {
// propagate the failure.
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::FILE_SYSTEM, "local")
- .Finish());
+ context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
auto commit_impl = dynamic_cast<FileStoreCommitImpl*>(commit.get());
ASSERT_TRUE(commit_impl);
@@ -1654,8 +1623,7 @@ TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) {
// materialized.
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.IgnoreEmptyCommit(true)
.Finish());
@@ -1680,8 +1648,7 @@ TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) {
TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.IgnoreEmptyCommit(true)
.Finish());
@@ -1701,8 +1668,7 @@ TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) {
TEST_F(FileStoreCommitImplTest, TestCollectChanges) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.AddOption(Options::BUCKET, "10")
.Finish());
@@ -1745,8 +1711,7 @@ TEST_F(FileStoreCommitImplTest, TestCollectChanges) {
TEST_F(FileStoreCommitImplTest, TestFilterCommitted) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -1777,8 +1742,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommitted) {
TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -1818,8 +1782,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) {
TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -1850,8 +1813,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers)
TEST_F(FileStoreCommitImplTest, FilterAndCommit) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -1898,8 +1860,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommit) {
TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -1921,8 +1882,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) {
TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -1956,8 +1916,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) {
TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -1990,8 +1949,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) {
TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -2030,8 +1988,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) {
TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -2061,8 +2018,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) {
TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.AddOption(Options::ROW_TRACKING_ENABLED, "true")
.AddOption(Options::DATA_EVOLUTION_ENABLED, "true")
@@ -2094,8 +2050,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) {
TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.IgnoreEmptyCommit(true)
.Finish());
@@ -2127,8 +2082,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) {
TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -2165,8 +2119,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) {
TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.IgnoreEmptyCommit(true)
.Finish());
@@ -2205,8 +2158,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) {
TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -2239,8 +2191,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) {
TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -2295,8 +2246,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) {
TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -2334,8 +2284,7 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) {
TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -2369,8 +2318,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) {
TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -2400,8 +2348,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) {
TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.AddOption(Options::COMMIT_DISCARD_DUPLICATE_FILES, "true")
.Finish());
@@ -2433,8 +2380,7 @@ TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) {
TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
@@ -2466,8 +2412,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) {
io_hook->Reset(i, IOHook::Mode::RETURN_ERROR);
CommitContextBuilder context_builder2(tmp_table_path, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context2,
- context_builder2.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder2.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
auto commit2 = FileStoreCommit::Create(std::move(commit_context2));
@@ -2495,9 +2440,7 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) {
// REST commit with local path should work without the object store flag
CommitContextBuilder builder(table_path_, "commit_user_1");
- ASSERT_OK_AND_ASSIGN(
- auto ctx,
- builder.AddOption(Options::MANIFEST_FORMAT, "orc").UseRESTCatalogCommit(true).Finish());
+ ASSERT_OK_AND_ASSIGN(auto ctx, builder.UseRESTCatalogCommit(true).Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(ctx)));
auto msgs =
@@ -2563,8 +2506,7 @@ TEST_F(FileStoreCommitImplTest, ValidateCommitOptionsAllowsManifestDeleteFileDro
TEST_F(FileStoreCommitImplTest, TestGetAllFilesKeepsValueStats) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::MANIFEST_DELETE_FILE_DROP_STATS, "true")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -2593,8 +2535,7 @@ TEST_F(FileStoreCommitImplTest, TestGetAllFilesKeepsValueStats) {
TEST_F(FileStoreCommitImplTest, TestOverwriteDropsDeleteFileStats) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::MANIFEST_DELETE_FILE_DROP_STATS, "true")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
@@ -2669,9 +2610,7 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteDropsDeleteFileStats) {
TEST_F(FileStoreCommitImplTest, DropPartitionWithEmptyPartitionsFails) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::FILE_SYSTEM, "local")
- .Finish());
+ context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
ASSERT_NOK_WITH_MSG(commit->DropPartition({}, /*commit_identifier=*/1),
"partitions list cannot be empty");
@@ -2680,8 +2619,7 @@ TEST_F(FileStoreCommitImplTest, DropPartitionWithEmptyPartitionsFails) {
TEST_F(FileStoreCommitImplTest, FilterAndCommitMultipleIdentifiersAndEmptyInput) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
@@ -2720,9 +2658,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitMultipleIdentifiersAndEmptyInput)
TEST_F(FileStoreCommitImplTest, CheckFilesExistenceFailsForNonImplCommitMessage) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::FILE_SYSTEM, "local")
- .Finish());
+ context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
auto commit_impl = std::dynamic_pointer_cast<FileStoreCommitImpl>(
std::shared_ptr<FileStoreCommit>(std::move(commit)));
@@ -2736,9 +2672,7 @@ TEST_F(FileStoreCommitImplTest, CheckFilesExistenceFailsForNonImplCommitMessage)
TEST_F(FileStoreCommitImplTest, CheckFilesExistenceCollectsIndexFilePaths) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::FILE_SYSTEM, "local")
- .Finish());
+ context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
auto commit_impl = std::dynamic_pointer_cast<FileStoreCommitImpl>(
std::shared_ptr<FileStoreCommit>(std::move(commit)));
@@ -2765,8 +2699,7 @@ TEST_F(FileStoreCommitImplTest, CheckFilesExistenceCollectsIndexFilePaths) {
TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.AddOption(Options::DYNAMIC_PARTITION_OVERWRITE, "false")
.Finish());
@@ -2794,8 +2727,7 @@ TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership)
TEST_F(FileStoreCommitImplTest, OverwriteWithChangelogFilesLogsWarning) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
@@ -2840,8 +2772,7 @@ TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles)
CommitContextBuilder builder(pk_table_path, "test_user");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::FILE_SYSTEM, "local")
+ builder.AddOption(Options::FILE_SYSTEM, "local")
.AddOption(Options::OVERWRITE_UPGRADE, "true")
.Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
@@ -2891,8 +2822,7 @@ TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles)
TEST_F(FileStoreCommitImplTest, CommitWithAppendCommitCheckConflict) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.AppendCommitCheckConflict(true)
.Finish());
@@ -2915,8 +2845,7 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) {
{
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
@@ -2931,8 +2860,7 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) {
// the max sequence number is recomputed by scanning the base manifests.
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot")
.Finish());
@@ -2961,8 +2889,7 @@ TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) {
TEST_F(FileStoreCommitImplTest, FilterAndOverwriteWithSpecifiedPartition) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
@@ -2988,8 +2915,7 @@ TEST_F(FileStoreCommitImplTest, FilterAndOverwriteWithSpecifiedPartition) {
TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenOverwriteUpgradeDisabled) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::FILE_SYSTEM, "local")
+ context_builder.AddOption(Options::FILE_SYSTEM, "local")
.AddOption(Options::OVERWRITE_UPGRADE, "false")
.Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
@@ -3022,8 +2948,7 @@ TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) {
CommitContextBuilder builder(pk_table_path, "test_user");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::FILE_SYSTEM, "local")
+ builder.AddOption(Options::FILE_SYSTEM, "local")
.AddOption(Options::OVERWRITE_UPGRADE, "true")
.Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
@@ -3043,8 +2968,7 @@ TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) {
TEST_F(FileStoreCommitImplTest, CheckSameBucketFromSnapshotReturnsOkForEmptyDelta) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
@@ -3067,9 +2991,7 @@ TEST_F(FileStoreCommitImplTest, CheckSameBucketFromSnapshotReturnsOkForEmptyDelt
TEST_F(FileStoreCommitImplTest, MaxSequenceNumberReturnsNulloptForEmptyManifests) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::FILE_SYSTEM, "local")
- .Finish());
+ context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
auto commit_impl = std::dynamic_pointer_cast<FileStoreCommitImpl>(
std::shared_ptr<FileStoreCommit>(std::move(commit)));
@@ -3082,9 +3004,7 @@ TEST_F(FileStoreCommitImplTest, MaxSequenceNumberReturnsNulloptForEmptyManifests
TEST_F(FileStoreCommitImplTest, RowIdCheckConflictSetsCheckSnapshotAndReturnsSelf) {
CommitContextBuilder context_builder(table_path_, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::FILE_SYSTEM, "local")
- .Finish());
+ context_builder.AddOption(Options::FILE_SYSTEM, "local").Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
auto commit_impl = std::dynamic_pointer_cast<FileStoreCommitImpl>(
std::shared_ptr<FileStoreCommit>(std::move(commit)));
diff --git a/src/paimon/core/operation/file_store_commit_test.cpp b/src/paimon/core/operation/file_store_commit_test.cpp
index b43e3215..07d55eb8 100644
--- a/src/paimon/core/operation/file_store_commit_test.cpp
+++ b/src/paimon/core/operation/file_store_commit_test.cpp
@@ -77,8 +77,7 @@ TEST(FileStoreCommitTest, TestCreate) {
CommitContextBuilder context_builder(table_path, "commit_user");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
@@ -115,8 +114,7 @@ TEST(FileStoreCommitTest, TestAppendDvIndexShouldUseOverwriteCommitKind) {
CommitContextBuilder context_builder(table_path, "commit_user");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
- context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
- .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+ context_builder.AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
.AddOption(Options::FILE_SYSTEM, "local")
.Finish());
ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
diff --git a/src/paimon/core/operation/key_value_file_store_scan_test.cpp b/src/paimon/core/operation/key_value_file_store_scan_test.cpp
index f9fd2ea2..f6c13cb3 100644
--- a/src/paimon/core/operation/key_value_file_store_scan_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_scan_test.cpp
@@ -70,12 +70,13 @@ class KeyValueFileStoreScanTest : public testing::Test {
Result<std::unique_ptr<KeyValueFileStoreScan>> CreateFileStoreScan(
const std::string& table_path, const std::shared_ptr<ScanFilter>& scan_filter,
int32_t table_schema_id, int32_t snapshot_id) const {
- std::map<std::string, std::string> options_map = {{Options::MANIFEST_FORMAT, "orc"}};
- PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options_map));
- auto fs = core_options.GetFileSystem();
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions bootstrap_options, CoreOptions::FromMap({}));
+ auto fs = bootstrap_options.GetFileSystem();
auto schema_manager = std::make_shared<SchemaManager>(fs, table_path);
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<TableSchema> table_schema,
schema_manager->ReadSchema(table_schema_id));
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options,
+ CoreOptions::FromMap(table_schema->Options()));
auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields());
PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> external_paths,
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index 88b848ee..90cd793b 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -743,7 +743,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriterRestoreKeepsValueStats) {
std::map<std::string, std::string> options = {
{Options::BUCKET, "1"},
{Options::FILE_FORMAT, "orc"},
- {Options::MANIFEST_FORMAT, "orc"},
+
{Options::MANIFEST_DELETE_FILE_DROP_STATS, "true"}};
ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(dir->Str(), options));
ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false));
diff --git a/src/paimon/core/operation/manifest_file_merger_test.cpp b/src/paimon/core/operation/manifest_file_merger_test.cpp
index d2617a65..89a78a5a 100644
--- a/src/paimon/core/operation/manifest_file_merger_test.cpp
+++ b/src/paimon/core/operation/manifest_file_merger_test.cpp
@@ -128,9 +128,8 @@ class ManifestFileMergerTest : public testing::Test {
private:
void CreateManifestFile(const std::string& path_str) {
file_system_ = std::make_shared<LocalFileSystem>();
- ASSERT_OK_AND_ASSIGN(
- std::shared_ptr<FileFormat> file_format,
- FileFormatFactory::Get("parquet", std::map<std::string, std::string>()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileFormat> file_format,
+ FileFormatFactory::Get("avro", std::map<std::string, std::string>()));
auto schema = arrow::schema(arrow::FieldVector({arrow::field("f0", partition_type_)}));
ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({}));
ASSERT_OK_AND_ASSIGN(std::vector<std::string> external_paths,
diff --git a/src/paimon/core/table/sink/commit_message_test.cpp b/src/paimon/core/table/sink/commit_message_test.cpp
index f372361c..8fd31c4c 100644
--- a/src/paimon/core/table/sink/commit_message_test.cpp
+++ b/src/paimon/core/table/sink/commit_message_test.cpp
@@ -1222,7 +1222,6 @@ TEST(CommitMessageTest, TestSerialize) {
ASSERT_TRUE(dir);
std::map<std::string, std::string> options = {{Options::FILE_FORMAT, "mock_format"},
- {Options::MANIFEST_FORMAT, "mock_format"},
{Options::TARGET_FILE_SIZE, "1024"}};
ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(dir->Str(), options));
ASSERT_OK(catalog->CreateDatabase("foo", options, /*ignore_if_exists=*/false));
@@ -1235,7 +1234,6 @@ TEST(CommitMessageTest, TestSerialize) {
"commit_user1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context,
context_builder.AddOption(Options::FILE_FORMAT, "mock_format")
- .AddOption(Options::MANIFEST_FORMAT, "mock_format")
.AddOption(Options::TARGET_FILE_SIZE, "1024")
.Finish());
diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp
index 6523588e..68ec9ac0 100644
--- a/src/paimon/core/table/system/global_system_tables.cpp
+++ b/src/paimon/core/table/system/global_system_tables.cpp
@@ -211,7 +211,6 @@ Result<AggregatedFileStats> AggregateFileStats(const std::shared_ptr<FileSystem>
core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(),
external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(),
pool));
-
PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ManifestList> manifest_list,
ManifestList::Create(fs, core_options.GetManifestFormat(),
core_options.GetManifestCompression(), path_factory,
diff --git a/src/paimon/core/utils/file_store_path_factory_test.cpp b/src/paimon/core/utils/file_store_path_factory_test.cpp
index 41a9635e..c78db99d 100644
--- a/src/paimon/core/utils/file_store_path_factory_test.cpp
+++ b/src/paimon/core/utils/file_store_path_factory_test.cpp
@@ -65,7 +65,6 @@ class FileStorePathFactoryTest : public ::testing::Test {
std::map<std::string, std::string> raw_options;
raw_options[Options::FILE_FORMAT] = "mock_format";
- raw_options[Options::MANIFEST_FORMAT] = "mock_format";
raw_options[Options::FILE_SYSTEM] = "local";
EXPECT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(raw_options));
EXPECT_OK_AND_ASSIGN(std::vector<std::string> external_paths,
diff --git a/src/paimon/core/utils/objects_file.h b/src/paimon/core/utils/objects_file.h
index b7945043..56ba6e31 100644
--- a/src/paimon/core/utils/objects_file.h
+++ b/src/paimon/core/utils/objects_file.h
@@ -56,6 +56,7 @@ class ObjectsFile {
ObjectsFile(const std::shared_ptr<FileSystem>& file_system,
const std::shared_ptr<ReaderBuilder>& reader_builder,
const std::shared_ptr<WriterBuilder>& writer_builder,
+ const std::string& file_format_identifier,
std::unique_ptr<ObjectSerializer<T>>&& serializer, const std::string& compression,
const std::shared_ptr<PathFactory>& path_factory,
const std::shared_ptr<Cache>& cache, const std::shared_ptr<MemoryPool>& pool);
@@ -78,6 +79,14 @@ class ObjectsFile {
Result<std::pair<std::string, int64_t>> WriteWithoutRolling(const std::vector<T>& records);
protected:
+ Status ValidateWrite() const {
+ if (file_format_identifier_ != "avro") {
+ return Status::Invalid("manifest.format '", file_format_identifier_,
+ "' is read-only; only 'avro' can be used for writing manifests");
+ }
+ return Status::OK();
+ }
+
Status ReadArrowBatches(
const std::string& file_name,
const std::function<Status(const std::shared_ptr<arrow::StructArray>&)>& consumer) const;
@@ -92,6 +101,7 @@ class ObjectsFile {
private:
std::shared_ptr<FileSystem> file_system_;
std::shared_ptr<ReaderBuilder> reader_builder_;
+ const std::string file_format_identifier_;
std::string compression_;
std::shared_ptr<Cache> cache_;
@@ -102,6 +112,7 @@ template <typename T>
ObjectsFile<T>::ObjectsFile(const std::shared_ptr<FileSystem>& file_system,
const std::shared_ptr<ReaderBuilder>& reader_builder,
const std::shared_ptr<WriterBuilder>& writer_builder,
+ const std::string& file_format_identifier,
std::unique_ptr<ObjectSerializer<T>>&& serializer,
const std::string& compression,
const std::shared_ptr<PathFactory>& path_factory,
@@ -114,6 +125,7 @@ ObjectsFile<T>::ObjectsFile(const std::shared_ptr<FileSystem>& file_system,
writer_builder_(std::move(writer_builder)),
file_system_(file_system),
reader_builder_(std::move(reader_builder)),
+ file_format_identifier_(file_format_identifier),
compression_(compression),
cache_(cache) {}
@@ -231,6 +243,7 @@ Result<MemorySegment> ObjectsFile<T>::ReadFileSegment(const std::string& file_pa
template <typename T>
Result<std::pair<std::string, int64_t>> ObjectsFile<T>::WriteWithoutRolling(
const std::vector<T>& records) {
+ PAIMON_RETURN_NOT_OK(ValidateWrite());
std::string file_path = path_factory_->NewPath();
std::vector<BinaryRow> rows;
rows.reserve(records.size());
diff --git a/src/paimon/format/avro/avro_stats_extractor_test.cpp b/src/paimon/format/avro/avro_stats_extractor_test.cpp
index c4d554d6..76f19ff1 100644
--- a/src/paimon/format/avro/avro_stats_extractor_test.cpp
+++ b/src/paimon/format/avro/avro_stats_extractor_test.cpp
@@ -74,8 +74,7 @@ class AvroStatsExtractorTest : public ::testing::Test {
}
private:
- std::map<std::string, std::string> options_ = {{Options::FILE_FORMAT, "avro"},
- {Options::MANIFEST_FORMAT, "avro"}};
+ std::map<std::string, std::string> options_ = {{Options::FILE_FORMAT, "avro"}};
};
TEST_F(AvroStatsExtractorTest, TestPrimitiveStatsExtractor) {
diff --git a/src/paimon/format/parquet/CMakeLists.txt b/src/paimon/format/parquet/CMakeLists.txt
index 96854658..60d89851 100644
--- a/src/paimon/format/parquet/CMakeLists.txt
+++ b/src/paimon/format/parquet/CMakeLists.txt
@@ -69,6 +69,7 @@ if(PAIMON_BUILD_TESTS)
paimon_shared
test_utils_static
${PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS}
+ ${PAIMON_AVRO_FILE_FORMAT_STATIC_LINK_LIBS}
${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS}
parquet
${GTEST_LINK_TOOLCHAIN})
diff --git a/src/paimon/format/parquet/parquet_format_defs.h b/src/paimon/format/parquet/parquet_format_defs.h
index d22506c4..6bc811e5 100644
--- a/src/paimon/format/parquet/parquet_format_defs.h
+++ b/src/paimon/format/parquet/parquet_format_defs.h
@@ -41,7 +41,7 @@ namespace paimon::parquet {
static inline const char PARQUET_BLOCK_SIZE[] = "parquet.block.size";
static inline const char PARQUET_PAGE_SIZE[] = "parquet.page.size";
static inline const char PARQUET_DICTIONARY_PAGE_SIZE[] = "parquet.dictionary.page.size";
-static inline const char PARQUET_ENABLE_DICTIONARY[] = "parquet.enable-dictionary";
+static inline const char PARQUET_ENABLE_DICTIONARY[] = "parquet.enable.dictionary";
static inline const char PARQUET_WRITER_VERSION[] = "parquet.writer.version";
static inline const char PARQUET_WRITE_MAX_ROW_GROUP_LENGTH[] =
"parquet.write.max-row-group-length";
diff --git a/src/paimon/format/parquet/parquet_format_writer_test.cpp b/src/paimon/format/parquet/parquet_format_writer_test.cpp
index 3d1c0087..9ba27646 100644
--- a/src/paimon/format/parquet/parquet_format_writer_test.cpp
+++ b/src/paimon/format/parquet/parquet_format_writer_test.cpp
@@ -471,12 +471,10 @@ TEST_F(ParquetFormatWriterTest, TestGetEstimateLength) {
TEST_F(ParquetFormatWriterTest, TestMemoryControl) {
auto run = [&](bool all_null_value, uint64_t max_memory_use) {
- ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<FileFormat> file_format,
- FileFormatFactory::Get(
- "parquet", {{Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "parquet"},
- {"parquet.writer.max.memory.use", std::to_string(max_memory_use)}}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileFormat> file_format,
+ FileFormatFactory::Get("parquet", {{Options::FILE_FORMAT, "parquet"},
+ {"parquet.writer.max.memory.use",
+ std::to_string(max_memory_use)}}));
std::shared_ptr<MemoryPool> pool = GetMemoryPool();
auto schema_pair = PrepareArrowSchema();
@@ -527,7 +525,6 @@ TEST_F(ParquetFormatWriterTest, TestMemoryControlForCheckRowGroupCount) {
ASSERT_OK_AND_ASSIGN(
std::unique_ptr<FileFormat> file_format,
FileFormatFactory::Get("parquet", {{Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "parquet"},
{"parquet.writer.max.memory.use", "1"}}));
auto schema_pair = PrepareArrowSchema();
diff --git a/src/paimon/format/parquet/parquet_writer_builder_test.cpp b/src/paimon/format/parquet/parquet_writer_builder_test.cpp
index 3a95a8de..796e002e 100644
--- a/src/paimon/format/parquet/parquet_writer_builder_test.cpp
+++ b/src/paimon/format/parquet/parquet_writer_builder_test.cpp
@@ -41,7 +41,6 @@ TEST(ParquetWriterBuilderTest, DefaultPrepareWriterProperties) {
std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
std::map<std::string, std::string> options;
options[Options::FILE_FORMAT] = "parquet";
- options[Options::MANIFEST_FORMAT] = "parquet";
ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options);
ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("zstd"));
ASSERT_EQ(::parquet::ParquetVersion::PARQUET_2_6, properties->version());
@@ -64,7 +63,6 @@ TEST(ParquetWriterBuilderTest, PrepareWriterProperties) {
options[PARQUET_COMPRESSION_CODEC_ZSTD_LEVEL] = "3";
options[PARQUET_BLOCK_SIZE] = "2048";
options[Options::FILE_FORMAT] = "parquet";
- options[Options::MANIFEST_FORMAT] = "parquet";
ParquetWriterBuilder builder(schema, /*batch_size=*/1024 * 1024, options);
ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("zstd"));
// Version numbering differs between C++ and Java Parquet implementations. Java's PARQUET_2_0
@@ -77,12 +75,24 @@ TEST(ParquetWriterBuilderTest, PrepareWriterProperties) {
ASSERT_EQ(3, properties->default_column_properties().compression_level());
}
+TEST(ParquetWriterBuilderTest, PrepareWriterPropertiesWithDictionaryDisabled) {
+ arrow::FieldVector fields;
+ std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
+ std::map<std::string, std::string> options = {
+ {Options::FILE_FORMAT, "parquet"},
+ {"parquet.enable.dictionary", "false"},
+ };
+ ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<::parquet::WriterProperties> properties,
+ builder.PrepareWriterProperties("zstd"));
+ ASSERT_FALSE(properties->default_column_properties().dictionary_enabled());
+}
+
TEST(ParquetWriterBuilderTest, PrepareWriterPropertiesWithFileBlockSize) {
arrow::FieldVector fields;
std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
std::map<std::string, std::string> options = {
{Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "parquet"},
{PARQUET_BLOCK_SIZE, "2048"},
{Options::FILE_BLOCK_SIZE, "8 KB"},
};
@@ -101,7 +111,6 @@ TEST(ParquetWriterBuilderTest, PrepareWriterPropertiesWithZstdLevelPriority) {
options[PARQUET_COMPRESSION_CODEC_ZSTD_LEVEL] = "3";
options[PARQUET_WRITER_VERSION] = "PARQUET_1_0";
options[Options::FILE_FORMAT] = "parquet";
- options[Options::MANIFEST_FORMAT] = "parquet";
ParquetWriterBuilder builder(schema, /*batch_size=*/1024 * 1024, options);
ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("zstd"));
ASSERT_EQ(3, properties->default_column_properties().compression_level());
@@ -111,7 +120,6 @@ TEST(ParquetWriterBuilderTest, PrepareWriterPropertiesWithZstdLevelPriority) {
std::map<std::string, std::string> options;
options[Options::FILE_COMPRESSION_ZSTD_LEVEL] = "4";
options[Options::FILE_FORMAT] = "parquet";
- options[Options::MANIFEST_FORMAT] = "parquet";
ParquetWriterBuilder builder(schema, /*batch_size=*/1024 * 1024, options);
ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("zstd"));
ASSERT_EQ(4, properties->default_column_properties().compression_level());
@@ -124,7 +132,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) {
{
std::map<std::string, std::string> options;
options[Options::FILE_FORMAT] = "parquet";
- options[Options::MANIFEST_FORMAT] = "parquet";
ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options);
ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("lz4"));
ASSERT_EQ(properties->default_column_properties().compression(),
@@ -133,7 +140,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) {
{
std::map<std::string, std::string> options;
options[Options::FILE_FORMAT] = "parquet";
- options[Options::MANIFEST_FORMAT] = "parquet";
ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options);
ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("lz4_raW"));
ASSERT_EQ(properties->default_column_properties().compression(), arrow::Compression::LZ4);
@@ -141,7 +147,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) {
{
std::map<std::string, std::string> options;
options[Options::FILE_FORMAT] = "parquet";
- options[Options::MANIFEST_FORMAT] = "parquet";
ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options);
ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("zstd"));
ASSERT_EQ(properties->default_column_properties().compression(), arrow::Compression::ZSTD);
@@ -149,7 +154,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) {
{
std::map<std::string, std::string> options;
options[Options::FILE_FORMAT] = "parquet";
- options[Options::MANIFEST_FORMAT] = "parquet";
ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options);
ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("LZ4"));
ASSERT_EQ(properties->default_column_properties().compression(),
@@ -158,7 +162,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) {
{
std::map<std::string, std::string> options;
options[Options::FILE_FORMAT] = "parquet";
- options[Options::MANIFEST_FORMAT] = "parquet";
ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options);
ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("ZSTd"));
ASSERT_EQ(properties->default_column_properties().compression(), arrow::Compression::ZSTD);
@@ -166,7 +169,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) {
{
std::map<std::string, std::string> options;
options[Options::FILE_FORMAT] = "parquet";
- options[Options::MANIFEST_FORMAT] = "parquet";
options[PARQUET_COMPRESSION_CODEC_ZLIB_LEVEL] = "3";
ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options);
ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("gzip"));
@@ -176,7 +178,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) {
{
std::map<std::string, std::string> options;
options[Options::FILE_FORMAT] = "parquet";
- options[Options::MANIFEST_FORMAT] = "parquet";
ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options);
ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("snappy"));
ASSERT_EQ(properties->default_column_properties().compression(),
@@ -185,7 +186,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) {
{
std::map<std::string, std::string> options;
options[Options::FILE_FORMAT] = "parquet";
- options[Options::MANIFEST_FORMAT] = "parquet";
ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options);
ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("lzo"));
ASSERT_EQ(properties->default_column_properties().compression(), arrow::Compression::LZO);
@@ -193,14 +193,12 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) {
{
std::map<std::string, std::string> options;
options[Options::FILE_FORMAT] = "parquet";
- options[Options::MANIFEST_FORMAT] = "parquet";
ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options);
ASSERT_NOK(builder.PrepareWriterProperties("unknown"));
}
{
std::map<std::string, std::string> options;
options[Options::FILE_FORMAT] = "parquet";
- options[Options::MANIFEST_FORMAT] = "parquet";
ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options);
ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("uncompressed"));
ASSERT_EQ(properties->default_column_properties().compression(),
@@ -209,7 +207,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) {
{
std::map<std::string, std::string> options;
options[Options::FILE_FORMAT] = "parquet";
- options[Options::MANIFEST_FORMAT] = "parquet";
ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options);
ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("lz4_hadoop"));
ASSERT_EQ(properties->default_column_properties().compression(),
@@ -218,7 +215,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) {
{
std::map<std::string, std::string> options;
options[Options::FILE_FORMAT] = "parquet";
- options[Options::MANIFEST_FORMAT] = "parquet";
options[PARQUET_COMPRESSION_CODEC_BROTLI_LEVEL] = "2";
ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options);
ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("brotli"));
@@ -229,7 +225,6 @@ TEST(ParquetWriterBuilderTest, TestPrepareWriterPropertiesFileCompression) {
{
std::map<std::string, std::string> options;
options[Options::FILE_FORMAT] = "parquet";
- options[Options::MANIFEST_FORMAT] = "parquet";
ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options);
ASSERT_OK_AND_ASSIGN(auto properties, builder.PrepareWriterProperties("None"));
ASSERT_EQ(properties->default_column_properties().compression(),
@@ -242,7 +237,6 @@ TEST(ParquetWriterBuilderTest, TestInvalidWriterVersion) {
std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
std::map<std::string, std::string> options;
options[Options::FILE_FORMAT] = "parquet";
- options[Options::MANIFEST_FORMAT] = "parquet";
options[PARQUET_WRITER_VERSION] = "PARQUET_3_0";
ParquetWriterBuilder builder(schema, /*batch_size=*/1024, options);
ASSERT_NOK_WITH_MSG(builder.PrepareWriterProperties("zstd"),
diff --git a/src/paimon/format/parquet/variant_parquet_test.cpp b/src/paimon/format/parquet/variant_parquet_test.cpp
index a1481dcf..46a63e38 100644
--- a/src/paimon/format/parquet/variant_parquet_test.cpp
+++ b/src/paimon/format/parquet/variant_parquet_test.cpp
@@ -711,7 +711,6 @@ TEST_F(VariantParquetTest, AdaptiveInferenceUntypedPhysicalWriteAndReadRoundTrip
auto logical = BuildArray(jsons);
ASSERT_OK_AND_ASSIGN(CoreOptions options,
CoreOptions::FromMap({
- {Options::MANIFEST_FORMAT, "parquet"},
{Options::VARIANT_INFER_SHREDDING_SCHEMA, "true"},
{Options::VARIANT_SHREDDING_INFERENCE_MODE, "adaptive"},
}));
diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp
index ed61d859..c0223af4 100644
--- a/src/paimon/rest/rest_catalog_test.cpp
+++ b/src/paimon/rest/rest_catalog_test.cpp
@@ -321,7 +321,6 @@ class RestCatalogTest : public ::testing::Test {
// registered in the binary's own registry even when the real format plugin
// dylibs register into a different one (macOS two-level namespace)
{Options::FILE_FORMAT, "mock_format"},
- {Options::MANIFEST_FORMAT, "mock_format"},
{"header.x-client-header", "from-client"},
{"header.x-shared-header", "from-client"},
};
diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp
index b26c278e..4b76cef1 100644
--- a/test/inte/blob_table_inte_test.cpp
+++ b/test/inte/blob_table_inte_test.cpp
@@ -124,8 +124,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter
}
void CreateTable(const std::vector<std::string>& partition_keys) const {
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, GetParam()},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -207,7 +206,6 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter
arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
BlobUtils::ToArrowField("view", true)};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, GetParam()},
{Options::BUCKET, "-1"},
{Options::ROW_TRACKING_ENABLED, "true"},
@@ -603,11 +601,13 @@ TEST_P(BlobTableInteTest, TestAppendTableWriteWithBlobAsDescriptorTrue) {
arrow::field("f1", arrow::int32()),
BlobUtils::ToArrowField("blob", true)};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()},
- {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"},
- {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
- {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::FILE_SYSTEM, "local"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
+ {Options::TARGET_FILE_SIZE, "700"},
+ {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"},
+ {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::BLOB_AS_DESCRIPTOR, "true"},
+ {Options::FILE_SYSTEM, "local"}};
CreateTable(fields, /*partition_keys=*/{}, options);
std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
@@ -644,11 +644,13 @@ TEST_P(BlobTableInteTest, TestAppendTableWriteWithBlobAsDescriptorFalse) {
arrow::field("f1", arrow::int32()),
BlobUtils::ToArrowField("blob", true)};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()},
- {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"},
- {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
- {Options::BLOB_AS_DESCRIPTOR, "false"}, {Options::FILE_SYSTEM, "local"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
+ {Options::TARGET_FILE_SIZE, "700"},
+ {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"},
+ {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::BLOB_AS_DESCRIPTOR, "false"},
+ {Options::FILE_SYSTEM, "local"}};
CreateTable(fields, /*partition_keys=*/{}, options);
std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
@@ -679,10 +681,13 @@ TEST_P(BlobTableInteTest, TestWriteNullOnMissingFile) {
BlobUtils::ToArrowField("blob", true)};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()},
- {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"},
- {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
- {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::BLOB_WRITE_NULL_ON_MISSING_FILE, "true"},
+ {Options::FILE_FORMAT, GetParam()},
+ {Options::TARGET_FILE_SIZE, "700"},
+ {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"},
+ {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::BLOB_AS_DESCRIPTOR, "true"},
+ {Options::BLOB_WRITE_NULL_ON_MISSING_FILE, "true"},
{Options::FILE_SYSTEM, "local"}};
CreateTable(fields, /*partition_keys=*/{}, options);
std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
@@ -739,11 +744,13 @@ TEST_P(BlobTableInteTest, TestMissingFileFailsWriteWhenWriteNullDisabled) {
arrow::field("f1", arrow::int32()),
BlobUtils::ToArrowField("blob", true)};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()},
- {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"},
- {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
- {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::FILE_SYSTEM, "local"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
+ {Options::TARGET_FILE_SIZE, "700"},
+ {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"},
+ {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::BLOB_AS_DESCRIPTOR, "true"},
+ {Options::FILE_SYSTEM, "local"}};
CreateTable(fields, /*partition_keys=*/{}, options);
std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
@@ -771,7 +778,6 @@ TEST_P(BlobTableInteTest, TestWriteNullOnFetchFailure) {
BlobUtils::ToArrowField("blob", true)};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, GetParam()},
{Options::TARGET_FILE_SIZE, "700"},
{Options::BUCKET, "-1"},
@@ -838,7 +844,6 @@ TEST_P(BlobTableInteTest, TestWriteNullOnFetchFailureCoversMissingFile) {
BlobUtils::ToArrowField("blob", true)};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, GetParam()},
{Options::TARGET_FILE_SIZE, "700"},
{Options::BUCKET, "-1"},
@@ -940,8 +945,7 @@ TEST_P(BlobTableInteTest, TestBasic) {
}
TEST_P(BlobTableInteTest, TestBlobFilesAcrossSchemaIds) {
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, GetParam()},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -1116,8 +1120,7 @@ TEST_P(BlobTableInteTest, TestMultipleAppends) {
TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyWriteWithFirstRowId) {
arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")};
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, GetParam()},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -1198,8 +1201,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateFallback) {
arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
arrow::field("f1", arrow::utf8()),
BlobUtils::ToArrowField("b0", /*nullable=*/true)};
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, GetParam()},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -1282,8 +1284,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateFallback) {
TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateMultipleLayers) {
arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")};
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, GetParam()},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -1353,10 +1354,11 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateMultipleLayers) {
TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithDeletionVectors) {
arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()},
- {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"},
- {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::DELETION_VECTORS_ENABLED, "true"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
+ {Options::FILE_SYSTEM, "local"},
+ {Options::ROW_TRACKING_ENABLED, "true"},
+ {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::DELETION_VECTORS_ENABLED, "true"}};
CreateTable(fields, /*partition_keys=*/{}, options);
std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
auto schema = arrow::schema(fields);
@@ -1437,8 +1439,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithDeletionVectors)
TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateCompactedLayers) {
arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")};
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, GetParam()},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -1504,8 +1505,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateCompactedLayers) {
TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithRowRanges) {
arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")};
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, GetParam()},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -1564,8 +1564,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithRowRanges) {
TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateRowTrackingWithSubrangeLayer) {
arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
BlobUtils::ToArrowField("b0", /*nullable=*/true)};
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, GetParam()},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -1650,8 +1649,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateRowTrackingWithSubra
TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateAllPlaceholderRowTracking) {
arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
BlobUtils::ToArrowField("b0", /*nullable=*/true)};
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, GetParam()},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -1706,8 +1704,7 @@ TEST_P(BlobTableInteTest, TestBlobValueEqualToPlaceholderSentinelBytes) {
arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
arrow::field("f1", arrow::utf8()),
BlobUtils::ToArrowField("b0", /*nullable=*/true)};
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, GetParam()},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -1773,8 +1770,7 @@ TEST_P(BlobTableInteTest, TestBlobSentinelValueInBaseLayerDegradesToNull) {
arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
arrow::field("f1", arrow::utf8()),
BlobUtils::ToArrowField("b0", /*nullable=*/true)};
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, GetParam()},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -1817,13 +1813,10 @@ TEST_P(BlobTableInteTest, TestUserSuppliedInternalPlaceholderOptionsIgnored) {
arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
arrow::field("f1", arrow::utf8()),
BlobUtils::ToArrowField("b0", /*nullable=*/true)};
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, GetParam()},
- {Options::FILE_SYSTEM, "local"},
- {Options::ROW_TRACKING_ENABLED, "true"},
- {Options::DATA_EVOLUTION_ENABLED, "true"},
- {BlobDefs::kWritePlaceholderKey, "true"},
- {BlobDefs::kEmitPlaceholderSentinelKey, "true"}};
+ std::map<std::string, std::string> options = {
+ {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"},
+ {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {BlobDefs::kWritePlaceholderKey, "true"}, {BlobDefs::kEmitPlaceholderSentinelKey, "true"}};
CreateTable(fields, /*partition_keys=*/{}, options);
std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
auto schema = arrow::schema(fields);
@@ -1844,8 +1837,7 @@ TEST_P(BlobTableInteTest, TestUserSuppliedInternalPlaceholderOptionsIgnored) {
TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyFirstCommitFailsWithoutFirstRowId) {
arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")};
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, GetParam()},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -2002,10 +1994,9 @@ TEST_P(BlobTableInteTest, TestMoreDataWithDataEvolution) {
TEST_P(BlobTableInteTest, TestBlobWriteMultiRound) {
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()},
- {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"},
- {Options::BLOB_TARGET_FILE_SIZE, "1000"}, {Options::TARGET_FILE_SIZE, "100"},
- {Options::DATA_EVOLUTION_ENABLED, "true"}};
+ {Options::FILE_FORMAT, GetParam()}, {Options::FILE_SYSTEM, "local"},
+ {Options::ROW_TRACKING_ENABLED, "true"}, {Options::BLOB_TARGET_FILE_SIZE, "1000"},
+ {Options::TARGET_FILE_SIZE, "100"}, {Options::DATA_EVOLUTION_ENABLED, "true"}};
CreateTable(/*partition_keys=*/{}, options);
std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
auto schema = arrow::schema(fields_);
@@ -2045,7 +2036,6 @@ TEST_P(BlobTableInteTest, TestExternalPath) {
std::string external_test_dir = external_dir->Str();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, GetParam()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
@@ -2106,10 +2096,11 @@ TEST_P(BlobTableInteTest, TestExternalPath) {
TEST_P(BlobTableInteTest, TestPartitionWithPredicate) {
auto file_format = GetParam();
std::vector<std::string> partition_keys = {"f0"};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()},
- {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"},
- {Options::DATA_EVOLUTION_ENABLED, "true"}, {"parquet.write.max-row-group-length", "1"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
+ {Options::FILE_SYSTEM, "local"},
+ {Options::ROW_TRACKING_ENABLED, "true"},
+ {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {"parquet.write.max-row-group-length", "1"}};
CreateTable(partition_keys, options);
std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
@@ -2227,8 +2218,7 @@ TEST_P(BlobTableInteTest, TestPredicate) {
return;
}
if (GetParam() == "mosaic") {
- CreateTable(/*partition_keys=*/{}, {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, GetParam()},
+ CreateTable(/*partition_keys=*/{}, {{Options::FILE_FORMAT, GetParam()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"},
@@ -2567,14 +2557,11 @@ TEST_P(BlobTableInteTest, TestWithRowIdsSimple) {
TEST_P(BlobTableInteTest, TestWithRowIdsForMultipleBlobFiles) {
auto file_format = GetParam();
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1000"},
- {Options::BLOB_TARGET_FILE_SIZE, "80"},
- {Options::BUCKET, "-1"},
- {Options::ROW_TRACKING_ENABLED, "true"},
- {Options::DATA_EVOLUTION_ENABLED, "true"},
- {Options::FILE_SYSTEM, "local"}};
+ std::map<std::string, std::string> options = {
+ {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1000"},
+ {Options::BLOB_TARGET_FILE_SIZE, "80"}, {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::FILE_SYSTEM, "local"}};
CreateTable(/*partition_keys=*/{}, options);
std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
auto schema = arrow::schema(fields_);
@@ -2677,11 +2664,12 @@ TEST_P(BlobTableInteTest, TestAppendTableWriteWithMultipleBlobFields) {
arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()),
BlobUtils::ToArrowField("blob1", true), BlobUtils::ToArrowField("blob2", true)};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()},
- {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"},
- {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
- {Options::FILE_SYSTEM, "local"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
+ {Options::TARGET_FILE_SIZE, "700"},
+ {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"},
+ {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::FILE_SYSTEM, "local"}};
CreateTable(fields, /*partition_keys=*/{}, options);
std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
@@ -2704,8 +2692,7 @@ TEST_P(BlobTableInteTest, TestAppendWriteWithNullBlob) {
arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
BlobUtils::ToArrowField("blob", true)};
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, GetParam()},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
{Options::BUCKET, "-1"},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
@@ -2811,11 +2798,13 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorField) {
BlobUtils::ToArrowField("b0", true),
BlobUtils::ToArrowField("b1", true)};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()},
- {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"},
- {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
- {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
+ {Options::TARGET_FILE_SIZE, "700"},
+ {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"},
+ {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"},
+ {Options::FILE_SYSTEM, "local"}};
CreateTable(fields, /*partition_keys=*/{}, options);
std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
@@ -2869,11 +2858,13 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialInline) {
BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true),
BlobUtils::ToArrowField("b3", true)};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()},
- {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"},
- {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
- {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
+ {Options::TARGET_FILE_SIZE, "700"},
+ {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"},
+ {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"},
+ {Options::FILE_SYSTEM, "local"}};
CreateTable(fields, /*partition_keys=*/{}, options);
std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
@@ -2932,11 +2923,13 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorMultiCommitAndShuffledReadSchema) {
BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true),
BlobUtils::ToArrowField("b3", true)};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()},
- {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"},
- {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
- {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
+ {Options::TARGET_FILE_SIZE, "700"},
+ {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"},
+ {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"},
+ {Options::FILE_SYSTEM, "local"}};
CreateTable(fields, /*partition_keys=*/{}, options);
std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
auto schema = arrow::schema(fields);
@@ -3061,7 +3054,6 @@ TEST_P(BlobTableInteTest, TestSharedShreddingWithBlobDataEvolution) {
BlobUtils::ToArrowField("payload"),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, GetParam()},
{Options::BUCKET, "-1"},
{Options::ROW_TRACKING_ENABLED, "true"},
@@ -3121,7 +3113,6 @@ TEST_P(BlobTableInteTest, TestMultipleSharedShreddingMapsWithBlobDataEvolution)
BlobUtils::ToArrowField("payload"),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, GetParam()},
{Options::BUCKET, "-1"},
{Options::ROW_TRACKING_ENABLED, "true"},
@@ -3182,7 +3173,6 @@ TEST_P(BlobTableInteTest, TestSharedShreddingMapOverrideWithBlobDataEvolution) {
BlobUtils::ToArrowField("payload"),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, GetParam()},
{Options::BUCKET, "-1"},
{Options::ROW_TRACKING_ENABLED, "true"},
@@ -3241,7 +3231,6 @@ TEST_P(BlobTableInteTest, TestOrcMapStorageLayoutEvolutionWithBlobDataEvolution)
BlobUtils::ToArrowField("payload"),
};
std::map<std::string, std::string> options_v0 = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, "orc"},
{Options::BUCKET, "-1"},
{Options::ROW_TRACKING_ENABLED, "true"},
@@ -3312,11 +3301,13 @@ TEST_P(BlobTableInteTest, TestDataEvolutionWithBlobDescriptorField) {
BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true),
BlobUtils::ToArrowField("b3", true)};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()},
- {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"},
- {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
- {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
+ {Options::TARGET_FILE_SIZE, "700"},
+ {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"},
+ {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"},
+ {Options::FILE_SYSTEM, "local"}};
CreateTable(fields, /*partition_keys=*/{}, options);
std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
@@ -3429,11 +3420,13 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorFieldWriteRawBytesDirectly) {
BlobUtils::ToArrowField("b0", true),
BlobUtils::ToArrowField("b1", true)};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()},
- {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"},
- {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
- {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, GetParam()},
+ {Options::TARGET_FILE_SIZE, "700"},
+ {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"},
+ {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"},
+ {Options::FILE_SYSTEM, "local"}};
CreateTable(fields, /*partition_keys=*/{}, options);
std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
@@ -3468,7 +3461,6 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamTable) {
arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
BlobUtils::ToArrowField("view", true)};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, file_format},
{Options::BUCKET, "-1"},
{Options::ROW_TRACKING_ENABLED, "true"},
@@ -3637,13 +3629,10 @@ TEST_P(BlobTableInteTest, TestForwardBlobViewReference) {
// dynamically disabled can succeed on it.
arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
BlobUtils::ToArrowField("view", true)};
- std::map<std::string, std::string> source_options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format},
- {Options::BUCKET, "-1"},
- {Options::ROW_TRACKING_ENABLED, "true"},
- {Options::DATA_EVOLUTION_ENABLED, "true"},
- {Options::BLOB_VIEW_FIELD, "view"},
- {Options::FILE_SYSTEM, "local"}};
+ std::map<std::string, std::string> source_options = {
+ {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::BLOB_VIEW_FIELD, "view"}, {Options::FILE_SYSTEM, "local"}};
CreateTable(fields, /*partition_keys=*/{}, source_options);
std::string source_table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
@@ -3788,13 +3777,9 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamDescriptorBlob) {
BlobUtils::ToArrowField("b1", true)};
auto upstream_schema = arrow::schema(upstream_fields);
std::map<std::string, std::string> upstream_options = {
- {Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format},
- {Options::BUCKET, "-1"},
- {Options::ROW_TRACKING_ENABLED, "true"},
- {Options::DATA_EVOLUTION_ENABLED, "true"},
- {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"},
- {Options::FILE_SYSTEM, "local"}};
+ {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}};
::ArrowSchema upstream_c_schema;
ASSERT_TRUE(arrow::ExportSchema(*upstream_schema, &upstream_c_schema).ok());
@@ -3830,7 +3815,6 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamDescriptorBlob) {
arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
BlobUtils::ToArrowField("view", true)};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, file_format},
{Options::BUCKET, "-1"},
{Options::ROW_TRACKING_ENABLED, "true"},
@@ -3936,7 +3920,6 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithMultipleUpstreamTables) {
BlobUtils::ToArrowField("view1", true),
BlobUtils::ToArrowField("view2", true)};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, file_format},
{Options::BUCKET, "-1"},
{Options::ROW_TRACKING_ENABLED, "true"},
@@ -4214,13 +4197,9 @@ TEST_P(BlobTableInteTest, TestBlobViewWithFallbackPath) {
BlobUtils::ToArrowField("blob", true)};
auto upstream_schema = arrow::schema(upstream_fields);
std::map<std::string, std::string> upstream_options = {
- {Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format},
- {Options::BUCKET, "-1"},
- {Options::ROW_TRACKING_ENABLED, "true"},
- {Options::DATA_EVOLUTION_ENABLED, "true"},
- {Options::BLOB_AS_DESCRIPTOR, "true"},
- {Options::FILE_SYSTEM, "local"}};
+ {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::FILE_SYSTEM, "local"}};
// Create the upstream table at the fallback path: <warehouse>/db/table (no .db).
auto upstream_dir = UniqueTestDirectory::Create("local");
@@ -4261,7 +4240,6 @@ TEST_P(BlobTableInteTest, TestBlobViewWithFallbackPath) {
arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
BlobUtils::ToArrowField("view", true)};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, file_format},
{Options::BUCKET, "-1"},
{Options::ROW_TRACKING_ENABLED, "true"},
diff --git a/test/inte/clean_inte_test.cpp b/test/inte/clean_inte_test.cpp
index 89cadd4f..3f7f353e 100644
--- a/test/inte/clean_inte_test.cpp
+++ b/test/inte/clean_inte_test.cpp
@@ -190,7 +190,7 @@ class CleanInteTest : public testing::Test {
};
TEST_F(CleanInteTest, TestExpireSnapshotFailover) {
- std::string test_data_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/";
+ std::string test_data_path = paimon::test::GetDataDir() + "/parquet/append_09.db/append_09/";
std::map<std::string, std::string> clean_options = {
{Options::MANIFEST_TARGET_FILE_SIZE, "8mb"},
{Options::FILE_SYSTEM, "local"},
@@ -218,7 +218,7 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) {
std::string table_path = dir->Str();
ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path));
ASSERT_OK(file_system_->Delete(PathUtil::JoinPath(
- table_path, "manifest/manifest-list-616d1847-a02c-495f-9cca-2c8b7def0fec-1")));
+ table_path, "manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1")));
CommitContextBuilder commit_context_builder(table_path, "commit_user_1");
ASSERT_OK_AND_ASSIGN(
std::unique_ptr<CommitContext> commit_context,
@@ -233,7 +233,7 @@ TEST_F(CleanInteTest, TestExpireSnapshotFailover) {
std::string table_path = dir->Str();
ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path));
ASSERT_OK(file_system_->Delete(PathUtil::JoinPath(
- table_path, "f1=10/bucket-1/data-10b9eea8-241d-4e4b-8ab8-2a82d72d79a2-0.orc")));
+ table_path, "f1=10/bucket-1/data-7a912f84-04b7-4bbb-8dc6-53f4a292ea25-0.parquet")));
CommitContextBuilder commit_context_builder(table_path, "commit_user_1");
ASSERT_OK_AND_ASSIGN(
std::unique_ptr<CommitContext> commit_context,
@@ -272,13 +272,9 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) {
ASSERT_TRUE(dir);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, "orc"},
- {Options::TARGET_FILE_SIZE, "1024"},
- {Options::FILE_SYSTEM, "local"},
- {Options::BUCKET, "2"},
- {Options::BUCKET_KEY, "f2"},
- {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"},
+ {Options::FILE_FORMAT, "orc"}, {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "2"},
+ {Options::BUCKET_KEY, "f2"}, {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"},
};
ASSERT_OK_AND_ASSIGN(
@@ -416,13 +412,9 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) {
auto schema =
arrow::schema(arrow::FieldVector({string_field, int_field, int_field1, double_field}));
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, "orc"},
- {Options::TARGET_FILE_SIZE, "1024"},
- {Options::FILE_SYSTEM, "local"},
- {Options::BUCKET, "2"},
- {Options::BUCKET_KEY, "f2"},
- {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"},
+ {Options::FILE_FORMAT, "orc"}, {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "2"},
+ {Options::BUCKET_KEY, "f2"}, {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"},
};
bool drop_partition_scanned_all_io_hook = false;
@@ -583,13 +575,9 @@ TEST_F(CleanInteTest, TestOrphanFilesClean) {
ASSERT_TRUE(arrow::ExportSchema(*schema, &arrow_schema).ok());
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, "orc"},
- {Options::TARGET_FILE_SIZE, "1024"},
- {Options::FILE_SYSTEM, "local"},
- {Options::BUCKET, "2"},
- {Options::BUCKET_KEY, "f3"},
- {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"},
+ {Options::FILE_FORMAT, "orc"}, {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "2"},
+ {Options::BUCKET_KEY, "f3"}, {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"},
};
auto dir = UniqueTestDirectory::Create();
@@ -701,13 +689,9 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithFileRetainCondition) {
ASSERT_TRUE(arrow::ExportSchema(*schema, &arrow_schema).ok());
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, "orc"},
- {Options::TARGET_FILE_SIZE, "1024"},
- {Options::FILE_SYSTEM, "local"},
- {Options::BUCKET, "2"},
- {Options::BUCKET_KEY, "f3"},
- {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"},
+ {Options::FILE_FORMAT, "orc"}, {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "2"},
+ {Options::BUCKET_KEY, "f3"}, {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"},
};
auto dir = UniqueTestDirectory::Create();
@@ -817,13 +801,9 @@ TEST_F(CleanInteTest, TestOrphanFilesCleanWithIOException) {
ASSERT_TRUE(dir);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, "orc"},
- {Options::TARGET_FILE_SIZE, "1024"},
- {Options::FILE_SYSTEM, "local"},
- {Options::BUCKET, "2"},
- {Options::BUCKET_KEY, "f2"},
- {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"}};
+ {Options::FILE_FORMAT, "orc"}, {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "2"},
+ {Options::BUCKET_KEY, "f2"}, {Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, "true"}};
ASSERT_OK_AND_ASSIGN(std::string table_path,
CreateTestTable(dir->Str(), /*db_name=*/"foo",
diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp
index 4fb26b6e..0d842585 100644
--- a/test/inte/data_evolution_table_test.cpp
+++ b/test/inte/data_evolution_table_test.cpp
@@ -79,8 +79,7 @@ class DataEvolutionTableTest : public ::testing::Test,
}
void CreateTable(const std::vector<std::string>& partition_keys) const {
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, FileFormat()},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, FileFormat()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -160,8 +159,7 @@ class DataEvolutionTableTest : public ::testing::Test,
std::map<std::string, std::string> CreateDataEvolutionTable(
bool deletion_vectors_enabled,
const std::map<std::string, std::string>& extra_options = {}) const {
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, FileFormat()},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, FileFormat()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -861,7 +859,6 @@ TEST_P(DataEvolutionTableTest, TestMultipleSharedShreddingMapsPartialOverwrite)
arrow::field("map2", map_type),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, FileFormat()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
@@ -1190,7 +1187,6 @@ TEST_P(DataEvolutionTableTest, TestMoreData) {
TEST_P(DataEvolutionTableTest, TestOnlyRowTrackingEnabled) {
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, FileFormat()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
@@ -1234,7 +1230,6 @@ TEST_P(DataEvolutionTableTest, TestExternalPath) {
std::string external_test_dir = external_dir->Str();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, FileFormat()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
@@ -1485,10 +1480,11 @@ TEST_P(DataEvolutionTableTest, TestPartitionWithPredicate) {
return;
}
std::vector<std::string> partition_keys = {"f1"};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, FileFormat()},
- {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"},
- {Options::DATA_EVOLUTION_ENABLED, "true"}, {"parquet.write.max-row-group-length", "1"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, FileFormat()},
+ {Options::FILE_SYSTEM, "local"},
+ {Options::ROW_TRACKING_ENABLED, "true"},
+ {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {"parquet.write.max-row-group-length", "1"}};
if (file_format == "mosaic") {
options.emplace(mosaic::MOSAIC_STATS_COLUMNS, "f0");
}
@@ -2134,7 +2130,7 @@ TEST_P(DataEvolutionTableTest, TestFormatPredicatePushDownWithoutFileIndex) {
std::map<std::string, std::string> options = {{Options::FILE_INDEX_READ_ENABLED, "false"},
{Options::WRITE_BATCH_SIZE, "1"},
{"parquet.page.size", "1"},
- {"parquet.enable-dictionary", "false"},
+ {"parquet.enable.dictionary", "false"},
{"parquet.write.enable-page-index", "true"},
{"parquet.read.enable-page-index-filter", "true"},
{"orc.stripe.size", "1"},
@@ -2185,8 +2181,7 @@ TEST_P(DataEvolutionTableTest, TestPredicate) {
return;
}
if (FileFormat() == "mosaic") {
- CreateTable(/*partition_keys=*/{}, {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, FileFormat()},
+ CreateTable(/*partition_keys=*/{}, {{Options::FILE_FORMAT, FileFormat()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"},
@@ -2329,8 +2324,7 @@ TEST_P(DataEvolutionTableTest, TestIOException) {
}
TEST_P(DataEvolutionTableTest, TestWithRowIds) {
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, FileFormat()},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, FileFormat()},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp
index e05dc4b2..2e410513 100644
--- a/test/inte/global_index_test.cpp
+++ b/test/inte/global_index_test.cpp
@@ -85,8 +85,7 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter
}
void CreateTable(const std::vector<std::string>& partition_keys) const {
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format_},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format_},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -264,8 +263,7 @@ TEST_P(GlobalIndexTest, TestWriteLuminaIndex) {
{"lumina.encoding.type", "rawf32"},
{"lumina.search.parallel_number", "10"}};
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format_},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format_},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -322,10 +320,11 @@ TEST_P(GlobalIndexTest, TestWriteLuminaIndexWithMismatchedDimension) {
{"lumina.encoding.type", "rawf32"},
{"lumina.search.parallel_number", "10"}};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format_},
- {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"},
- {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::READ_BATCH_SIZE, "1"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format_},
+ {Options::FILE_SYSTEM, "local"},
+ {Options::ROW_TRACKING_ENABLED, "true"},
+ {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::READ_BATCH_SIZE, "1"}};
CreateTable(/*partition_keys=*/{}, schema, options);
std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
@@ -366,10 +365,9 @@ TEST_P(GlobalIndexTest, TestWriteAndQueryLuminaIndexWithOrcDictionaryStringTags)
R"([{"key_name":"color","type":"enum","value_type":"string"},)"
R"({"key_name":"labels","type":"enum","value_type":"string"}])"}};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format_},
- {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"},
- {Options::DATA_EVOLUTION_ENABLED, "true"}, {"orc.dictionary-key-size-threshold", "1"},
- {"orc.read.enable-lazy-decoding", "true"}};
+ {Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"},
+ {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {"orc.dictionary-key-size-threshold", "1"}, {"orc.read.enable-lazy-decoding", "true"}};
CreateTable(/*partition_keys=*/{}, schema, options);
std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
@@ -961,8 +959,7 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadIndexWithPartition) {
{"lumina.encoding.type", "rawf32"},
{"lumina.search.parallel_number", "10"}};
auto schema = arrow::schema(fields);
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format_},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format_},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -1102,8 +1099,7 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadIndexWithScore) {
{"lumina.encoding.type", "rawf32"},
{"lumina.search.parallel_number", "10"}};
auto schema = arrow::schema(fields);
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format_},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format_},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -1248,8 +1244,7 @@ TEST_P(GlobalIndexTest, TestWriteAndQueryLuminaIndexWithTagNullAndEmptyValues) {
R"({"key_name":"scores","type":"range","value_type":"float"},)"
R"({"key_name":"category","type":"enum","value_type":"int32"},)"
R"({"key_name":"category_ids","type":"enum","value_type":"int32"}])"}};
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format_},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format_},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -1871,8 +1866,7 @@ TEST_P(GlobalIndexTest, TestScanIndexWithTwoIndexes) {
{"lumina.encoding.type", "rawf32"},
{"lumina.search.parallel_number", "10"}};
auto schema = arrow::schema(fields);
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format_},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format_},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -1939,8 +1933,7 @@ TEST_P(GlobalIndexTest, TestDataEvolutionBatchScanWithExternalPath) {
arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::list(arrow::float32())),
arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())};
auto schema = arrow::schema(fields);
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format_},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format_},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -2006,8 +1999,7 @@ TEST_P(GlobalIndexTest, TestIOException) {
])")
.ValueOrDie();
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format_},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format_},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -2332,8 +2324,7 @@ TEST_P(GlobalIndexTest, TestLuceneWriteCommitScanReadIndexWithScore) {
{"lucene-fts.write.omit-term-freq-and-position", "false"},
{"lucene-fts.write.tmp.directory", tmp_dir->Str()}};
auto schema = arrow::schema(fields);
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format_},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format_},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -2418,8 +2409,7 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadLuceneIndexWithPartition) {
{"lucene-fts.write.omit-term-freq-and-position", "false"},
{"lucene-fts.write.tmp.directory", tmp_dir->Str()}};
auto schema = arrow::schema(fields);
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format_},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format_},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -2781,8 +2771,7 @@ TEST_P(GlobalIndexTest, TestBTreeEmptyStringKeyPredicates) {
TEST_P(GlobalIndexTest, TestBTreeWriteCommitScanReadIndexWithPartition) {
// BTree index with partitioned table. Each partition's data is sorted by f0 independently.
auto schema = arrow::schema(fields_);
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format_},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format_},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -2926,8 +2915,7 @@ TEST_P(GlobalIndexTest, TestBTreeWithPartitionAndCustomExecutor) {
// Test that UnionGlobalIndexReader uses a custom 8-thread executor to read
// btree indexes from two partitions in parallel.
auto schema = arrow::schema(fields_);
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format_},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format_},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
@@ -3347,8 +3335,7 @@ TEST_P(GlobalIndexTest, TestBTreeWithLumina) {
{"lumina.encoding.type", "rawf32"},
{"lumina.search.parallel_number", "10"}};
auto schema = arrow::schema(fields);
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format_},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format_},
{Options::FILE_SYSTEM, "local"},
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"}};
diff --git a/test/inte/nested_column_pruning_inte_test.cpp b/test/inte/nested_column_pruning_inte_test.cpp
index 119517af..0f113cb2 100644
--- a/test/inte/nested_column_pruning_inte_test.cpp
+++ b/test/inte/nested_column_pruning_inte_test.cpp
@@ -145,7 +145,6 @@ TEST_P(NestedColumnPruningInteTest, PruneStructSubFields) {
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -202,7 +201,6 @@ TEST_P(NestedColumnPruningInteTest, ProjectStructColumnAsEmptyStructReturnsNullC
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -259,7 +257,6 @@ TEST_P(NestedColumnPruningInteTest, PruneSameNestedFieldNameFromDifferentStructC
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -314,7 +311,6 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsAllNonExistent) {
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -376,7 +372,6 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsWithNonExistentField) {
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -439,7 +434,6 @@ TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsWithTypeMismatchShouldFa
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -523,7 +517,6 @@ TEST_P(NestedColumnPruningInteTest,
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -615,7 +608,6 @@ TEST_P(NestedColumnPruningInteTest, PruneEntireStructField) {
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -670,7 +662,6 @@ TEST_P(NestedColumnPruningInteTest, PruneDeepNestedStruct) {
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -731,7 +722,6 @@ TEST_P(NestedColumnPruningInteTest, PruneNestedStructWithSpecialFields) {
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -819,7 +809,6 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeys) {
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -874,7 +863,6 @@ TEST_P(NestedColumnPruningInteTest, NestedMapSelectedKeysInStruct) {
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -930,7 +918,6 @@ TEST_P(NestedColumnPruningInteTest, PruneStructSubFieldsWithNestedMapSelectedKey
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -988,7 +975,6 @@ TEST_P(NestedColumnPruningInteTest, PruneStructSubFieldsWithNestedMapSelectedKey
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -1045,7 +1031,6 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeysEmptyStringKey) {
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -1096,7 +1081,6 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeysPreserveOrder) {
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -1160,13 +1144,12 @@ TEST_P(NestedColumnPruningInteTest, NestedStructMapSelectedKeysWithPredicate) {
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1048576"},
{Options::BUCKET, "-1"},
{Options::WRITE_BATCH_SIZE, "1"},
{"parquet.page.size", "1"},
- {"parquet.enable-dictionary", "false"},
+ {"parquet.enable.dictionary", "false"},
{"parquet.write.enable-page-index", "true"},
{"parquet.write.max-row-group-length", "1"},
{"parquet.read.enable-page-index-filter", "true"},
@@ -1264,7 +1247,6 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeysWithOrcDictionaryEncodedMap)
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -1367,7 +1349,6 @@ TEST_P(NestedColumnPruningInteTest, PruneDeeperNestedStruct) {
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -1428,7 +1409,6 @@ TEST_P(NestedColumnPruningInteTest, PruneListStructSubFields) {
auto table_schema = arrow::schema(table_fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
diff --git a/test/inte/pk_compaction_inte_test.cpp b/test/inte/pk_compaction_inte_test.cpp
index 13053517..b312e685 100644
--- a/test/inte/pk_compaction_inte_test.cpp
+++ b/test/inte/pk_compaction_inte_test.cpp
@@ -538,7 +538,7 @@ TEST_P(PkCompactionInteTest, TestKeyValueTableDvCompactionWithMapSharedShredding
{"fields.tags.map.storage-layout", "shared-shredding"},
{"fields.tags.map.shared-shredding.max-columns", "2"},
{"parquet.page.size", "1"},
- {"parquet.enable-dictionary", "false"},
+ {"parquet.enable.dictionary", "false"},
{"parquet.write.enable-page-index", "true"},
{"parquet.write.max-row-group-length", "1"},
{"parquet.read.enable-page-index-filter", "true"},
diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp
index a22e3a10..de93a035 100644
--- a/test/inte/read_inte_test.cpp
+++ b/test/inte/read_inte_test.cpp
@@ -634,7 +634,6 @@ TEST_P(ReadInteTest, TestReadOnlyPartitionField) {
TEST(SystemTableReadInteTest, TestReadOptionsSystemTable) {
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
{Options::FILE_FORMAT, "orc"},
- {Options::MANIFEST_FORMAT, "orc"},
{"custom.option", "custom-value"}};
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
@@ -666,10 +665,8 @@ TEST(SystemTableReadInteTest, TestReadOptionsSystemTable) {
ASSERT_OK_AND_ASSIGN(auto result, ReadResultCollector::CollectResult(std::move(batch_reader)));
ASSERT_TRUE(result);
- std::map<std::string, std::string> expected = {{"custom.option", "custom-value"},
- {"file-system", "local"},
- {"file.format", "orc"},
- {"manifest.format", "orc"}};
+ std::map<std::string, std::string> expected = {
+ {"custom.option", "custom-value"}, {"file-system", "local"}, {"file.format", "orc"}};
ASSERT_EQ(CollectStringMap(result), expected) << result->ToString();
}
@@ -683,8 +680,7 @@ TEST(SystemTableReadInteTest, TestReadBranchOptionsSystemTable) {
std::filesystem::path(table_path)));
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
- {Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "avro"}};
+ {Options::FILE_FORMAT, "parquet"}};
std::string system_table_path = table_path + "$branch_rt$options";
ScanContextBuilder scan_context_builder(system_table_path);
scan_context_builder.SetOptions(options);
@@ -713,7 +709,7 @@ TEST(SystemTableReadInteTest, TestReadMetadataSystemTables) {
auto schema = arrow::schema(fields);
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
{Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "avro"},
+
{Options::BUCKET, "1"}};
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
@@ -879,7 +875,7 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTable) {
auto schema = arrow::schema(fields);
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
{Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "avro"},
+
{Options::BUCKET, "1"},
{Options::NUM_LEVELS, "3"}};
auto dir = UniqueTestDirectory::Create();
@@ -938,7 +934,7 @@ TEST(SystemTableReadInteTest, TestReadOptimizedAppendOnlySystemTableWithStreamin
auto schema = arrow::schema(fields);
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
{Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "avro"},
+
{Options::BUCKET, "1"},
{Options::BUCKET_KEY, "k"}};
auto dir = UniqueTestDirectory::Create();
@@ -979,12 +975,12 @@ TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicateP
std::map<std::string, std::string> options = {
{Options::FILE_SYSTEM, "local"},
{Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "avro"},
+
{Options::BUCKET, "1"},
{Options::BUCKET_KEY, "k"},
{Options::WRITE_BATCH_SIZE, "1"},
{"parquet.page.size", "1"},
- {"parquet.enable-dictionary", "false"},
+ {"parquet.enable.dictionary", "false"},
{"parquet.write.enable-page-index", "true"},
{"parquet.write.max-row-group-length", "1"},
{"parquet.read.enable-page-index-filter", "true"}};
@@ -1048,7 +1044,7 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) {
auto schema = arrow::schema(fields);
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
{Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "avro"},
+
{Options::BUCKET, "1"},
{Options::BUCKET_KEY, "k"}};
auto dir = UniqueTestDirectory::Create();
@@ -1109,7 +1105,7 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableWithBranch) {
auto schema = arrow::schema(fields);
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
{Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "avro"},
+
{Options::BUCKET, "1"},
{Options::NUM_LEVELS, "3"}};
auto dir = UniqueTestDirectory::Create();
@@ -1166,10 +1162,9 @@ TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableWithFirstRowMergeEngin
};
auto schema = arrow::schema(fields);
std::map<std::string, std::string> options = {
- {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "avro"}, {Options::BUCKET, "1"},
- {Options::BUCKET_KEY, "k"}, {Options::NUM_LEVELS, "5"},
- {Options::MERGE_ENGINE, "first-row"}};
+ {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"},
+ {Options::BUCKET, "1"}, {Options::BUCKET_KEY, "k"},
+ {Options::NUM_LEVELS, "5"}, {Options::MERGE_ENGINE, "first-row"}};
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
ASSERT_OK_AND_ASSIGN(
@@ -1207,8 +1202,8 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedTable) {
};
auto schema = arrow::schema(fields);
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
- {Options::FILE_FORMAT, "orc"},
- {Options::MANIFEST_FORMAT, "orc"},
+ {Options::FILE_FORMAT, "parquet"},
+
{Options::BUCKET, "1"}};
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
@@ -1262,9 +1257,11 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedPartialWrite
};
auto schema = arrow::schema(fields);
std::map<std::string, std::string> options = {
- {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "avro"}, {Options::BUCKET, "-1"},
- {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::FILE_SYSTEM, "local"},
+ {Options::FILE_FORMAT, "parquet"},
+ {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"},
+ {Options::DATA_EVOLUTION_ENABLED, "true"},
};
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
@@ -1331,8 +1328,8 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForDatePartition) {
};
auto schema = arrow::schema(fields);
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
- {Options::FILE_FORMAT, "orc"},
- {Options::MANIFEST_FORMAT, "orc"},
+ {Options::FILE_FORMAT, "parquet"},
+
{Options::BUCKET, "1"},
{Options::BUCKET_KEY, "v"}};
auto dir = UniqueTestDirectory::Create();
@@ -1433,8 +1430,7 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableWithSchemaEvolutionStats)
TEST(SystemTableReadInteTest, TestReadManifestAndFilesSystemTablesForEmptyTable) {
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
- {Options::FILE_FORMAT, "orc"},
- {Options::MANIFEST_FORMAT, "orc"}};
+ {Options::FILE_FORMAT, "orc"}};
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse");
@@ -1498,8 +1494,7 @@ TEST(SystemTableReadInteTest, TestReadTagBranchAndConsumerSystemTables) {
/*overwrite=*/true));
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
- {Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "avro"}};
+ {Options::FILE_FORMAT, "parquet"}};
ASSERT_OK_AND_ASSIGN(auto branches_result, ReadSystemTable(table_path + "$branches", options));
auto branches_array = SingleStructChunk(branches_result);
@@ -1568,7 +1563,7 @@ TEST(SystemTableReadInteTest, TestReadAuditLogSystemTable) {
std::map<std::string, std::string> options = {
{Options::FILE_SYSTEM, "local"},
{Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "avro"},
+
{Options::BUCKET, "1"},
};
auto dir = UniqueTestDirectory::Create();
@@ -1605,7 +1600,7 @@ TEST(SystemTableReadInteTest, TestReadAuditLogSystemTableWithSequenceNumber) {
std::map<std::string, std::string> options = {
{Options::FILE_SYSTEM, "local"},
{Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "avro"},
+
{Options::BUCKET, "1"},
};
auto dir = UniqueTestDirectory::Create();
@@ -1643,7 +1638,7 @@ TEST(SystemTableReadInteTest, TestReadBinlogSystemTable) {
std::map<std::string, std::string> options = {
{Options::FILE_SYSTEM, "local"},
{Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "avro"},
+
{Options::BUCKET, "1"},
};
auto dir = UniqueTestDirectory::Create();
@@ -1679,7 +1674,7 @@ TEST(SystemTableReadInteTest, TestReadAuditLogAndBinlogSystemTableWithChangelogR
std::map<std::string, std::string> options = {
{Options::FILE_SYSTEM, "local"},
{Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "avro"},
+
{Options::BUCKET, "1"},
};
auto dir = UniqueTestDirectory::Create();
@@ -1734,9 +1729,11 @@ TEST(SystemTableReadInteTest, TestStreamingBinlogPacksUpdateBeforeAndAfter) {
};
auto schema = arrow::schema(fields);
std::map<std::string, std::string> options = {
- {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "avro"}, {Options::BUCKET, "1"},
- {Options::WRITE_BUFFER_SIZE, "1"}, {Options::WRITE_BUFFER_SPILLABLE, "false"},
+ {Options::FILE_SYSTEM, "local"},
+ {Options::FILE_FORMAT, "parquet"},
+ {Options::BUCKET, "1"},
+ {Options::WRITE_BUFFER_SIZE, "1"},
+ {Options::WRITE_BUFFER_SPILLABLE, "false"},
};
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
@@ -1797,7 +1794,7 @@ TEST(SystemTableReadInteTest, TestReadBinlogSystemTableWithNullValue) {
std::map<std::string, std::string> options = {
{Options::FILE_SYSTEM, "local"},
{Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "avro"},
+
{Options::BUCKET, "1"},
};
auto dir = UniqueTestDirectory::Create();
@@ -1830,7 +1827,7 @@ TEST(SystemTableReadInteTest, TestReadAuditLogAndBinlogSystemTableWithBranch) {
auto schema = arrow::schema(fields);
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
{Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "avro"},
+
{Options::BUCKET, "1"}};
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
@@ -1885,7 +1882,7 @@ TEST(SystemTableReadInteTest, TestReadAuditLogAndBinlogSystemTableWithNonPrimary
std::map<std::string, std::string> options = {
{Options::FILE_SYSTEM, "local"},
{Options::FILE_FORMAT, "parquet"},
- {Options::MANIFEST_FORMAT, "avro"},
+
{Options::BUCKET, "-1"},
};
auto dir = UniqueTestDirectory::Create();
@@ -4197,7 +4194,6 @@ TEST(SystemTableReadInteTest, TestReadGlobalCatalogOptions) {
TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) {
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
{Options::FILE_FORMAT, "orc"},
- {Options::MANIFEST_FORMAT, "orc"},
{"table.option.custom", "my-value"}};
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
@@ -4274,15 +4270,12 @@ TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) {
}
TEST(SystemTableReadInteTest, TestReadGlobalTables) {
- std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
- {Options::FILE_FORMAT, "orc"},
- {Options::MANIFEST_FORMAT, "orc"},
- {Options::BUCKET, "1"},
- {"owner", "alice"},
- {"createdAt", "1000"},
- {"createdBy", "creator"},
- {"updatedAt", "2000"},
- {"updatedBy", "updater"}};
+ std::map<std::string, std::string> options = {
+ {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"},
+
+ {Options::BUCKET, "1"}, {"owner", "alice"},
+ {"createdAt", "1000"}, {"createdBy", "creator"},
+ {"updatedAt", "2000"}, {"updatedBy", "updater"}};
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse");
@@ -4393,8 +4386,8 @@ TEST(SystemTableReadInteTest, TestReadGlobalTables) {
TEST(SystemTableReadInteTest, TestReadGlobalPartitions) {
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
- {Options::FILE_FORMAT, "orc"},
- {Options::MANIFEST_FORMAT, "orc"},
+ {Options::FILE_FORMAT, "parquet"},
+
{Options::BUCKET, "1"},
{Options::BUCKET_KEY, "v"}};
auto dir = UniqueTestDirectory::Create();
@@ -4450,7 +4443,6 @@ TEST(SystemTableReadInteTest, TestReadGlobalPartitions) {
TEST(SystemTableReadInteTest, TestGlobalSystemTablesPropagateCorruptSchema) {
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
{Options::FILE_FORMAT, "orc"},
- {Options::MANIFEST_FORMAT, "orc"},
{Options::BUCKET, "1"},
{Options::BUCKET_KEY, "v"}};
auto dir = UniqueTestDirectory::Create();
@@ -4480,8 +4472,8 @@ TEST(SystemTableReadInteTest, TestGlobalSystemTablesPropagateCorruptSchema) {
TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptSnapshot) {
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
- {Options::FILE_FORMAT, "orc"},
- {Options::MANIFEST_FORMAT, "orc"},
+ {Options::FILE_FORMAT, "parquet"},
+
{Options::BUCKET, "1"},
{Options::BUCKET_KEY, "v"}};
auto dir = UniqueTestDirectory::Create();
@@ -4515,8 +4507,8 @@ TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptSnapshot
TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptManifest) {
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
- {Options::FILE_FORMAT, "orc"},
- {Options::MANIFEST_FORMAT, "orc"},
+ {Options::FILE_FORMAT, "parquet"},
+
{Options::BUCKET, "1"},
{Options::BUCKET_KEY, "v"}};
auto dir = UniqueTestDirectory::Create();
diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp
index 5bc7bcb6..6a58a97f 100644
--- a/test/inte/realtime_write_inte_test.cpp
+++ b/test/inte/realtime_write_inte_test.cpp
@@ -324,9 +324,11 @@ class RealtimeWriteInteTest : public ::testing::Test {
arrow::field("pt", arrow::utf8())};
schema_ = arrow::schema(fields_);
options_ = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "orc"},
- {Options::FILE_SYSTEM, "local"}, {Options::BUCKET, "1"},
- {Options::BUCKET_KEY, "id"}, {Options::TARGET_FILE_SIZE, "1048576"},
+ {Options::FILE_FORMAT, "orc"},
+ {Options::FILE_SYSTEM, "local"},
+ {Options::BUCKET, "1"},
+ {Options::BUCKET_KEY, "id"},
+ {Options::TARGET_FILE_SIZE, "1048576"},
{Options::REALTIME_ENABLED, "true"},
};
}
@@ -2541,7 +2543,7 @@ TEST_F(RealtimeWriteInteTest, TestDiskPredicatePushdownWithoutMemoryFiltering) {
options_[Options::FILE_FORMAT] = "parquet";
options_[Options::WRITE_BATCH_SIZE] = "1";
options_["parquet.page.size"] = "1";
- options_["parquet.enable-dictionary"] = "false";
+ options_["parquet.enable.dictionary"] = "false";
options_["parquet.write.enable-page-index"] = "true";
options_["parquet.read.enable-page-index-filter"] = "true";
CreateTable(/*partition_keys=*/{});
@@ -2676,7 +2678,7 @@ TEST_F(RealtimeWriteInteTest, TestMemoryBatchStatisticsPredicatePushdownWithDisk
options_[Options::WRITE_BATCH_SIZE] = "1";
options_[Options::REALTIME_STORE_STATS_MODE] = "full";
options_["parquet.page.size"] = "1";
- options_["parquet.enable-dictionary"] = "false";
+ options_["parquet.enable.dictionary"] = "false";
options_["parquet.write.enable-page-index"] = "true";
options_["parquet.read.enable-page-index-filter"] = "true";
CreateTable(/*partition_keys=*/{});
@@ -2730,7 +2732,7 @@ TEST_F(RealtimeWriteInteTest, TestNullPredicateForMemoryAndDisk) {
options_[Options::WRITE_BATCH_SIZE] = "1";
options_[Options::REALTIME_STORE_STATS_MODE] = "full";
options_["parquet.page.size"] = "1";
- options_["parquet.enable-dictionary"] = "false";
+ options_["parquet.enable.dictionary"] = "false";
options_["parquet.write.enable-page-index"] = "true";
options_["parquet.write.max-row-group-length"] = "1";
options_["parquet.read.enable-page-index-filter"] = "true";
diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp
index de2d347f..c4c50cc4 100644
--- a/test/inte/scan_and_read_inte_test.cpp
+++ b/test/inte/scan_and_read_inte_test.cpp
@@ -2177,8 +2177,7 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) {
arrow::field("_VALUE_KIND", arrow::int8()));
auto schema = arrow::schema(fields);
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-2"},
{Options::FILE_SYSTEM, "local"}};
diff --git a/test/inte/variant_table_inte_test.cpp b/test/inte/variant_table_inte_test.cpp
index ba41c59c..14f05899 100644
--- a/test/inte/variant_table_inte_test.cpp
+++ b/test/inte/variant_table_inte_test.cpp
@@ -179,7 +179,6 @@ class VariantTableInteTest : public ::testing::Test {
double admission_ratio,
double retention_ratio) const {
return {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, "parquet"},
{Options::BUCKET, "-1"},
{Options::WRITE_ONLY, "true"},
@@ -206,7 +205,6 @@ class VariantTableInteTest : public ::testing::Test {
TEST_F(VariantTableInteTest, TestAppendTable) {
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, "parquet"},
{Options::BUCKET, "-1"},
};
@@ -597,7 +595,6 @@ TEST_F(VariantTableInteTest, TestAdaptiveInferenceWithNestedVariant) {
TEST_F(VariantTableInteTest, TestPrimaryKeyTable) {
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, "parquet"},
{Options::BUCKET, "1"},
};
@@ -623,7 +620,6 @@ TEST_F(VariantTableInteTest, TestPrimaryKeyTable) {
TEST_F(VariantTableInteTest, TestVariantAccessRead) {
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, "parquet"},
{Options::BUCKET, "-1"},
};
@@ -667,7 +663,6 @@ TEST_F(VariantTableInteTest, TestNestedRowVariantAccessRead) {
arrow::field("t", arrow::utf8())}));
auto table_schema = arrow::schema({fields_[0], struct_field});
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, "parquet"},
{Options::BUCKET, "-1"},
};
@@ -730,7 +725,6 @@ TEST_F(VariantTableInteTest, TestArrayVariantAccessRead) {
auto list_field = arrow::field("arr", arrow::list(VariantTypeUtils::ToArrowField("element")));
auto table_schema = arrow::schema({fields_[0], list_field});
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, "parquet"},
{Options::BUCKET, "-1"},
};
@@ -788,7 +782,6 @@ TEST_F(VariantTableInteTest, TestReadWithIOException) {
// Injects an IO error at every position of the scan+read path and verifies each failure
// surfaces as a clean error status.
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, "parquet"},
{Options::BUCKET, "-1"},
};
@@ -826,7 +819,6 @@ TEST_F(VariantTableInteTest, TestWriteWithIOException) {
// Injects an IO error at every position of the create+write+commit path (on a fresh table
// directory per attempt) and verifies each failure surfaces as a clean error status.
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, "parquet"},
{Options::BUCKET, "-1"},
};
@@ -862,7 +854,6 @@ TEST_F(VariantTableInteTest, TestWriteWithIOException) {
TEST_F(VariantTableInteTest, TestOrcFormatRejected) {
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, "orc"},
{Options::BUCKET, "-1"},
};
diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp
index 72bc3765..43d616e8 100644
--- a/test/inte/write_and_read_inte_test.cpp
+++ b/test/inte/write_and_read_inte_test.cpp
@@ -313,9 +313,8 @@ TEST_P(WriteAndReadInteTest, TestAppendSimple) {
arrow::field("f1", arrow::int32())};
auto schema = arrow::schema(fields);
auto [file_format, file_system] = GetParam();
- // manifest and file format are upper case
+ // file format is upper case
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "AVRO"},
{Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format)},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -368,8 +367,9 @@ TEST_P(WriteAndReadInteTest, TestAppendVector) {
arrow::FieldVector fields = {arrow::field("id", arrow::int32()),
arrow::field("embedding", vector_type)};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "-1"},
{Options::FILE_SYSTEM, file_system},
};
if (file_system == "jindo") {
@@ -427,8 +427,9 @@ TEST_P(WriteAndReadInteTest, TestAppendNestedVector) {
arrow::field("by_name", arrow::map(arrow::utf8(), vector_type)),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "-1"},
{Options::FILE_SYSTEM, file_system},
};
if (file_system == "jindo") {
@@ -476,7 +477,6 @@ TEST_P(WriteAndReadInteTest, TestAppendVectorWithPredicate) {
arrow::FieldVector fields = {arrow::field("id", arrow::int32()),
arrow::field("embedding", vector_type)};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1048576"},
{Options::BUCKET, "-1"},
@@ -540,7 +540,6 @@ TEST_P(WriteAndReadInteTest, TestAppendWithExternalBitmapAndRangeBitmapIndexes)
arrow::field("score", arrow::int32())};
auto [file_format, file_system] = GetParam();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1MB"},
{Options::BUCKET, "-1"},
@@ -626,9 +625,11 @@ TEST_P(WriteAndReadInteTest, TestPKSimple) {
auto schema = arrow::schema(fields);
auto [file_format, file_system] = GetParam();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
- {Options::FILE_SYSTEM, file_system}, {"orc.read.enable-lazy-decoding", "true"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "1"},
+ {Options::FILE_SYSTEM, file_system},
+ {"orc.read.enable-lazy-decoding", "true"},
{"orc.dictionary-key-size-threshold", "1"},
};
if (file_system == "jindo") {
@@ -685,9 +686,11 @@ TEST_P(WriteAndReadInteTest, TestInputChangelogStreamRead) {
};
auto [file_format, file_system] = GetParam();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
- {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "input"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "1"},
+ {Options::FILE_SYSTEM, file_system},
+ {Options::CHANGELOG_PRODUCER, "input"},
};
if (file_system == "jindo") {
options = AddOptionsForJindo(options);
@@ -731,10 +734,11 @@ TEST_P(WriteAndReadInteTest, TestFullCompactionChangelogStreamRead) {
auto [file_format, file_system] = GetParam();
arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()),
arrow::field("value", arrow::int32())};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
- {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "full-compaction"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "1"},
+ {Options::FILE_SYSTEM, file_system},
+ {Options::CHANGELOG_PRODUCER, "full-compaction"}};
if (file_system == "jindo") {
options = AddOptionsForJindo(options);
}
@@ -793,10 +797,11 @@ TEST_P(WriteAndReadInteTest, TestFullCompactionChangelogInitialScanOnlyReadsMaxL
auto [file_format, file_system] = GetParam();
arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()),
arrow::field("value", arrow::int32())};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
- {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "full-compaction"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "1"},
+ {Options::FILE_SYSTEM, file_system},
+ {Options::CHANGELOG_PRODUCER, "full-compaction"}};
if (file_system == "jindo") {
options = AddOptionsForJindo(options);
}
@@ -846,7 +851,6 @@ TEST_P(WriteAndReadInteTest, TestFullCompactionChangelogRowDeduplicate) {
arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()),
arrow::field("value", arrow::int32())};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "1"},
@@ -907,7 +911,6 @@ TEST_P(WriteAndReadInteTest, TestFullCompactionChangelogWithSharedShredding) {
arrow::FieldVector fields = {arrow::field("pk", arrow::int32()),
arrow::field("tags", map_type)};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "1"},
@@ -979,9 +982,11 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogStreamRead) {
arrow::field("value", arrow::int32()),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
- {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "1"},
+ {Options::FILE_SYSTEM, file_system},
+ {Options::CHANGELOG_PRODUCER, "lookup"},
};
ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options));
@@ -1028,9 +1033,11 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogInitialFullScanExcludesLevelZero
arrow::field("value", arrow::int32()),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
- {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "1"},
+ {Options::FILE_SYSTEM, file_system},
+ {Options::CHANGELOG_PRODUCER, "lookup"},
};
ASSERT_OK_AND_ASSIGN(auto helper, CreateLookupTestHelper(arrow::schema(fields), options));
@@ -1086,10 +1093,11 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogInsertUpdateDelete) {
auto [file_format, file_system] = GetParam();
arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()),
arrow::field("value", arrow::int32())};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
- {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "1"},
+ {Options::FILE_SYSTEM, file_system},
+ {Options::CHANGELOG_PRODUCER, "lookup"}};
if (file_system == "jindo") {
options = AddOptionsForJindo(options);
}
@@ -1134,11 +1142,12 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogWithFirstRow) {
auto [file_format, file_system] = GetParam();
arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()),
arrow::field("value", arrow::int32())};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
- {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"},
- {Options::MERGE_ENGINE, "first-row"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "1"},
+ {Options::FILE_SYSTEM, file_system},
+ {Options::CHANGELOG_PRODUCER, "lookup"},
+ {Options::MERGE_ENGINE, "first-row"}};
if (file_system == "jindo") {
options = AddOptionsForJindo(options);
}
@@ -1177,11 +1186,12 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogWithDeletionVector) {
auto [file_format, file_system] = GetParam();
arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()),
arrow::field("value", arrow::int32())};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
- {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"},
- {Options::DELETION_VECTORS_ENABLED, "true"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "1"},
+ {Options::FILE_SYSTEM, file_system},
+ {Options::CHANGELOG_PRODUCER, "lookup"},
+ {Options::DELETION_VECTORS_ENABLED, "true"}};
if (file_system == "jindo") {
options = AddOptionsForJindo(options);
}
@@ -1228,7 +1238,6 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogRowDeduplicate) {
arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()),
arrow::field("value", arrow::int32())};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "1"},
@@ -1286,7 +1295,6 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogRowDeduplicateIgnoreFields) {
arrow::field("value", arrow::int32()),
arrow::field("ignored", arrow::int32())};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "1"},
@@ -1346,10 +1354,11 @@ TEST_P(WriteAndReadInteTest, TestChangelogWithSchemaEvolution) {
auto [file_format, file_system] = GetParam();
arrow::FieldVector fields_v0 = {arrow::field("pk", arrow::utf8()),
arrow::field("value", arrow::int32())};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
- {Options::FILE_SYSTEM, file_system}, {Options::CHANGELOG_PRODUCER, "lookup"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "1"},
+ {Options::FILE_SYSTEM, file_system},
+ {Options::CHANGELOG_PRODUCER, "lookup"}};
if (file_system == "jindo") {
options = AddOptionsForJindo(options);
}
@@ -1457,7 +1466,6 @@ TEST_P(WriteAndReadInteTest, TestLookupChangelogWithExternalPath) {
arrow::FieldVector fields = {arrow::field("pk", arrow::utf8()),
arrow::field("value", arrow::int32())};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "1"},
@@ -1526,8 +1534,9 @@ TEST_P(WriteAndReadInteTest, TestNestedType) {
return;
}
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "-1"},
{Options::FILE_SYSTEM, file_system},
};
if (file_system == "jindo") {
@@ -1586,7 +1595,6 @@ TEST_P(WriteAndReadInteTest, TestSchemaEvolutionAddFieldInsideListAndMap) {
arrow::field("props", arrow::map(arrow::utf8(), map_value)),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -1863,10 +1871,11 @@ TEST_P(WriteAndReadInteTest, TestAppendTimestampType) {
if (file_format == "mosaic") {
return;
}
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"},
- {Options::FILE_SYSTEM, file_system}, {"orc.timestamp-ltz.legacy.type", "false"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "-1"},
+ {Options::FILE_SYSTEM, file_system},
+ {"orc.timestamp-ltz.legacy.type", "false"}};
if (file_system == "jindo") {
options = AddOptionsForJindo(options);
}
@@ -1919,10 +1928,11 @@ TEST_P(WriteAndReadInteTest, TestPkTimestampType) {
if (file_format == "mosaic") {
return;
}
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
- {Options::FILE_SYSTEM, file_system}, {"orc.timestamp-ltz.legacy.type", "false"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "1"},
+ {Options::FILE_SYSTEM, file_system},
+ {"orc.timestamp-ltz.legacy.type", "false"}};
if (file_system == "jindo") {
options = AddOptionsForJindo(options);
}
@@ -1979,10 +1989,11 @@ TEST_P(WriteAndReadInteTest, TestAppendNestedTimestampSecondPrecision) {
arrow::field("events", arrow::list(arrow::field("element", event_type))),
arrow::field("marks", arrow::map(arrow::utf8(), arrow::timestamp(arrow::TimeUnit::SECOND))),
};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"},
- {Options::FILE_SYSTEM, file_system}, {"orc.timestamp-ltz.legacy.type", "false"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "-1"},
+ {Options::FILE_SYSTEM, file_system},
+ {"orc.timestamp-ltz.legacy.type", "false"}};
if (file_system == "jindo") {
options = AddOptionsForJindo(options);
}
@@ -2038,10 +2049,11 @@ TEST_P(WriteAndReadInteTest, TestAppendNestedTimestampLtzMicroTimezoneOnly) {
arrow::field("marks",
arrow::map(arrow::utf8(), arrow::timestamp(arrow::TimeUnit::MICRO, timezone))),
};
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"},
- {Options::FILE_SYSTEM, file_system}, {"orc.timestamp-ltz.legacy.type", "false"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "-1"},
+ {Options::FILE_SYSTEM, file_system},
+ {"orc.timestamp-ltz.legacy.type", "false"}};
if (file_system == "jindo") {
options = AddOptionsForJindo(options);
}
@@ -2085,10 +2097,13 @@ TEST_P(WriteAndReadInteTest, TestPKWithSequenceFieldInPKField) {
auto schema = arrow::schema(fields);
auto [file_format, file_system] = GetParam();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
- {Options::FILE_SYSTEM, file_system}, {Options::SEQUENCE_FIELD, "p2"},
- {"orc.read.enable-lazy-decoding", "true"}, {"orc.dictionary-key-size-threshold", "1"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "1"},
+ {Options::FILE_SYSTEM, file_system},
+ {Options::SEQUENCE_FIELD, "p2"},
+ {"orc.read.enable-lazy-decoding", "true"},
+ {"orc.dictionary-key-size-threshold", "1"},
};
if (file_system == "jindo") {
options = AddOptionsForJindo(options);
@@ -2150,10 +2165,13 @@ TEST_P(WriteAndReadInteTest, TestPKWithSequenceFieldPartialInPKField) {
auto schema = arrow::schema(fields);
auto [file_format, file_system] = GetParam();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
- {Options::FILE_SYSTEM, file_system}, {Options::SEQUENCE_FIELD, "p2,f1"},
- {"orc.read.enable-lazy-decoding", "true"}, {"orc.dictionary-key-size-threshold", "1"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "1"},
+ {Options::FILE_SYSTEM, file_system},
+ {Options::SEQUENCE_FIELD, "p2,f1"},
+ {"orc.read.enable-lazy-decoding", "true"},
+ {"orc.dictionary-key-size-threshold", "1"},
};
if (file_system == "jindo") {
options = AddOptionsForJindo(options);
@@ -2404,8 +2422,9 @@ TEST_P(WriteAndReadInteTest, TestCharVarcharBinaryVarbinaryTypes) {
arrow::field("vb", arrow::binary()),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "-1"},
{Options::FILE_SYSTEM, file_system},
};
if (file_system == "jindo") {
@@ -2484,7 +2503,6 @@ TEST_P(WriteAndReadInteTest, TestPKWithParquetPageIndexFilter) {
arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())};
auto schema = arrow::schema(fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, "parquet"},
{Options::TARGET_FILE_SIZE, "1048576"},
{Options::BUCKET, "1"},
@@ -2500,7 +2518,7 @@ TEST_P(WriteAndReadInteTest, TestPKWithParquetPageIndexFilter) {
// filter is enabled below).
{Options::WRITE_BATCH_SIZE, "1"},
{"parquet.page.size", "1"},
- {"parquet.enable-dictionary", "false"},
+ {"parquet.enable.dictionary", "false"},
{"parquet.write.enable-page-index", "true"},
};
ASSERT_OK_AND_ASSIGN(auto helper,
@@ -2600,7 +2618,6 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) {
arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())};
auto schema = arrow::schema(fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, "parquet"},
{Options::TARGET_FILE_SIZE, "1048576"},
{Options::BUCKET, "-1"},
@@ -2611,7 +2628,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) {
// without row-level filter the reader output is precisely that one row.
{Options::WRITE_BATCH_SIZE, "1"},
{"parquet.page.size", "1"},
- {"parquet.enable-dictionary", "false"},
+ {"parquet.enable.dictionary", "false"},
{"parquet.write.enable-page-index", "true"},
};
ASSERT_OK_AND_ASSIGN(auto helper,
@@ -2708,7 +2725,6 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) {
arrow::field("f1", arrow::utf8())};
auto schema = arrow::schema(fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, "parquet"},
{Options::TARGET_FILE_SIZE, "1048576"},
{Options::BUCKET, "-1"},
@@ -2718,7 +2734,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) {
// in 4 row groups of 4 single-row pages.
{Options::WRITE_BATCH_SIZE, "1"},
{"parquet.page.size", "1"},
- {"parquet.enable-dictionary", "false"},
+ {"parquet.enable.dictionary", "false"},
{"parquet.write.enable-page-index", "true"},
{"parquet.write.max-row-group-length", "4"},
{"parquet.read.enable-page-index-filter", "true"},
@@ -2796,8 +2812,9 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) {
arrow::field("f1", arrow::int32())};
auto schema = arrow::schema(fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "parquet"},
- {Options::TARGET_FILE_SIZE, "1048576"}, {Options::BUCKET, "-1"},
+ {Options::FILE_FORMAT, "parquet"},
+ {Options::TARGET_FILE_SIZE, "1048576"},
+ {Options::BUCKET, "-1"},
{Options::FILE_SYSTEM, "local"},
};
ASSERT_OK_AND_ASSIGN(
@@ -2886,7 +2903,6 @@ TEST_P(WriteAndReadInteTest, TestAppendSharedShreddingMap) {
};
auto schema = arrow::schema(fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -2945,7 +2961,6 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingColumnPlacementPolicies) {
arrow::field("lru_metrics", map_type),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -3028,7 +3043,6 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPartitionAndBucket)
};
auto schema = arrow::schema(fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "2"},
@@ -3135,14 +3149,13 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) {
};
auto schema = arrow::schema(fields);
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1048576"},
{Options::BUCKET, "-1"},
{Options::FILE_SYSTEM, file_system},
{Options::WRITE_BATCH_SIZE, "1"},
{"parquet.page.size", "1"},
- {"parquet.enable-dictionary", "false"},
+ {"parquet.enable.dictionary", "false"},
{"parquet.write.enable-page-index", "true"},
{"parquet.write.max-row-group-length", "1"},
{"parquet.read.enable-page-index-filter", "true"},
@@ -3229,7 +3242,6 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingNewWriterStartsWithMaxColumnC
arrow::field("metrics", map_type),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "1"},
@@ -3298,7 +3310,6 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingAdaptsAcrossRollingFiles) {
arrow::field("metrics", map_type),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1048576"},
{Options::TARGET_FILE_ROW_NUM, "2"},
@@ -3372,7 +3383,6 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingSwitchMapLayoutAndUseMaxColum
arrow::field("labels", map_type),
};
std::map<std::string, std::string> options_v0 = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "1"},
@@ -3454,7 +3464,6 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingReadAfterRenameColumn) {
arrow::field("metrics", map_type),
};
std::map<std::string, std::string> options_v0 = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -3539,7 +3548,6 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingWithSchemaEvolution) {
arrow::field("k1", arrow::utf8()),
};
std::map<std::string, std::string> options_v0 = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -3648,9 +3656,11 @@ TEST_P(WriteAndReadInteTest, TestMapStorageLayoutDefaultToSharedShredding) {
arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())),
};
std::map<std::string, std::string> options_v0 = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"},
- {Options::FILE_SYSTEM, file_system}, {"fields.tags.map.storage-layout", "default"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "-1"},
+ {Options::FILE_SYSTEM, file_system},
+ {"fields.tags.map.storage-layout", "default"},
};
if (file_system == "jindo") {
options_v0 = AddOptionsForJindo(options_v0);
@@ -3726,7 +3736,6 @@ TEST_P(WriteAndReadInteTest, TestMapStorageLayoutSharedShreddingToDefault) {
arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())),
};
std::map<std::string, std::string> options_v0 = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -3790,7 +3799,6 @@ TEST_P(WriteAndReadInteTest, TestAppendMapStorageLayoutSharedShreddingToDefaultC
arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())),
};
std::map<std::string, std::string> options_v0 = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "1"},
@@ -3882,7 +3890,6 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingWithStructValue) {
arrow::field("tags", arrow::map(arrow::utf8(), value_type)),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -3955,7 +3962,6 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingWithComplexValue) {
arrow::field("tags", map_type),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -4094,7 +4100,6 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingWithAllSupportedComplexValueT
arrow::field("metrics", map_type),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -4222,7 +4227,6 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingStructValueSchemaEvolutionRea
arrow::field("profile", profile_type),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -4340,10 +4344,13 @@ TEST_P(WriteAndReadInteTest, TestOrcDictionaryLazyDecodingWithSharedShredding) {
arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"},
- {Options::FILE_SYSTEM, file_system}, {"fields.tags.map.storage-layout", "default"},
- {"orc.read.enable-lazy-decoding", "true"}, {"orc.dictionary-key-size-threshold", "1"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "-1"},
+ {Options::FILE_SYSTEM, file_system},
+ {"fields.tags.map.storage-layout", "default"},
+ {"orc.read.enable-lazy-decoding", "true"},
+ {"orc.dictionary-key-size-threshold", "1"},
};
if (file_system == "jindo") {
options = AddOptionsForJindo(options);
@@ -4426,7 +4433,6 @@ TEST_P(WriteAndReadInteTest, TestPkSharedShreddingMap) {
arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "1"},
@@ -4497,7 +4503,6 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallWithOverflow) {
arrow::field("tags", map_type),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -4632,7 +4637,6 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallWithNullOrMissin
arrow::field("tags", map_type),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -4744,7 +4748,6 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallMultipleColumns)
arrow::field("metrics", map_type),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -4855,9 +4858,11 @@ TEST_P(WriteAndReadInteTest, TestMapStorageLayoutDefaultToSharedShreddingPartial
arrow::field("tags", map_type),
};
std::map<std::string, std::string> options_v0 = {
- {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"},
- {Options::FILE_SYSTEM, file_system}, {"fields.tags.map.storage-layout", "default"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "-1"},
+ {Options::FILE_SYSTEM, file_system},
+ {"fields.tags.map.storage-layout", "default"},
};
if (file_system == "jindo") {
options_v0 = AddOptionsForJindo(options_v0);
@@ -4945,7 +4950,6 @@ TEST_P(WriteAndReadInteTest, TestMapStorageLayoutSharedShreddingToDefaultPartial
arrow::field("tags", map_type),
};
std::map<std::string, std::string> options_v0 = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -5040,7 +5044,6 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingDuplicateSelectedKeys) {
arrow::field("tags", map_type),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
@@ -5088,7 +5091,6 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingAllNullMapColumn) {
arrow::field("tags", map_type),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp
index 63768416..27bcea67 100644
--- a/test/inte/write_inte_test.cpp
+++ b/test/inte/write_inte_test.cpp
@@ -409,8 +409,7 @@ TEST_P(WriteInteTest, TestAppendTableBatchWrite) {
auto schema = arrow::schema(fields);
auto file_format = GetParam();
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
{Options::FILE_SYSTEM, "local"}};
@@ -514,9 +513,11 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithOneBucket) {
auto file_format = GetParam();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
- {Options::BUCKET_KEY, "f5"}, {Options::FILE_SYSTEM, "local"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "1"},
+ {Options::BUCKET_KEY, "f5"},
+ {Options::FILE_SYSTEM, "local"},
};
ASSERT_OK_AND_ASSIGN(
@@ -668,9 +669,11 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithPartitionAndMultiBuckets) {
std::vector<std::string> partition_keys = {"f2", "f1"};
auto file_format = GetParam();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"},
- {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "2"},
+ {Options::BUCKET_KEY, "f0"},
+ {Options::FILE_SYSTEM, "local"},
};
ASSERT_OK_AND_ASSIGN(
auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options,
@@ -824,9 +827,11 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithComplexType) {
auto file_format = GetParam();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
- {Options::BUCKET_KEY, "f5"}, {Options::FILE_SYSTEM, "local"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "1"},
+ {Options::BUCKET_KEY, "f5"},
+ {Options::FILE_SYSTEM, "local"},
};
ASSERT_OK_AND_ASSIGN(
auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{},
@@ -974,9 +979,11 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) {
std::vector<std::string> partition_keys = {"f1"};
auto file_format = GetParam();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"},
- {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "2"},
+ {Options::BUCKET_KEY, "f0"},
+ {Options::FILE_SYSTEM, "local"},
};
ASSERT_OK_AND_ASSIGN(
auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options,
@@ -1250,9 +1257,11 @@ TEST_P(WriteInteTest, TestPkTableBatchWrite) {
std::vector<std::string> partition_keys = {"f1"};
auto file_format = GetParam();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"},
- {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "2"},
+ {Options::BUCKET_KEY, "f0"},
+ {Options::FILE_SYSTEM, "local"},
};
ASSERT_OK_AND_ASSIGN(
auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options,
@@ -1411,9 +1420,11 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) {
std::vector<std::string> partition_keys = {};
auto file_format = GetParam();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"},
- {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "2"},
+ {Options::BUCKET_KEY, "f0"},
+ {Options::FILE_SYSTEM, "local"},
};
ASSERT_OK_AND_ASSIGN(
auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options,
@@ -1650,9 +1661,11 @@ TEST_P(WriteInteTest, TestPkTableWriteWithComplexType) {
std::vector<std::string> partition_keys = {};
auto file_format = GetParam();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
- {Options::BUCKET_KEY, "f5"}, {Options::FILE_SYSTEM, "local"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "1"},
+ {Options::BUCKET_KEY, "f5"},
+ {Options::FILE_SYSTEM, "local"},
};
ASSERT_OK_AND_ASSIGN(
auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options,
@@ -1821,11 +1834,13 @@ TEST_P(WriteInteTest, TestPkTableForceLookup) {
std::vector<std::string> primary_keys = {"f0", "f1"};
std::vector<std::string> partition_keys = {};
auto file_format = GetParam();
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "1"},
- {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"},
- {Options::FORCE_LOOKUP, "true"}, {Options::WRITE_ONLY, "true"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "1"},
+ {Options::BUCKET_KEY, "f0"},
+ {Options::FILE_SYSTEM, "local"},
+ {Options::FORCE_LOOKUP, "true"},
+ {Options::WRITE_ONLY, "true"}};
ASSERT_OK_AND_ASSIGN(
auto helper, TestHelper::Create(dir->Str(), schema, partition_keys, primary_keys, options,
/*is_streaming_mode=*/true));
@@ -1883,8 +1898,7 @@ TEST_P(WriteInteTest, TestPkTableEnableDeletionVector) {
std::vector<std::string> primary_keys = {"f0", "f1"};
std::vector<std::string> partition_keys = {};
auto file_format = GetParam();
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "1"},
{Options::BUCKET_KEY, "f0"},
@@ -1943,9 +1957,11 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) {
std::vector<std::string> primary_keys = {"f0", "f1"};
std::vector<std::string> partition_keys = {"f1"};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"},
- {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "2"},
+ {Options::BUCKET_KEY, "f0"},
+ {Options::FILE_SYSTEM, "local"},
};
bool run_complete = false;
auto io_hook = IOHook::GetInstance();
@@ -2235,7 +2251,7 @@ TEST_F(WriteInteTest, TestAppendTableWriteWithAlterTable) {
arrow::field("e", arrow::int32()),
};
std::map<std::string, std::string> options = {{Options::FILE_FORMAT, "orc"},
- {Options::MANIFEST_FORMAT, "orc"},
+
{Options::TARGET_FILE_SIZE, "1024"},
{Options::FILE_SYSTEM, "local"}};
ASSERT_OK_AND_ASSIGN(auto helper,
@@ -2314,7 +2330,7 @@ TEST_F(WriteInteTest, TestPKTableWriteWithAlterTable) {
arrow::field("v2", arrow::int32()),
};
std::map<std::string, std::string> options = {{Options::FILE_FORMAT, "orc"},
- {Options::MANIFEST_FORMAT, "orc"},
+
{Options::TARGET_FILE_SIZE, "1024"},
{Options::FILE_SYSTEM, "local"}};
ASSERT_OK_AND_ASSIGN(auto helper,
@@ -2409,7 +2425,7 @@ TEST_P(WriteInteTest, TestWriteAndCommitIOException) {
auto file_format = GetParam();
std::map<std::string, std::string> options = {
{Options::FILE_FORMAT, file_format},
- {Options::MANIFEST_FORMAT, "orc"},
+
{Options::TARGET_FILE_SIZE, "1024"},
{Options::FILE_SYSTEM, "local"},
{Options::BUCKET, "2"},
@@ -2513,7 +2529,6 @@ TEST_P(WriteInteTest, TestWriteWithFieldId) {
::ArrowSchema c_schema;
ASSERT_TRUE(arrow::ExportType(*arrow_data_type, &c_schema).ok());
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::FILE_SYSTEM, "local"},
@@ -2745,7 +2760,6 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithExternalPath) {
auto file_format = GetParam();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "1"},
@@ -3179,7 +3193,7 @@ TEST_P(WriteInteTest, TestWriteWithIOException) {
auto file_format = GetParam();
std::map<std::string, std::string> options = {
{Options::FILE_FORMAT, file_format},
- {Options::MANIFEST_FORMAT, "orc"},
+
{Options::TARGET_FILE_SIZE, "1024"},
{Options::FILE_SYSTEM, "local"},
{Options::BUCKET, "2"},
@@ -3239,7 +3253,7 @@ TEST_P(WriteInteTest, TestCommitWithIOException) {
auto file_format = GetParam();
std::map<std::string, std::string> options = {
{Options::FILE_FORMAT, file_format},
- {Options::MANIFEST_FORMAT, "orc"},
+
{Options::TARGET_FILE_SIZE, "1024"},
{Options::FILE_SYSTEM, "local"},
{Options::BUCKET, "2"},
@@ -3311,7 +3325,7 @@ TEST_P(WriteInteTest, TestWriteMemoryUse) {
auto file_format = GetParam();
std::map<std::string, std::string> options = {
{Options::FILE_FORMAT, file_format},
- {Options::MANIFEST_FORMAT, "orc"},
+
{Options::TARGET_FILE_SIZE, "1024"},
{Options::FILE_SYSTEM, "local"},
{Options::BUCKET, "2"},
@@ -3377,8 +3391,9 @@ TEST_P(WriteInteTest, TestAppendTableWithAllNull) {
auto file_format = GetParam();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "-1"},
{Options::FILE_SYSTEM, "local"},
};
@@ -3417,8 +3432,7 @@ TEST_P(WriteInteTest, TestPkTablePostponeBucket) {
auto schema = arrow::schema(fields);
std::vector<std::string> primary_keys = {"f0", "f1"};
auto file_format = GetParam();
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-2"},
{Options::FILE_SYSTEM, "local"}};
@@ -3512,7 +3526,7 @@ TEST_F(WriteInteTest, TestBranchWrite) {
ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, table_path));
std::map<std::string, std::string> options = {{Options::FILE_FORMAT, "orc"},
- {Options::MANIFEST_FORMAT, "orc"},
+
{Options::FILE_SYSTEM, "local"}};
WriteContextBuilder context_builder(table_path, "commit_user_1");
ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context,
@@ -3613,7 +3627,7 @@ TEST_P(WriteInteTest, TestDataEvolutionWrite) {
auto file_format = GetParam();
auto dir = UniqueTestDirectory::Create();
std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format},
- {Options::MANIFEST_FORMAT, "orc"},
+
{Options::ROW_TRACKING_ENABLED, "true"},
{Options::DATA_EVOLUTION_ENABLED, "true"},
{Options::FILE_SYSTEM, "local"}};
@@ -3804,11 +3818,13 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) {
auto schema = arrow::schema(fields);
auto file_format = GetParam();
- std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"},
- {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
- {Options::FILE_SYSTEM, "local"}, {Options::BLOB_AS_DESCRIPTOR, "true"}};
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "700"},
+ {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"},
+ {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::FILE_SYSTEM, "local"},
+ {Options::BLOB_AS_DESCRIPTOR, "true"}};
ASSERT_OK_AND_ASSIGN(
auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{},
@@ -3916,8 +3932,7 @@ TEST_P(WriteInteTest, TestAppendTableWithDateFieldAsPartitionField) {
auto schema = arrow::schema(fields);
auto file_format = GetParam();
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format},
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT, file_format},
{Options::TARGET_FILE_SIZE, "1024"},
{Options::BUCKET, "-1"},
{Options::FILE_SYSTEM, "local"}};
@@ -4039,7 +4054,6 @@ TEST_P(WriteInteTest, TestPkSpillableMapSharedShreddingReadWrite) {
arrow::field("metrics", map_type),
};
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "avro"},
{Options::FILE_FORMAT, file_format},
{Options::BUCKET, "1"},
{Options::BUCKET_KEY, "id"},
@@ -4572,10 +4586,13 @@ TEST_P(WriteInteTest, TestPkSpillableWithIOException) {
std::vector<std::string> partition_keys = {"f1"};
auto file_format = GetParam();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format},
- {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"},
- {Options::BUCKET_KEY, "f0"}, {Options::FILE_SYSTEM, "local"},
- {Options::WRITE_BUFFER_SIZE, "1"}, {Options::WRITE_BUFFER_SPILLABLE, "true"},
+ {Options::FILE_FORMAT, file_format},
+ {Options::TARGET_FILE_SIZE, "1024"},
+ {Options::BUCKET, "2"},
+ {Options::BUCKET_KEY, "f0"},
+ {Options::FILE_SYSTEM, "local"},
+ {Options::WRITE_BUFFER_SIZE, "1"},
+ {Options::WRITE_BUFFER_SPILLABLE, "true"},
{Options::WRITE_ONLY, "true"},
};
bool run_complete = false;
@@ -4683,14 +4700,11 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) {
auto schema = arrow::schema(fields);
auto file_format = GetParam();
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format},
- {Options::BUCKET, "-1"},
- {Options::ROW_TRACKING_ENABLED, "true"},
- {Options::DATA_EVOLUTION_ENABLED, "true"},
- {Options::FILE_SYSTEM, "local"},
- {Options::BLOB_AS_DESCRIPTOR, "true"},
- {Options::BLOB_FIELD, "blob2,blob1"}};
+ std::map<std::string, std::string> options = {
+ {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::FILE_SYSTEM, "local"}, {Options::BLOB_AS_DESCRIPTOR, "true"},
+ {Options::BLOB_FIELD, "blob2,blob1"}};
ASSERT_OK_AND_ASSIGN(
auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{},
@@ -4828,7 +4842,6 @@ TEST_P(WriteInteTest, TestRowTrackingPartitionGroupOnCommit) {
auto file_format = GetParam();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, file_format},
{Options::BUCKET, "-1"},
{Options::ROW_TRACKING_ENABLED, "true"},
@@ -4943,7 +4956,6 @@ TEST_P(WriteInteTest, TestRowTrackingPartitionGroupOnCommitDisabled) {
auto file_format = GetParam();
std::map<std::string, std::string> options = {
- {Options::MANIFEST_FORMAT, "orc"},
{Options::FILE_FORMAT, file_format},
{Options::BUCKET, "-1"},
{Options::ROW_TRACKING_ENABLED, "true"},
@@ -5041,14 +5053,11 @@ TEST_P(WriteInteTest, TestMultipleBlobFieldsSplitByTargetSize) {
auto file_format = GetParam();
// Set a very small blob target file size to force splitting
- std::map<std::string, std::string> options = {{Options::MANIFEST_FORMAT, "orc"},
- {Options::FILE_FORMAT, file_format},
- {Options::BUCKET, "-1"},
- {Options::ROW_TRACKING_ENABLED, "true"},
- {Options::DATA_EVOLUTION_ENABLED, "true"},
- {Options::FILE_SYSTEM, "local"},
- {Options::BLOB_AS_DESCRIPTOR, "false"},
- {Options::BLOB_TARGET_FILE_SIZE, "1"}};
+ std::map<std::string, std::string> options = {
+ {Options::FILE_FORMAT, file_format}, {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::FILE_SYSTEM, "local"}, {Options::BLOB_AS_DESCRIPTOR, "false"},
+ {Options::BLOB_TARGET_FILE_SIZE, "1"}};
ASSERT_OK_AND_ASSIGN(
auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{},
diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-ed03e3fd-3ff4-4d88-9d5d-683700f78967-0 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-ed03e3fd-3ff4-4d88-9d5d-683700f78967-0
index 1a0ea1eb..427e6859 100644
Binary files a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-ed03e3fd-3ff4-4d88-9d5d-683700f78967-0 and b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-ed03e3fd-3ff4-4d88-9d5d-683700f78967-0 differ
diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-f2299c3d-c3f1-400f-ad3d-124e3a342389-0 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-f2299c3d-c3f1-400f-ad3d-124e3a342389-0
index 8f99c331..d8ca5da5 100644
Binary files a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-f2299c3d-c3f1-400f-ad3d-124e3a342389-0 and b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-f2299c3d-c3f1-400f-ad3d-124e3a342389-0 differ
diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-0 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-0
index eea939b2..f8d0e6d1 100644
Binary files a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-0 and b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-0 differ
diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-1 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-1
index fbf9cda3..d025ffe7 100644
Binary files a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-1 and b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-12b37c17-d02f-4409-8993-ac01a56210bf-1 differ
diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-0 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-0
index c7de3598..e4cfe141 100644
Binary files a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-0 and b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-0 differ
diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-1 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-1
index eea939b2..d12b180c 100644
Binary files a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-1 and b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/manifest/manifest-list-83964df9-8a98-4f91-a4e9-05f3e07be3f9-1 differ
diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-0 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-0
index a1bd3ded..b631fc3a 100644
--- a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-0
+++ b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-0
@@ -34,8 +34,8 @@
"partitionKeys" : [ "key0", "key1" ],
"primaryKeys" : [ ],
"options" : {
- "manifest.format" : "orc",
+ "manifest.format" : "avro",
"file.format" : "orc"
},
"timeMillis" : 1730430023755
-}
\ No newline at end of file
+}
diff --git a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-1 b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-1
index cb81068f..83b8cc9e 100644
--- a/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-1
+++ b/test/test_data/orc/append_table_with_alter_table.db/append_table_with_alter_table/schema/schema-1
@@ -35,8 +35,8 @@
"partitionKeys" : [ "key0", "key1" ],
"primaryKeys" : [ ],
"options" : {
- "manifest.format" : "orc",
+ "manifest.format" : "avro",
"file.format" : "orc"
},
"timeMillis" : 1730430297413
-}
\ No newline at end of file
+}
diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-0 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-0
index 55f74707..d9a092c8 100644
--- a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-0
+++ b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-0
@@ -18,8 +18,8 @@
"partitionKeys" : [ "dt" ],
"primaryKeys" : [ ],
"options" : {
- "manifest.format" : "orc",
+ "manifest.format" : "avro",
"file.format" : "orc"
},
"timeMillis" : 1744885904315
-}
\ No newline at end of file
+}
diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-1 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-1
index 9e0b0819..724aef67 100644
--- a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-1
+++ b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/branch/branch-rt/schema/schema-1
@@ -19,8 +19,8 @@
"primaryKeys" : [ "dt", "name" ],
"options" : {
"bucket" : "2",
- "manifest.format" : "orc",
+ "manifest.format" : "avro",
"file.format" : "orc"
},
"timeMillis" : 1744885904631
-}
\ No newline at end of file
+}
diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-6090bcc3-a5a6-4c51-bc5e-7af6498e5b99-0 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-6090bcc3-a5a6-4c51-bc5e-7af6498e5b99-0
index 3469a68b..4120d9e0 100644
Binary files a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-6090bcc3-a5a6-4c51-bc5e-7af6498e5b99-0 and b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-6090bcc3-a5a6-4c51-bc5e-7af6498e5b99-0 differ
diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-b204b605-41df-407f-85d9-722bef2754f4-0 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-b204b605-41df-407f-85d9-722bef2754f4-0
index 6e7a8b14..f7b10cd0 100644
Binary files a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-b204b605-41df-407f-85d9-722bef2754f4-0 and b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-b204b605-41df-407f-85d9-722bef2754f4-0 differ
diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-0 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-0
index c9c69601..5a21b96d 100644
Binary files a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-0 and b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-0 differ
diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-1 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-1
index 7c3e8f78..230e1e70 100644
Binary files a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-1 and b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-1c1bf05e-c1d0-4513-bc39-b2cb7e2d4973-1 differ
diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-0 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-0
index c9c69601..8b4e5a62 100644
Binary files a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-0 and b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-0 differ
diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-1 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-1
index 2d8d6479..2070099f 100644
Binary files a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-1 and b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/manifest/manifest-list-779da483-08f4-4271-8a62-d6576f56bc80-1 differ
diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-0 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-0
index 55f74707..d9a092c8 100644
--- a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-0
+++ b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-0
@@ -18,8 +18,8 @@
"partitionKeys" : [ "dt" ],
"primaryKeys" : [ ],
"options" : {
- "manifest.format" : "orc",
+ "manifest.format" : "avro",
"file.format" : "orc"
},
"timeMillis" : 1744885904315
-}
\ No newline at end of file
+}
diff --git a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-1 b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-1
index db5856dc..499fccc5 100644
--- a/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-1
+++ b/test/test_data/orc/append_table_with_rt_branch.db/append_table_with_rt_branch/schema/schema-1
@@ -18,9 +18,9 @@
"partitionKeys" : [ "dt" ],
"primaryKeys" : [ ],
"options" : {
- "manifest.format" : "orc",
+ "manifest.format" : "avro",
"file.format" : "orc",
"scan.fallback-branch" : "rt"
},
"timeMillis" : 1744885904624
-}
\ No newline at end of file
+}
diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-0 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-0
index e02a4bc7..3da1ad5d 100644
Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-0 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-0 differ
diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-1 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-1
index 8a2b9720..5dcc67f7 100644
Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-1 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-6f30e9a0-53e1-4030-bf82-1e9b1d62d43f-1 differ
diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-0 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-0
index 5eb55ba4..2897e9b7 100644
Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-0 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-0 differ
diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-1 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-1
index 1518ddb1..2ee0906d 100644
Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-1 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-bebb1b9f-3785-400b-8f58-8e6902f5016e-1 differ
diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-0 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-0
index 1641b8d3..34248ccc 100644
Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-0 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-0 differ
diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-1 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-1
index daea579f..ee59a318 100644
Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-1 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-1 differ
diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-2 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-2
index 42382569..07723ec4 100644
Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-2 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-2 differ
diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-3 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-3
index a69ce5f0..2b31bc2b 100644
Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-3 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-32868327-8690-4675-8593-f8c6ade64177-3 differ
diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-0 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-0
index 04ebc021..c6a9e75d 100644
Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-0 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-0 differ
diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-1 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-1
index 65664835..4b597576 100644
Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-1 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-1 differ
diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-2 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-2
index 65664835..3b778efd 100644
Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-2 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-2 differ
diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-3 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-3
index 04ebc021..995dcac9 100644
Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-3 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-3 differ
diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-4 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-4
index 65664835..0dbea592 100644
Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-4 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-4 differ
diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-5 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-5
index 0336fb8d..a47a14d7 100644
Binary files a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-5 and b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/manifest/manifest-list-d73b867c-362b-4128-b7cb-d3a7ffb7992c-5 differ
diff --git a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/schema/schema-0 b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/schema/schema-0
index 96dd5b13..bbc18eec 100644
--- a/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/schema/schema-0
+++ b/test/test_data/orc/pk_compact_lookup.db/pk_compact_lookup/schema/schema-0
@@ -32,9 +32,9 @@
"fields.default-aggregate-function" : "min",
"lookup.remote-file.level-threshold" : "1",
"merge-engine" : "aggregation",
- "manifest.format" : "orc",
+ "manifest.format" : "avro",
"file.format" : "orc",
"deletion-vectors.enabled" : "true"
},
"timeMillis" : 1775650042347
-}
\ No newline at end of file
+}
diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-0 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-0
index a95c97f7..22f5a4a0 100644
Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-0 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-0 differ
diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-1 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-1
index 295cfe0b..25e07f91 100644
Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-1 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-167e8d7d-d3ea-4e3b-81d5-fda85c774c29-1 differ
diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-daf4fd15-fe55-4216-8668-bac0b6a93781-0 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-daf4fd15-fe55-4216-8668-bac0b6a93781-0
index 4ad23b28..64916dfe 100644
Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-daf4fd15-fe55-4216-8668-bac0b6a93781-0 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-daf4fd15-fe55-4216-8668-bac0b6a93781-0 differ
diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-0 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-0
index 68eff159..76da03aa 100644
Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-0 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-0 differ
diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-1 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-1
index 2dddc469..d6d5f0a0 100644
Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-1 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-1 differ
diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-2 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-2
index 2dddc469..93e23c94 100644
Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-2 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-2 differ
diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-3 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-3
index 8a6e7ed4..58754933 100644
Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-3 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-4c6d8995-1be3-4d3b-acd0-d26d9bd7fbfd-3 differ
diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-0 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-0
index d2c7c01e..266ffa1a 100644
Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-0 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-0 differ
diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-1 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-1
index 65e68ab9..3a0a5211 100644
Binary files a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-1 and b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/manifest/manifest-list-c62379f7-6e8b-4a39-af6b-1ec7baecf2e9-1 differ
diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-0 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-0
index 277d45f2..82fd8df8 100644
--- a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-0
+++ b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-0
@@ -39,9 +39,9 @@
"primaryKeys" : [ "k0", "k1", "p0", "p1" ],
"options" : {
"bucket" : "1",
- "manifest.format" : "orc",
+ "manifest.format" : "avro",
"file.format" : "orc",
"sequence.field" : "s0,s1"
},
"timeMillis" : 1735120478343
-}
\ No newline at end of file
+}
diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-1 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-1
index 0bf55dbf..1d6e3cc8 100644
--- a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-1
+++ b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-1
@@ -39,9 +39,9 @@
"primaryKeys" : [ "k0", "tmp", "p0", "p1" ],
"options" : {
"bucket" : "1",
- "manifest.format" : "orc",
+ "manifest.format" : "avro",
"file.format" : "orc",
"sequence.field" : "s0,s1"
},
"timeMillis" : 1735207491195
-}
\ No newline at end of file
+}
diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-2 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-2
index 13db1668..bc157552 100644
--- a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-2
+++ b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-2
@@ -39,9 +39,9 @@
"primaryKeys" : [ "k1", "tmp", "p0", "p1" ],
"options" : {
"bucket" : "1",
- "manifest.format" : "orc",
+ "manifest.format" : "avro",
"file.format" : "orc",
"sequence.field" : "s0,s1"
},
"timeMillis" : 1735207514342
-}
\ No newline at end of file
+}
diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-3 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-3
index 8ff9465a..e1037a15 100644
--- a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-3
+++ b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-3
@@ -39,9 +39,9 @@
"primaryKeys" : [ "k1", "k0", "p0", "p1" ],
"options" : {
"bucket" : "1",
- "manifest.format" : "orc",
+ "manifest.format" : "avro",
"file.format" : "orc",
"sequence.field" : "s0,s1"
},
"timeMillis" : 1735207576625
-}
\ No newline at end of file
+}
diff --git a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-4 b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-4
index a7e52509..23f2e45c 100644
--- a/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-4
+++ b/test/test_data/orc/pk_table_with_mor.db/pk_table_with_mor/schema/schema-4
@@ -43,9 +43,9 @@
"primaryKeys" : [ "k1", "k0", "p0", "p1" ],
"options" : {
"bucket" : "1",
- "manifest.format" : "orc",
+ "manifest.format" : "avro",
"file.format" : "orc",
"sequence.field" : "s0,s1"
},
"timeMillis" : 1735207715028
-}
\ No newline at end of file
+}
diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-0
index fb7dcc92..0713a2fc 100644
Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-0 differ
diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-1 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-1
index 135345f6..0ff37d23 100644
Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-1 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-7008d4c0-b1b0-4237-ac6d-845b887efc92-1 differ
diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-97ddb443-4cd9-42e9-9871-4d51d4fa1b49-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-97ddb443-4cd9-42e9-9871-4d51d4fa1b49-0
index 0a519877..1317128e 100644
Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-97ddb443-4cd9-42e9-9871-4d51d4fa1b49-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-97ddb443-4cd9-42e9-9871-4d51d4fa1b49-0 differ
diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-eb34bdd2-2c23-49af-b5aa-537a596d8fe3-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-eb34bdd2-2c23-49af-b5aa-537a596d8fe3-0
index cffc81ea..15c7d991 100644
Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-eb34bdd2-2c23-49af-b5aa-537a596d8fe3-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-eb34bdd2-2c23-49af-b5aa-537a596d8fe3-0 differ
diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-ed67f349-7c6f-4f51-8723-274a060042d1-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-ed67f349-7c6f-4f51-8723-274a060042d1-0
index 0a6862a6..22f88c68 100644
Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-ed67f349-7c6f-4f51-8723-274a060042d1-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-ed67f349-7c6f-4f51-8723-274a060042d1-0 differ
diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-0
index 0e0eb50e..89a934ca 100644
Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-0 differ
diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-1 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-1
index 22f85d5d..65ae8012 100644
Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-1 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-2663284f-7bac-441a-bfa1-d11bb24af95d-1 differ
diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-0
index c7de3598..722a9a06 100644
Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-0 differ
diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1
index 0e0eb50e..2077b919 100644
Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-55a3b658-08e3-40aa-b692-29dc1e3ebbdc-1 differ
diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-0
index 04d7af1a..f60aee5d 100644
Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-0 differ
diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-1 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-1
index a2042811..456e1eb2 100644
Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-1 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-1 differ
diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-2 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-2
index b07ea33d..e8d00699 100644
Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-2 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-2 differ
diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-3 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-3
index 7a034580..15f30ebb 100644
Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-3 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-739ea106-ac05-4953-a13f-c7260482dbf1-3 differ
diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-0 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-0
index 3023df2d..77010b30 100644
Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-0 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-0 differ
diff --git a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-1 b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-1
index 41ad2bdf..0097ed46 100644
Binary files a/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-1 and b/test/test_data/parquet/append_09.db/append_09/manifest/manifest-list-ba2c5ba9-2dfd-4d39-af49-9f88fa5c029f-1 differ
diff --git a/test/test_data/parquet/append_09.db/append_09/schema/schema-0 b/test/test_data/parquet/append_09.db/append_09/schema/schema-0
index 2314054c..b992da38 100644
--- a/test/test_data/parquet/append_09.db/append_09/schema/schema-0
+++ b/test/test_data/parquet/append_09.db/append_09/schema/schema-0
@@ -24,7 +24,7 @@
"options" : {
"bucket" : "2",
"bucket-key" : "f2",
- "manifest.format" : "orc",
+ "manifest.format" : "avro",
"file.format" : "parquet"
},
"timeMillis" : 1755758260435