You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@iceberg.apache.org by "wgtmac (via GitHub)" <gi...@apache.org> on 2025/09/04 15:29:29 UTC
[PR] feat: define table properties with default values [iceberg-cpp]
wgtmac opened a new pull request, #212:
URL: https://github.com/apache/iceberg-cpp/pull/212
Just copied everything from the TableProperties.java as of today
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: define table properties with default values [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #212:
URL: https://github.com/apache/iceberg-cpp/pull/212#discussion_r2391797073
##########
src/iceberg/table_properties.h:
##########
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <limits>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "iceberg/iceberg_export.h"
+#include "iceberg/util/config.h"
+
+namespace iceberg {
+
+/// \brief Table properties for Iceberg tables.
+///
+/// This class provides configuration entries for various Iceberg table properties
+/// including format settings, commit behavior, file formats, compression settings,
+/// and other table-level configurations.
+class ICEBERG_EXPORT TableProperties : public ConfigBase<TableProperties> {
+ public:
+ template <typename T>
+ using Entry = const ConfigBase<TableProperties>::Entry<T>;
+
+ // Reserved table properties
+
+ /// \brief Reserved table property for table format version.
+ ///
+ /// Iceberg will default a new table's format version to the latest stable and
+ /// recommended version. This reserved property keyword allows users to override the
+ /// Iceberg format version of the table metadata.
+ ///
+ /// If this table property exists when creating a table, the table will use the
+ /// specified format version. If a table updates this property, it will try to upgrade
+ /// to the specified format version.
+ ///
+ /// \note incomplete or unstable versions cannot be selected using this property.
+ inline static Entry<std::string> kFormatVersion{"format-version", ""};
+ /// \brief Reserved table property for table UUID.
+ inline static Entry<std::string> kUuid{"uuid", ""};
+ /// \brief Reserved table property for the total number of snapshots.
+ inline static Entry<std::string> kSnapshotCount{"snapshot-count", ""};
+ /// \brief Reserved table property for current snapshot summary.
+ inline static Entry<std::string> kCurrentSnapshotSummary{"current-snapshot-summary",
+ ""};
+ /// \brief Reserved table property for current snapshot id.
+ inline static Entry<std::string> kCurrentSnapshotId{"current-snapshot-id", ""};
+ /// \brief Reserved table property for current snapshot timestamp.
+ inline static Entry<std::string> kCurrentSnapshotTimestamp{
+ "current-snapshot-timestamp-ms", ""};
+ /// \brief Reserved table property for the JSON representation of current schema.
+ inline static Entry<std::string> kCurrentSchema{"current-schema", ""};
+ /// \brief Reserved table property for the JSON representation of current(default)
+ /// partition spec.
+ inline static Entry<std::string> kDefaultPartitionSpec{"default-partition-spec", ""};
+ /// \brief Reserved table property for the JSON representation of current(default) sort
+ /// order.
+ inline static Entry<std::string> kDefaultSortOrder{"default-sort-order", ""};
+
+ // Commit properties
+
+ inline static Entry<int32_t> kCommitNumRetries{"commit.retry.num-retries", 4};
+ inline static Entry<int32_t> kCommitMinRetryWaitMs{"commit.retry.min-wait-ms", 100};
+ inline static Entry<int32_t> kCommitMaxRetryWaitMs{"commit.retry.max-wait-ms",
+ 60 * 1000}; // 1 minute
+ inline static Entry<int32_t> kCommitTotalRetryTimeMs{"commit.retry.total-timeout-ms",
+ 30 * 60 * 1000}; // 30 minutes
+ inline static Entry<int32_t> kCommitNumStatusChecks{"commit.status-check.num-retries",
+ 3};
+ inline static Entry<int64_t> kCommitStatusChecksMinWaitMs{
+ "commit.status-check.min-wait-ms", int64_t{1000}}; // 1 second
+ inline static Entry<int64_t> kCommitStatusChecksMaxWaitMs{
+ "commit.status-check.max-wait-ms", int64_t{60 * 1000}}; // 1 minute
+ inline static Entry<int64_t> kCommitStatusChecksTotalWaitMs{
+ "commit.status-check.total-timeout-ms", int64_t{30 * 60 * 1000}}; // 30 minutes
+
+ // Manifest properties
+
+ inline static Entry<int64_t> kManifestTargetSizeBytes{
+ "commit.manifest.target-size-bytes", int64_t{8 * 1024 * 1024}}; // 8 MB
+ inline static Entry<int32_t> kManifestMinMergeCount{
+ "commit.manifest.min-count-to-merge", 100};
+ inline static Entry<bool> kManifestMergeEnabled{"commit.manifest-merge.enabled", true};
+
+ // File format properties
+
+ inline static Entry<std::string> kDefaultFileFormat{"write.format.default", "parquet"};
+ inline static Entry<std::string> kDeleteDefaultFileFormat{"write.delete.format.default",
+ "parquet"};
+
+ // Parquet properties
+
+ inline static Entry<int32_t> kParquetRowGroupSizeBytes{
+ "write.parquet.row-group-size-bytes", 128 * 1024 * 1024}; // 128 MB
+ inline static Entry<int32_t> kDeleteParquetRowGroupSizeBytes{
+ "write.delete.parquet.row-group-size-bytes", 128 * 1024 * 1024}; // 128 MB
+ inline static Entry<int32_t> kParquetPageSizeBytes{"write.parquet.page-size-bytes",
+ 1024 * 1024}; // 1 MB
+ inline static Entry<int32_t> kDeleteParquetPageSizeBytes{
+ "write.delete.parquet.page-size-bytes", 1024 * 1024}; // 1 MB
+ inline static Entry<int32_t> kParquetPageRowLimit{"write.parquet.page-row-limit",
+ 20'000};
+ inline static Entry<int32_t> kDeleteParquetPageRowLimit{
+ "write.delete.parquet.page-row-limit", 20'000};
+ inline static Entry<int32_t> kParquetDictSizeBytes{"write.parquet.dict-size-bytes",
+ 2 * 1024 * 1024}; // 2 MB
+ inline static Entry<int32_t> kDeleteParquetDictSizeBytes{
+ "write.delete.parquet.dict-size-bytes", 2 * 1024 * 1024}; // 2 MB
+ inline static Entry<std::string> kParquetCompression{"write.parquet.compression-codec",
+ "zstd"};
+ inline static Entry<std::string> kDeleteParquetCompression{
+ "write.delete.parquet.compression-codec", "zstd"};
+ inline static Entry<std::string> kParquetCompressionLevel{
+ "write.parquet.compression-level", ""};
+ inline static Entry<std::string> kDeleteParquetCompressionLevel{
+ "write.delete.parquet.compression-level", ""};
+ inline static Entry<int32_t> kParquetRowGroupCheckMinRecordCount{
+ "write.parquet.row-group-check-min-record-count", 100};
+ inline static Entry<int32_t> kDeleteParquetRowGroupCheckMinRecordCount{
+ "write.delete.parquet.row-group-check-min-record-count", 100};
+ inline static Entry<int32_t> kParquetRowGroupCheckMaxRecordCount{
+ "write.parquet.row-group-check-max-record-count", 10'000};
+ inline static Entry<int32_t> kDeleteParquetRowGroupCheckMaxRecordCount{
+ "write.delete.parquet.row-group-check-max-record-count", 10'000};
+ inline static Entry<int32_t> kParquetBloomFilterMaxBytes{
+ "write.parquet.bloom-filter-max-bytes", 1024 * 1024}; // 1 MB
+ inline static std::string_view kParquetBloomFilterColumnFppPrefix{
+ "write.parquet.bloom-filter-fpp.column."};
+ inline static std::string_view kParquetBloomFilterColumnEnabledPrefix{
+ "write.parquet.bloom-filter-enabled.column."};
+ inline static std::string_view kParquetColumnStatsEnabledPrefix{
+ "write.parquet.stats-enabled.column."};
+
+ // Avro properties
+ inline static Entry<std::string> kAvroCompression{"write.avro.compression-codec",
+ "gzip"};
+ inline static Entry<std::string> kDeleteAvroCompression{
+ "write.delete.avro.compression-codec", "gzip"};
+ inline static Entry<std::string> kAvroCompressionLevel{"write.avro.compression-level",
+ ""};
+ inline static Entry<std::string> kDeleteAvroCompressionLevel{
+ "write.delete.avro.compression-level", ""};
+
+ // ORC properties
+ inline static Entry<int64_t> kOrcStripeSizeBytes{"write.orc.stripe-size-bytes",
Review Comment:
These align with the Java definitions.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: define table properties with default values [iceberg-cpp]
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #212:
URL: https://github.com/apache/iceberg-cpp/pull/212#discussion_r2390092521
##########
src/iceberg/table_properties.h:
##########
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <limits>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "iceberg/iceberg_export.h"
+#include "iceberg/util/config.h"
+
+namespace iceberg {
+
+/// \brief Table properties for Iceberg tables.
+///
+/// This class provides configuration entries for various Iceberg table properties
+/// including format settings, commit behavior, file formats, compression settings,
+/// and other table-level configurations.
+class ICEBERG_EXPORT TableProperties : public ConfigBase<TableProperties> {
+ public:
+ template <typename T>
+ using Entry = const ConfigBase<TableProperties>::Entry<T>;
+
+ // Reserved table properties
+
+ /// \brief Reserved table property for table format version.
+ ///
+ /// Iceberg will default a new table's format version to the latest stable and
+ /// recommended version. This reserved property keyword allows users to override the
+ /// Iceberg format version of the table metadata.
+ ///
+ /// If this table property exists when creating a table, the table will use the
+ /// specified format version. If a table updates this property, it will try to upgrade
+ /// to the specified format version.
+ ///
+ /// \note incomplete or unstable versions cannot be selected using this property.
+ inline static Entry<std::string> kFormatVersion{"format-version", ""};
+ /// \brief Reserved table property for table UUID.
+ inline static Entry<std::string> kUuid{"uuid", ""};
+ /// \brief Reserved table property for the total number of snapshots.
+ inline static Entry<std::string> kSnapshotCount{"snapshot-count", ""};
+ /// \brief Reserved table property for current snapshot summary.
+ inline static Entry<std::string> kCurrentSnapshotSummary{"current-snapshot-summary",
+ ""};
+ /// \brief Reserved table property for current snapshot id.
+ inline static Entry<std::string> kCurrentSnapshotId{"current-snapshot-id", ""};
+ /// \brief Reserved table property for current snapshot timestamp.
+ inline static Entry<std::string> kCurrentSnapshotTimestamp{
+ "current-snapshot-timestamp-ms", ""};
+ /// \brief Reserved table property for the JSON representation of current schema.
+ inline static Entry<std::string> kCurrentSchema{"current-schema", ""};
+ /// \brief Reserved table property for the JSON representation of current(default)
+ /// partition spec.
+ inline static Entry<std::string> kDefaultPartitionSpec{"default-partition-spec", ""};
+ /// \brief Reserved table property for the JSON representation of current(default) sort
+ /// order.
+ inline static Entry<std::string> kDefaultSortOrder{"default-sort-order", ""};
+
+ // Commit properties
+
+ inline static Entry<int32_t> kCommitNumRetries{"commit.retry.num-retries", 4};
+ inline static Entry<int32_t> kCommitMinRetryWaitMs{"commit.retry.min-wait-ms", 100};
+ inline static Entry<int32_t> kCommitMaxRetryWaitMs{"commit.retry.max-wait-ms",
+ 60 * 1000}; // 1 minute
+ inline static Entry<int32_t> kCommitTotalRetryTimeMs{"commit.retry.total-timeout-ms",
+ 30 * 60 * 1000}; // 30 minutes
+ inline static Entry<int32_t> kCommitNumStatusChecks{"commit.status-check.num-retries",
+ 3};
+ inline static Entry<int64_t> kCommitStatusChecksMinWaitMs{
+ "commit.status-check.min-wait-ms", int64_t{1000}}; // 1 second
+ inline static Entry<int64_t> kCommitStatusChecksMaxWaitMs{
+ "commit.status-check.max-wait-ms", int64_t{60 * 1000}}; // 1 minute
+ inline static Entry<int64_t> kCommitStatusChecksTotalWaitMs{
+ "commit.status-check.total-timeout-ms", int64_t{30 * 60 * 1000}}; // 30 minutes
+
+ // Manifest properties
+
+ inline static Entry<int64_t> kManifestTargetSizeBytes{
+ "commit.manifest.target-size-bytes", int64_t{8 * 1024 * 1024}}; // 8 MB
+ inline static Entry<int32_t> kManifestMinMergeCount{
+ "commit.manifest.min-count-to-merge", 100};
+ inline static Entry<bool> kManifestMergeEnabled{"commit.manifest-merge.enabled", true};
+
+ // File format properties
+
+ inline static Entry<std::string> kDefaultFileFormat{"write.format.default", "parquet"};
+ inline static Entry<std::string> kDeleteDefaultFileFormat{"write.delete.format.default",
+ "parquet"};
+
+ // Parquet properties
+
+ inline static Entry<int32_t> kParquetRowGroupSizeBytes{
+ "write.parquet.row-group-size-bytes", 128 * 1024 * 1024}; // 128 MB
+ inline static Entry<int32_t> kDeleteParquetRowGroupSizeBytes{
+ "write.delete.parquet.row-group-size-bytes", 128 * 1024 * 1024}; // 128 MB
+ inline static Entry<int32_t> kParquetPageSizeBytes{"write.parquet.page-size-bytes",
+ 1024 * 1024}; // 1 MB
+ inline static Entry<int32_t> kDeleteParquetPageSizeBytes{
+ "write.delete.parquet.page-size-bytes", 1024 * 1024}; // 1 MB
+ inline static Entry<int32_t> kParquetPageRowLimit{"write.parquet.page-row-limit",
+ 20'000};
+ inline static Entry<int32_t> kDeleteParquetPageRowLimit{
+ "write.delete.parquet.page-row-limit", 20'000};
+ inline static Entry<int32_t> kParquetDictSizeBytes{"write.parquet.dict-size-bytes",
+ 2 * 1024 * 1024}; // 2 MB
+ inline static Entry<int32_t> kDeleteParquetDictSizeBytes{
+ "write.delete.parquet.dict-size-bytes", 2 * 1024 * 1024}; // 2 MB
+ inline static Entry<std::string> kParquetCompression{"write.parquet.compression-codec",
+ "zstd"};
+ inline static Entry<std::string> kDeleteParquetCompression{
+ "write.delete.parquet.compression-codec", "zstd"};
+ inline static Entry<std::string> kParquetCompressionLevel{
+ "write.parquet.compression-level", ""};
+ inline static Entry<std::string> kDeleteParquetCompressionLevel{
+ "write.delete.parquet.compression-level", ""};
+ inline static Entry<int32_t> kParquetRowGroupCheckMinRecordCount{
+ "write.parquet.row-group-check-min-record-count", 100};
+ inline static Entry<int32_t> kDeleteParquetRowGroupCheckMinRecordCount{
+ "write.delete.parquet.row-group-check-min-record-count", 100};
+ inline static Entry<int32_t> kParquetRowGroupCheckMaxRecordCount{
+ "write.parquet.row-group-check-max-record-count", 10'000};
+ inline static Entry<int32_t> kDeleteParquetRowGroupCheckMaxRecordCount{
+ "write.delete.parquet.row-group-check-max-record-count", 10'000};
+ inline static Entry<int32_t> kParquetBloomFilterMaxBytes{
+ "write.parquet.bloom-filter-max-bytes", 1024 * 1024}; // 1 MB
+ inline static std::string_view kParquetBloomFilterColumnFppPrefix{
+ "write.parquet.bloom-filter-fpp.column."};
+ inline static std::string_view kParquetBloomFilterColumnEnabledPrefix{
+ "write.parquet.bloom-filter-enabled.column."};
+ inline static std::string_view kParquetColumnStatsEnabledPrefix{
+ "write.parquet.stats-enabled.column."};
+
+ // Avro properties
+ inline static Entry<std::string> kAvroCompression{"write.avro.compression-codec",
+ "gzip"};
+ inline static Entry<std::string> kDeleteAvroCompression{
+ "write.delete.avro.compression-codec", "gzip"};
+ inline static Entry<std::string> kAvroCompressionLevel{"write.avro.compression-level",
+ ""};
+ inline static Entry<std::string> kDeleteAvroCompressionLevel{
+ "write.delete.avro.compression-level", ""};
+
+ // ORC properties
+ inline static Entry<int64_t> kOrcStripeSizeBytes{"write.orc.stripe-size-bytes",
+ int64_t{64} * 1024 * 1024};
+ inline static Entry<std::string> kOrcBloomFilterColumns{
+ "write.orc.bloom.filter.columns", ""};
+ inline static Entry<double> kOrcBloomFilterFpp{"write.orc.bloom.filter.fpp", 0.05};
+ inline static Entry<int64_t> kDeleteOrcStripeSizeBytes{
+ "write.delete.orc.stripe-size-bytes", int64_t{64} * 1024 * 1024}; // 64 MB
+ inline static Entry<int64_t> kOrcBlockSizeBytes{"write.orc.block-size-bytes",
+ int64_t{256} * 1024 * 1024}; // 256 MB
+ inline static Entry<int64_t> kDeleteOrcBlockSizeBytes{
+ "write.delete.orc.block-size-bytes", int64_t{256} * 1024 * 1024}; // 256 MB
+ inline static Entry<int32_t> kOrcWriteBatchSize{"write.orc.vectorized.batch-size",
+ 1024};
+ inline static Entry<int32_t> kDeleteOrcWriteBatchSize{
+ "write.delete.orc.vectorized.batch-size", 1024};
+ inline static Entry<std::string> kOrcCompression{"write.orc.compression-codec", "zlib"};
+ inline static Entry<std::string> kDeleteOrcCompression{
+ "write.delete.orc.compression-codec", "zlib"};
+ inline static Entry<std::string> kOrcCompressionStrategy{
+ "write.orc.compression-strategy", "speed"};
+ inline static Entry<std::string> kDeleteOrcCompressionStrategy{
+ "write.delete.orc.compression-strategy", "speed"};
+
+ // Read properties
+
+ inline static Entry<int64_t> kSplitSize{"read.split.target-size",
+ int64_t{128} * 1024 * 1024}; // 128 MB
+ inline static Entry<int64_t> kMetadataSplitSize{"read.split.metadata-target-size",
+ int64_t{32} * 1024 * 1024}; // 32 MB
+ inline static Entry<int32_t> kSplitLookback{"read.split.planning-lookback", 10};
+ inline static Entry<int64_t> kSplitOpenFileCost{"read.split.open-file-cost",
+ int64_t{4} * 1024 * 1024}; // 4 MB
+ inline static Entry<bool> kAdaptiveSplitSizeEnabled{"read.split.adaptive-size.enabled",
+ true};
+ inline static Entry<bool> kParquetVectorizationEnabled{
+ "read.parquet.vectorization.enabled", true};
+ inline static Entry<int32_t> kParquetBatchSize{"read.parquet.vectorization.batch-size",
+ 5000};
+ inline static Entry<bool> kOrcVectorizationEnabled{"read.orc.vectorization.enabled",
+ false};
+ inline static Entry<int32_t> kOrcBatchSize{"read.orc.vectorization.batch-size", 5000};
+ inline static Entry<std::string> kDataPlanningMode{"read.data-planning-mode", "auto"};
+ inline static Entry<std::string> kDeletePlanningMode{"read.delete-planning-mode",
+ "auto"};
+
+ // Write properties
+
+ inline static Entry<bool> kObjectStoreEnabled{"write.object-storage.enabled", false};
+ /// \brief Excludes the partition values in the path when set to true and object store
+ /// is enabled.
+ inline static Entry<bool> kWriteObjectStorePartitionedPaths{
+ "write.object-storage.partitioned-paths", true};
+ /// \brief This only applies to files written after this property is set. Files
+ /// previously written aren't relocated to reflect this parameter. If not set, defaults
+ /// to a "data" folder underneath the root path of the table.
+ inline static Entry<std::string> kWriteDataLocation{"write.data.path", ""};
+ /// \brief This only applies to files written after this property is set. Files
+ /// previously written aren't relocated to reflect this parameter. If not set, defaults
+ /// to a "metadata" folder underneath the root path of the table.
+ inline static Entry<std::string> kWriteMetadataLocation{"write.metadata.path", ""};
+ inline static Entry<int32_t> kWritePartitionSummaryLimit{
+ "write.summary.partition-limit", 0};
+ inline static Entry<std::string> kMetadataCompression{
+ "write.metadata.compression-codec", "none"};
+ inline static Entry<int32_t> kMetadataPreviousVersionsMax{
+ "write.metadata.previous-versions-max", 100};
+ /// \brief This enables to delete the oldest metadata file after commit.
+ inline static Entry<bool> kMetadataDeleteAfterCommitEnabled{
+ "write.metadata.delete-after-commit.enabled", false};
+ inline static Entry<int32_t> kMetricsMaxInferredColumnDefaults{
+ "write.metadata.metrics.max-inferred-column-defaults", 100};
+ inline static Entry<std::string> kDefaultWriteMetricsMode{
+ "write.metadata.metrics.default", "truncate(16)"};
+
+ inline static std::string_view kDefaultNameMapping{"schema.name-mapping.default"};
+
+ inline static Entry<bool> kWriteAuditPublishEnabled{"write.wap.enabled", false};
+ inline static Entry<int64_t> kWriteTargetFileSizeBytes{
+ "write.target-file-size-bytes", int64_t{512} * 1024 * 1024}; // 512 MB
+ inline static Entry<int64_t> kDeleteTargetFileSizeBytes{
+ "write.delete.target-file-size-bytes", int64_t{64} * 1024 * 1024}; // 64 MB
+
+ inline static Entry<bool> kSnapshotIdInheritanceEnabled{
+ "compatibility.snapshot-id-inheritance.enabled", false};
Review Comment:
I would not backport this part. It was used before V2 was released to test out the snapshot inheritance, but I don't think anyone is really using it since upgrading to V2 is easier.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: define table properties with default values [iceberg-cpp]
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on PR #212:
URL: https://github.com/apache/iceberg-cpp/pull/212#issuecomment-3362392084
Nice one, thanks for adding this @wgtmac 🙌
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: define table properties with default values [iceberg-cpp]
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #212:
URL: https://github.com/apache/iceberg-cpp/pull/212#discussion_r2390069067
##########
src/iceberg/table_properties.h:
##########
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <limits>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "iceberg/iceberg_export.h"
+#include "iceberg/util/config.h"
+
+namespace iceberg {
+
+/// \brief Table properties for Iceberg tables.
+///
+/// This class provides configuration entries for various Iceberg table properties
+/// including format settings, commit behavior, file formats, compression settings,
+/// and other table-level configurations.
+class ICEBERG_EXPORT TableProperties : public ConfigBase<TableProperties> {
+ public:
+ template <typename T>
+ using Entry = const ConfigBase<TableProperties>::Entry<T>;
+
+ // Reserved table properties
+
+ /// \brief Reserved table property for table format version.
+ ///
+ /// Iceberg will default a new table's format version to the latest stable and
+ /// recommended version. This reserved property keyword allows users to override the
+ /// Iceberg format version of the table metadata.
+ ///
+ /// If this table property exists when creating a table, the table will use the
+ /// specified format version. If a table updates this property, it will try to upgrade
+ /// to the specified format version.
+ ///
+ /// \note incomplete or unstable versions cannot be selected using this property.
+ inline static Entry<std::string> kFormatVersion{"format-version", ""};
+ /// \brief Reserved table property for table UUID.
+ inline static Entry<std::string> kUuid{"uuid", ""};
+ /// \brief Reserved table property for the total number of snapshots.
+ inline static Entry<std::string> kSnapshotCount{"snapshot-count", ""};
+ /// \brief Reserved table property for current snapshot summary.
+ inline static Entry<std::string> kCurrentSnapshotSummary{"current-snapshot-summary",
+ ""};
+ /// \brief Reserved table property for current snapshot id.
+ inline static Entry<std::string> kCurrentSnapshotId{"current-snapshot-id", ""};
+ /// \brief Reserved table property for current snapshot timestamp.
+ inline static Entry<std::string> kCurrentSnapshotTimestamp{
+ "current-snapshot-timestamp-ms", ""};
+ /// \brief Reserved table property for the JSON representation of current schema.
+ inline static Entry<std::string> kCurrentSchema{"current-schema", ""};
+ /// \brief Reserved table property for the JSON representation of current(default)
+ /// partition spec.
+ inline static Entry<std::string> kDefaultPartitionSpec{"default-partition-spec", ""};
+ /// \brief Reserved table property for the JSON representation of current(default) sort
+ /// order.
+ inline static Entry<std::string> kDefaultSortOrder{"default-sort-order", ""};
+
+ // Commit properties
+
+ inline static Entry<int32_t> kCommitNumRetries{"commit.retry.num-retries", 4};
+ inline static Entry<int32_t> kCommitMinRetryWaitMs{"commit.retry.min-wait-ms", 100};
+ inline static Entry<int32_t> kCommitMaxRetryWaitMs{"commit.retry.max-wait-ms",
+ 60 * 1000}; // 1 minute
+ inline static Entry<int32_t> kCommitTotalRetryTimeMs{"commit.retry.total-timeout-ms",
+ 30 * 60 * 1000}; // 30 minutes
+ inline static Entry<int32_t> kCommitNumStatusChecks{"commit.status-check.num-retries",
+ 3};
+ inline static Entry<int64_t> kCommitStatusChecksMinWaitMs{
+ "commit.status-check.min-wait-ms", int64_t{1000}}; // 1 second
+ inline static Entry<int64_t> kCommitStatusChecksMaxWaitMs{
+ "commit.status-check.max-wait-ms", int64_t{60 * 1000}}; // 1 minute
+ inline static Entry<int64_t> kCommitStatusChecksTotalWaitMs{
+ "commit.status-check.total-timeout-ms", int64_t{30 * 60 * 1000}}; // 30 minutes
+
+ // Manifest properties
+
+ inline static Entry<int64_t> kManifestTargetSizeBytes{
+ "commit.manifest.target-size-bytes", int64_t{8 * 1024 * 1024}}; // 8 MB
+ inline static Entry<int32_t> kManifestMinMergeCount{
+ "commit.manifest.min-count-to-merge", 100};
+ inline static Entry<bool> kManifestMergeEnabled{"commit.manifest-merge.enabled", true};
+
+ // File format properties
+
+ inline static Entry<std::string> kDefaultFileFormat{"write.format.default", "parquet"};
+ inline static Entry<std::string> kDeleteDefaultFileFormat{"write.delete.format.default",
+ "parquet"};
+
+ // Parquet properties
+
+ inline static Entry<int32_t> kParquetRowGroupSizeBytes{
+ "write.parquet.row-group-size-bytes", 128 * 1024 * 1024}; // 128 MB
+ inline static Entry<int32_t> kDeleteParquetRowGroupSizeBytes{
+ "write.delete.parquet.row-group-size-bytes", 128 * 1024 * 1024}; // 128 MB
+ inline static Entry<int32_t> kParquetPageSizeBytes{"write.parquet.page-size-bytes",
+ 1024 * 1024}; // 1 MB
+ inline static Entry<int32_t> kDeleteParquetPageSizeBytes{
+ "write.delete.parquet.page-size-bytes", 1024 * 1024}; // 1 MB
+ inline static Entry<int32_t> kParquetPageRowLimit{"write.parquet.page-row-limit",
+ 20'000};
+ inline static Entry<int32_t> kDeleteParquetPageRowLimit{
+ "write.delete.parquet.page-row-limit", 20'000};
+ inline static Entry<int32_t> kParquetDictSizeBytes{"write.parquet.dict-size-bytes",
+ 2 * 1024 * 1024}; // 2 MB
+ inline static Entry<int32_t> kDeleteParquetDictSizeBytes{
+ "write.delete.parquet.dict-size-bytes", 2 * 1024 * 1024}; // 2 MB
+ inline static Entry<std::string> kParquetCompression{"write.parquet.compression-codec",
+ "zstd"};
+ inline static Entry<std::string> kDeleteParquetCompression{
+ "write.delete.parquet.compression-codec", "zstd"};
+ inline static Entry<std::string> kParquetCompressionLevel{
+ "write.parquet.compression-level", ""};
+ inline static Entry<std::string> kDeleteParquetCompressionLevel{
+ "write.delete.parquet.compression-level", ""};
+ inline static Entry<int32_t> kParquetRowGroupCheckMinRecordCount{
+ "write.parquet.row-group-check-min-record-count", 100};
+ inline static Entry<int32_t> kDeleteParquetRowGroupCheckMinRecordCount{
+ "write.delete.parquet.row-group-check-min-record-count", 100};
+ inline static Entry<int32_t> kParquetRowGroupCheckMaxRecordCount{
+ "write.parquet.row-group-check-max-record-count", 10'000};
+ inline static Entry<int32_t> kDeleteParquetRowGroupCheckMaxRecordCount{
+ "write.delete.parquet.row-group-check-max-record-count", 10'000};
+ inline static Entry<int32_t> kParquetBloomFilterMaxBytes{
+ "write.parquet.bloom-filter-max-bytes", 1024 * 1024}; // 1 MB
+ inline static std::string_view kParquetBloomFilterColumnFppPrefix{
+ "write.parquet.bloom-filter-fpp.column."};
+ inline static std::string_view kParquetBloomFilterColumnEnabledPrefix{
+ "write.parquet.bloom-filter-enabled.column."};
+ inline static std::string_view kParquetColumnStatsEnabledPrefix{
+ "write.parquet.stats-enabled.column."};
+
+ // Avro properties
+ inline static Entry<std::string> kAvroCompression{"write.avro.compression-codec",
+ "gzip"};
+ inline static Entry<std::string> kDeleteAvroCompression{
+ "write.delete.avro.compression-codec", "gzip"};
+ inline static Entry<std::string> kAvroCompressionLevel{"write.avro.compression-level",
+ ""};
+ inline static Entry<std::string> kDeleteAvroCompressionLevel{
+ "write.delete.avro.compression-level", ""};
+
+ // ORC properties
+ inline static Entry<int64_t> kOrcStripeSizeBytes{"write.orc.stripe-size-bytes",
Review Comment:
Why are we going with int64 here, and with int32 in Parquet? Int32 seems to be sufficient?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: define table properties with default values [iceberg-cpp]
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko merged PR #212:
URL: https://github.com/apache/iceberg-cpp/pull/212
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: define table properties with default values [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #212:
URL: https://github.com/apache/iceberg-cpp/pull/212#discussion_r2392126778
##########
src/iceberg/table_properties.h:
##########
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <limits>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "iceberg/iceberg_export.h"
+#include "iceberg/util/config.h"
+
+namespace iceberg {
+
+/// \brief Table properties for Iceberg tables.
+///
+/// This class provides configuration entries for various Iceberg table properties
+/// including format settings, commit behavior, file formats, compression settings,
+/// and other table-level configurations.
+class ICEBERG_EXPORT TableProperties : public ConfigBase<TableProperties> {
+ public:
+ template <typename T>
+ using Entry = const ConfigBase<TableProperties>::Entry<T>;
+
+ // Reserved table properties
+
+ /// \brief Reserved table property for table format version.
+ ///
+ /// Iceberg will default a new table's format version to the latest stable and
+ /// recommended version. This reserved property keyword allows users to override the
+ /// Iceberg format version of the table metadata.
+ ///
+ /// If this table property exists when creating a table, the table will use the
+ /// specified format version. If a table updates this property, it will try to upgrade
+ /// to the specified format version.
+ ///
+ /// \note incomplete or unstable versions cannot be selected using this property.
+ inline static Entry<std::string> kFormatVersion{"format-version", ""};
+ /// \brief Reserved table property for table UUID.
+ inline static Entry<std::string> kUuid{"uuid", ""};
+ /// \brief Reserved table property for the total number of snapshots.
+ inline static Entry<std::string> kSnapshotCount{"snapshot-count", ""};
+ /// \brief Reserved table property for current snapshot summary.
+ inline static Entry<std::string> kCurrentSnapshotSummary{"current-snapshot-summary",
+ ""};
+ /// \brief Reserved table property for current snapshot id.
+ inline static Entry<std::string> kCurrentSnapshotId{"current-snapshot-id", ""};
+ /// \brief Reserved table property for current snapshot timestamp.
+ inline static Entry<std::string> kCurrentSnapshotTimestamp{
+ "current-snapshot-timestamp-ms", ""};
+ /// \brief Reserved table property for the JSON representation of current schema.
+ inline static Entry<std::string> kCurrentSchema{"current-schema", ""};
+ /// \brief Reserved table property for the JSON representation of current(default)
+ /// partition spec.
+ inline static Entry<std::string> kDefaultPartitionSpec{"default-partition-spec", ""};
+ /// \brief Reserved table property for the JSON representation of current(default) sort
+ /// order.
+ inline static Entry<std::string> kDefaultSortOrder{"default-sort-order", ""};
+
+ // Commit properties
+
+ inline static Entry<int32_t> kCommitNumRetries{"commit.retry.num-retries", 4};
+ inline static Entry<int32_t> kCommitMinRetryWaitMs{"commit.retry.min-wait-ms", 100};
+ inline static Entry<int32_t> kCommitMaxRetryWaitMs{"commit.retry.max-wait-ms",
+ 60 * 1000}; // 1 minute
+ inline static Entry<int32_t> kCommitTotalRetryTimeMs{"commit.retry.total-timeout-ms",
+ 30 * 60 * 1000}; // 30 minutes
+ inline static Entry<int32_t> kCommitNumStatusChecks{"commit.status-check.num-retries",
+ 3};
+ inline static Entry<int64_t> kCommitStatusChecksMinWaitMs{
+ "commit.status-check.min-wait-ms", int64_t{1000}}; // 1 second
+ inline static Entry<int64_t> kCommitStatusChecksMaxWaitMs{
+ "commit.status-check.max-wait-ms", int64_t{60 * 1000}}; // 1 minute
+ inline static Entry<int64_t> kCommitStatusChecksTotalWaitMs{
+ "commit.status-check.total-timeout-ms", int64_t{30 * 60 * 1000}}; // 30 minutes
+
+ // Manifest properties
+
+ inline static Entry<int64_t> kManifestTargetSizeBytes{
+ "commit.manifest.target-size-bytes", int64_t{8 * 1024 * 1024}}; // 8 MB
+ inline static Entry<int32_t> kManifestMinMergeCount{
+ "commit.manifest.min-count-to-merge", 100};
+ inline static Entry<bool> kManifestMergeEnabled{"commit.manifest-merge.enabled", true};
+
+ // File format properties
+
+ inline static Entry<std::string> kDefaultFileFormat{"write.format.default", "parquet"};
+ inline static Entry<std::string> kDeleteDefaultFileFormat{"write.delete.format.default",
+ "parquet"};
+
+ // Parquet properties
+
+ inline static Entry<int32_t> kParquetRowGroupSizeBytes{
+ "write.parquet.row-group-size-bytes", 128 * 1024 * 1024}; // 128 MB
+ inline static Entry<int32_t> kDeleteParquetRowGroupSizeBytes{
+ "write.delete.parquet.row-group-size-bytes", 128 * 1024 * 1024}; // 128 MB
+ inline static Entry<int32_t> kParquetPageSizeBytes{"write.parquet.page-size-bytes",
+ 1024 * 1024}; // 1 MB
+ inline static Entry<int32_t> kDeleteParquetPageSizeBytes{
+ "write.delete.parquet.page-size-bytes", 1024 * 1024}; // 1 MB
+ inline static Entry<int32_t> kParquetPageRowLimit{"write.parquet.page-row-limit",
+ 20'000};
+ inline static Entry<int32_t> kDeleteParquetPageRowLimit{
+ "write.delete.parquet.page-row-limit", 20'000};
+ inline static Entry<int32_t> kParquetDictSizeBytes{"write.parquet.dict-size-bytes",
+ 2 * 1024 * 1024}; // 2 MB
+ inline static Entry<int32_t> kDeleteParquetDictSizeBytes{
+ "write.delete.parquet.dict-size-bytes", 2 * 1024 * 1024}; // 2 MB
+ inline static Entry<std::string> kParquetCompression{"write.parquet.compression-codec",
+ "zstd"};
+ inline static Entry<std::string> kDeleteParquetCompression{
+ "write.delete.parquet.compression-codec", "zstd"};
+ inline static Entry<std::string> kParquetCompressionLevel{
+ "write.parquet.compression-level", ""};
+ inline static Entry<std::string> kDeleteParquetCompressionLevel{
+ "write.delete.parquet.compression-level", ""};
+ inline static Entry<int32_t> kParquetRowGroupCheckMinRecordCount{
+ "write.parquet.row-group-check-min-record-count", 100};
+ inline static Entry<int32_t> kDeleteParquetRowGroupCheckMinRecordCount{
+ "write.delete.parquet.row-group-check-min-record-count", 100};
+ inline static Entry<int32_t> kParquetRowGroupCheckMaxRecordCount{
+ "write.parquet.row-group-check-max-record-count", 10'000};
+ inline static Entry<int32_t> kDeleteParquetRowGroupCheckMaxRecordCount{
+ "write.delete.parquet.row-group-check-max-record-count", 10'000};
+ inline static Entry<int32_t> kParquetBloomFilterMaxBytes{
+ "write.parquet.bloom-filter-max-bytes", 1024 * 1024}; // 1 MB
+ inline static std::string_view kParquetBloomFilterColumnFppPrefix{
+ "write.parquet.bloom-filter-fpp.column."};
+ inline static std::string_view kParquetBloomFilterColumnEnabledPrefix{
+ "write.parquet.bloom-filter-enabled.column."};
+ inline static std::string_view kParquetColumnStatsEnabledPrefix{
+ "write.parquet.stats-enabled.column."};
+
+ // Avro properties
+ inline static Entry<std::string> kAvroCompression{"write.avro.compression-codec",
+ "gzip"};
+ inline static Entry<std::string> kDeleteAvroCompression{
+ "write.delete.avro.compression-codec", "gzip"};
+ inline static Entry<std::string> kAvroCompressionLevel{"write.avro.compression-level",
+ ""};
+ inline static Entry<std::string> kDeleteAvroCompressionLevel{
+ "write.delete.avro.compression-level", ""};
+
+ // ORC properties
+ inline static Entry<int64_t> kOrcStripeSizeBytes{"write.orc.stripe-size-bytes",
+ int64_t{64} * 1024 * 1024};
+ inline static Entry<std::string> kOrcBloomFilterColumns{
+ "write.orc.bloom.filter.columns", ""};
+ inline static Entry<double> kOrcBloomFilterFpp{"write.orc.bloom.filter.fpp", 0.05};
+ inline static Entry<int64_t> kDeleteOrcStripeSizeBytes{
+ "write.delete.orc.stripe-size-bytes", int64_t{64} * 1024 * 1024}; // 64 MB
+ inline static Entry<int64_t> kOrcBlockSizeBytes{"write.orc.block-size-bytes",
+ int64_t{256} * 1024 * 1024}; // 256 MB
+ inline static Entry<int64_t> kDeleteOrcBlockSizeBytes{
+ "write.delete.orc.block-size-bytes", int64_t{256} * 1024 * 1024}; // 256 MB
+ inline static Entry<int32_t> kOrcWriteBatchSize{"write.orc.vectorized.batch-size",
+ 1024};
+ inline static Entry<int32_t> kDeleteOrcWriteBatchSize{
+ "write.delete.orc.vectorized.batch-size", 1024};
+ inline static Entry<std::string> kOrcCompression{"write.orc.compression-codec", "zlib"};
+ inline static Entry<std::string> kDeleteOrcCompression{
+ "write.delete.orc.compression-codec", "zlib"};
+ inline static Entry<std::string> kOrcCompressionStrategy{
+ "write.orc.compression-strategy", "speed"};
+ inline static Entry<std::string> kDeleteOrcCompressionStrategy{
+ "write.delete.orc.compression-strategy", "speed"};
+
+ // Read properties
+
+ inline static Entry<int64_t> kSplitSize{"read.split.target-size",
+ int64_t{128} * 1024 * 1024}; // 128 MB
+ inline static Entry<int64_t> kMetadataSplitSize{"read.split.metadata-target-size",
+ int64_t{32} * 1024 * 1024}; // 32 MB
+ inline static Entry<int32_t> kSplitLookback{"read.split.planning-lookback", 10};
+ inline static Entry<int64_t> kSplitOpenFileCost{"read.split.open-file-cost",
+ int64_t{4} * 1024 * 1024}; // 4 MB
+ inline static Entry<bool> kAdaptiveSplitSizeEnabled{"read.split.adaptive-size.enabled",
+ true};
+ inline static Entry<bool> kParquetVectorizationEnabled{
+ "read.parquet.vectorization.enabled", true};
+ inline static Entry<int32_t> kParquetBatchSize{"read.parquet.vectorization.batch-size",
+ 5000};
+ inline static Entry<bool> kOrcVectorizationEnabled{"read.orc.vectorization.enabled",
+ false};
+ inline static Entry<int32_t> kOrcBatchSize{"read.orc.vectorization.batch-size", 5000};
+ inline static Entry<std::string> kDataPlanningMode{"read.data-planning-mode", "auto"};
+ inline static Entry<std::string> kDeletePlanningMode{"read.delete-planning-mode",
+ "auto"};
+
+ // Write properties
+
+ inline static Entry<bool> kObjectStoreEnabled{"write.object-storage.enabled", false};
+ /// \brief Excludes the partition values in the path when set to true and object store
+ /// is enabled.
+ inline static Entry<bool> kWriteObjectStorePartitionedPaths{
+ "write.object-storage.partitioned-paths", true};
+ /// \brief This only applies to files written after this property is set. Files
+ /// previously written aren't relocated to reflect this parameter. If not set, defaults
+ /// to a "data" folder underneath the root path of the table.
+ inline static Entry<std::string> kWriteDataLocation{"write.data.path", ""};
+ /// \brief This only applies to files written after this property is set. Files
+ /// previously written aren't relocated to reflect this parameter. If not set, defaults
+ /// to a "metadata" folder underneath the root path of the table.
+ inline static Entry<std::string> kWriteMetadataLocation{"write.metadata.path", ""};
+ inline static Entry<int32_t> kWritePartitionSummaryLimit{
+ "write.summary.partition-limit", 0};
+ inline static Entry<std::string> kMetadataCompression{
+ "write.metadata.compression-codec", "none"};
+ inline static Entry<int32_t> kMetadataPreviousVersionsMax{
+ "write.metadata.previous-versions-max", 100};
+ /// \brief This enables to delete the oldest metadata file after commit.
+ inline static Entry<bool> kMetadataDeleteAfterCommitEnabled{
+ "write.metadata.delete-after-commit.enabled", false};
+ inline static Entry<int32_t> kMetricsMaxInferredColumnDefaults{
+ "write.metadata.metrics.max-inferred-column-defaults", 100};
+ inline static Entry<std::string> kDefaultWriteMetricsMode{
+ "write.metadata.metrics.default", "truncate(16)"};
+
+ inline static std::string_view kDefaultNameMapping{"schema.name-mapping.default"};
+
+ inline static Entry<bool> kWriteAuditPublishEnabled{"write.wap.enabled", false};
+ inline static Entry<int64_t> kWriteTargetFileSizeBytes{
+ "write.target-file-size-bytes", int64_t{512} * 1024 * 1024}; // 512 MB
+ inline static Entry<int64_t> kDeleteTargetFileSizeBytes{
+ "write.delete.target-file-size-bytes", int64_t{64} * 1024 * 1024}; // 64 MB
+
+ inline static Entry<bool> kSnapshotIdInheritanceEnabled{
+ "compatibility.snapshot-id-inheritance.enabled", false};
Review Comment:
Just removed it. Thanks for the information!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: define table properties with default values [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #212:
URL: https://github.com/apache/iceberg-cpp/pull/212#discussion_r2391800421
##########
src/iceberg/table_properties.h:
##########
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <limits>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "iceberg/iceberg_export.h"
+#include "iceberg/util/config.h"
+
+namespace iceberg {
+
+/// \brief Table properties for Iceberg tables.
+///
+/// This class provides configuration entries for various Iceberg table properties
+/// including format settings, commit behavior, file formats, compression settings,
+/// and other table-level configurations.
+class ICEBERG_EXPORT TableProperties : public ConfigBase<TableProperties> {
+ public:
+ template <typename T>
+ using Entry = const ConfigBase<TableProperties>::Entry<T>;
+
+ // Reserved table properties
+
+ /// \brief Reserved table property for table format version.
+ ///
+ /// Iceberg will default a new table's format version to the latest stable and
+ /// recommended version. This reserved property keyword allows users to override the
+ /// Iceberg format version of the table metadata.
+ ///
+ /// If this table property exists when creating a table, the table will use the
+ /// specified format version. If a table updates this property, it will try to upgrade
+ /// to the specified format version.
+ ///
+ /// \note incomplete or unstable versions cannot be selected using this property.
+ inline static Entry<std::string> kFormatVersion{"format-version", ""};
+ /// \brief Reserved table property for table UUID.
+ inline static Entry<std::string> kUuid{"uuid", ""};
+ /// \brief Reserved table property for the total number of snapshots.
+ inline static Entry<std::string> kSnapshotCount{"snapshot-count", ""};
+ /// \brief Reserved table property for current snapshot summary.
+ inline static Entry<std::string> kCurrentSnapshotSummary{"current-snapshot-summary",
+ ""};
+ /// \brief Reserved table property for current snapshot id.
+ inline static Entry<std::string> kCurrentSnapshotId{"current-snapshot-id", ""};
+ /// \brief Reserved table property for current snapshot timestamp.
+ inline static Entry<std::string> kCurrentSnapshotTimestamp{
+ "current-snapshot-timestamp-ms", ""};
+ /// \brief Reserved table property for the JSON representation of current schema.
+ inline static Entry<std::string> kCurrentSchema{"current-schema", ""};
+ /// \brief Reserved table property for the JSON representation of current(default)
+ /// partition spec.
+ inline static Entry<std::string> kDefaultPartitionSpec{"default-partition-spec", ""};
+ /// \brief Reserved table property for the JSON representation of current(default) sort
+ /// order.
+ inline static Entry<std::string> kDefaultSortOrder{"default-sort-order", ""};
+
+ // Commit properties
+
+ inline static Entry<int32_t> kCommitNumRetries{"commit.retry.num-retries", 4};
+ inline static Entry<int32_t> kCommitMinRetryWaitMs{"commit.retry.min-wait-ms", 100};
+ inline static Entry<int32_t> kCommitMaxRetryWaitMs{"commit.retry.max-wait-ms",
+ 60 * 1000}; // 1 minute
+ inline static Entry<int32_t> kCommitTotalRetryTimeMs{"commit.retry.total-timeout-ms",
+ 30 * 60 * 1000}; // 30 minutes
+ inline static Entry<int32_t> kCommitNumStatusChecks{"commit.status-check.num-retries",
+ 3};
+ inline static Entry<int64_t> kCommitStatusChecksMinWaitMs{
+ "commit.status-check.min-wait-ms", int64_t{1000}}; // 1 second
+ inline static Entry<int64_t> kCommitStatusChecksMaxWaitMs{
+ "commit.status-check.max-wait-ms", int64_t{60 * 1000}}; // 1 minute
+ inline static Entry<int64_t> kCommitStatusChecksTotalWaitMs{
+ "commit.status-check.total-timeout-ms", int64_t{30 * 60 * 1000}}; // 30 minutes
+
+ // Manifest properties
+
+ inline static Entry<int64_t> kManifestTargetSizeBytes{
+ "commit.manifest.target-size-bytes", int64_t{8 * 1024 * 1024}}; // 8 MB
+ inline static Entry<int32_t> kManifestMinMergeCount{
+ "commit.manifest.min-count-to-merge", 100};
+ inline static Entry<bool> kManifestMergeEnabled{"commit.manifest-merge.enabled", true};
+
+ // File format properties
+
+ inline static Entry<std::string> kDefaultFileFormat{"write.format.default", "parquet"};
+ inline static Entry<std::string> kDeleteDefaultFileFormat{"write.delete.format.default",
+ "parquet"};
+
+ // Parquet properties
+
+ inline static Entry<int32_t> kParquetRowGroupSizeBytes{
+ "write.parquet.row-group-size-bytes", 128 * 1024 * 1024}; // 128 MB
+ inline static Entry<int32_t> kDeleteParquetRowGroupSizeBytes{
+ "write.delete.parquet.row-group-size-bytes", 128 * 1024 * 1024}; // 128 MB
+ inline static Entry<int32_t> kParquetPageSizeBytes{"write.parquet.page-size-bytes",
+ 1024 * 1024}; // 1 MB
+ inline static Entry<int32_t> kDeleteParquetPageSizeBytes{
+ "write.delete.parquet.page-size-bytes", 1024 * 1024}; // 1 MB
+ inline static Entry<int32_t> kParquetPageRowLimit{"write.parquet.page-row-limit",
+ 20'000};
+ inline static Entry<int32_t> kDeleteParquetPageRowLimit{
+ "write.delete.parquet.page-row-limit", 20'000};
+ inline static Entry<int32_t> kParquetDictSizeBytes{"write.parquet.dict-size-bytes",
+ 2 * 1024 * 1024}; // 2 MB
+ inline static Entry<int32_t> kDeleteParquetDictSizeBytes{
+ "write.delete.parquet.dict-size-bytes", 2 * 1024 * 1024}; // 2 MB
+ inline static Entry<std::string> kParquetCompression{"write.parquet.compression-codec",
+ "zstd"};
+ inline static Entry<std::string> kDeleteParquetCompression{
+ "write.delete.parquet.compression-codec", "zstd"};
+ inline static Entry<std::string> kParquetCompressionLevel{
+ "write.parquet.compression-level", ""};
+ inline static Entry<std::string> kDeleteParquetCompressionLevel{
+ "write.delete.parquet.compression-level", ""};
+ inline static Entry<int32_t> kParquetRowGroupCheckMinRecordCount{
+ "write.parquet.row-group-check-min-record-count", 100};
+ inline static Entry<int32_t> kDeleteParquetRowGroupCheckMinRecordCount{
+ "write.delete.parquet.row-group-check-min-record-count", 100};
+ inline static Entry<int32_t> kParquetRowGroupCheckMaxRecordCount{
+ "write.parquet.row-group-check-max-record-count", 10'000};
+ inline static Entry<int32_t> kDeleteParquetRowGroupCheckMaxRecordCount{
+ "write.delete.parquet.row-group-check-max-record-count", 10'000};
+ inline static Entry<int32_t> kParquetBloomFilterMaxBytes{
+ "write.parquet.bloom-filter-max-bytes", 1024 * 1024}; // 1 MB
+ inline static std::string_view kParquetBloomFilterColumnFppPrefix{
+ "write.parquet.bloom-filter-fpp.column."};
+ inline static std::string_view kParquetBloomFilterColumnEnabledPrefix{
+ "write.parquet.bloom-filter-enabled.column."};
+ inline static std::string_view kParquetColumnStatsEnabledPrefix{
+ "write.parquet.stats-enabled.column."};
+
+ // Avro properties
+ inline static Entry<std::string> kAvroCompression{"write.avro.compression-codec",
+ "gzip"};
+ inline static Entry<std::string> kDeleteAvroCompression{
+ "write.delete.avro.compression-codec", "gzip"};
+ inline static Entry<std::string> kAvroCompressionLevel{"write.avro.compression-level",
+ ""};
+ inline static Entry<std::string> kDeleteAvroCompressionLevel{
+ "write.delete.avro.compression-level", ""};
+
+ // ORC properties
+ inline static Entry<int64_t> kOrcStripeSizeBytes{"write.orc.stripe-size-bytes",
+ int64_t{64} * 1024 * 1024};
+ inline static Entry<std::string> kOrcBloomFilterColumns{
+ "write.orc.bloom.filter.columns", ""};
+ inline static Entry<double> kOrcBloomFilterFpp{"write.orc.bloom.filter.fpp", 0.05};
+ inline static Entry<int64_t> kDeleteOrcStripeSizeBytes{
+ "write.delete.orc.stripe-size-bytes", int64_t{64} * 1024 * 1024}; // 64 MB
+ inline static Entry<int64_t> kOrcBlockSizeBytes{"write.orc.block-size-bytes",
+ int64_t{256} * 1024 * 1024}; // 256 MB
+ inline static Entry<int64_t> kDeleteOrcBlockSizeBytes{
+ "write.delete.orc.block-size-bytes", int64_t{256} * 1024 * 1024}; // 256 MB
+ inline static Entry<int32_t> kOrcWriteBatchSize{"write.orc.vectorized.batch-size",
+ 1024};
+ inline static Entry<int32_t> kDeleteOrcWriteBatchSize{
+ "write.delete.orc.vectorized.batch-size", 1024};
+ inline static Entry<std::string> kOrcCompression{"write.orc.compression-codec", "zlib"};
+ inline static Entry<std::string> kDeleteOrcCompression{
+ "write.delete.orc.compression-codec", "zlib"};
+ inline static Entry<std::string> kOrcCompressionStrategy{
+ "write.orc.compression-strategy", "speed"};
+ inline static Entry<std::string> kDeleteOrcCompressionStrategy{
+ "write.delete.orc.compression-strategy", "speed"};
+
+ // Read properties
+
+ inline static Entry<int64_t> kSplitSize{"read.split.target-size",
+ int64_t{128} * 1024 * 1024}; // 128 MB
+ inline static Entry<int64_t> kMetadataSplitSize{"read.split.metadata-target-size",
+ int64_t{32} * 1024 * 1024}; // 32 MB
+ inline static Entry<int32_t> kSplitLookback{"read.split.planning-lookback", 10};
+ inline static Entry<int64_t> kSplitOpenFileCost{"read.split.open-file-cost",
+ int64_t{4} * 1024 * 1024}; // 4 MB
+ inline static Entry<bool> kAdaptiveSplitSizeEnabled{"read.split.adaptive-size.enabled",
+ true};
+ inline static Entry<bool> kParquetVectorizationEnabled{
+ "read.parquet.vectorization.enabled", true};
+ inline static Entry<int32_t> kParquetBatchSize{"read.parquet.vectorization.batch-size",
+ 5000};
+ inline static Entry<bool> kOrcVectorizationEnabled{"read.orc.vectorization.enabled",
+ false};
+ inline static Entry<int32_t> kOrcBatchSize{"read.orc.vectorization.batch-size", 5000};
+ inline static Entry<std::string> kDataPlanningMode{"read.data-planning-mode", "auto"};
+ inline static Entry<std::string> kDeletePlanningMode{"read.delete-planning-mode",
+ "auto"};
+
+ // Write properties
+
+ inline static Entry<bool> kObjectStoreEnabled{"write.object-storage.enabled", false};
+ /// \brief Excludes the partition values in the path when set to true and object store
+ /// is enabled.
+ inline static Entry<bool> kWriteObjectStorePartitionedPaths{
+ "write.object-storage.partitioned-paths", true};
+ /// \brief This only applies to files written after this property is set. Files
+ /// previously written aren't relocated to reflect this parameter. If not set, defaults
+ /// to a "data" folder underneath the root path of the table.
+ inline static Entry<std::string> kWriteDataLocation{"write.data.path", ""};
+ /// \brief This only applies to files written after this property is set. Files
+ /// previously written aren't relocated to reflect this parameter. If not set, defaults
+ /// to a "metadata" folder underneath the root path of the table.
+ inline static Entry<std::string> kWriteMetadataLocation{"write.metadata.path", ""};
+ inline static Entry<int32_t> kWritePartitionSummaryLimit{
+ "write.summary.partition-limit", 0};
+ inline static Entry<std::string> kMetadataCompression{
+ "write.metadata.compression-codec", "none"};
+ inline static Entry<int32_t> kMetadataPreviousVersionsMax{
+ "write.metadata.previous-versions-max", 100};
+ /// \brief This enables to delete the oldest metadata file after commit.
+ inline static Entry<bool> kMetadataDeleteAfterCommitEnabled{
+ "write.metadata.delete-after-commit.enabled", false};
+ inline static Entry<int32_t> kMetricsMaxInferredColumnDefaults{
+ "write.metadata.metrics.max-inferred-column-defaults", 100};
+ inline static Entry<std::string> kDefaultWriteMetricsMode{
+ "write.metadata.metrics.default", "truncate(16)"};
+
+ inline static std::string_view kDefaultNameMapping{"schema.name-mapping.default"};
+
+ inline static Entry<bool> kWriteAuditPublishEnabled{"write.wap.enabled", false};
+ inline static Entry<int64_t> kWriteTargetFileSizeBytes{
+ "write.target-file-size-bytes", int64_t{512} * 1024 * 1024}; // 512 MB
+ inline static Entry<int64_t> kDeleteTargetFileSizeBytes{
+ "write.delete.target-file-size-bytes", int64_t{64} * 1024 * 1024}; // 64 MB
+
+ inline static Entry<bool> kSnapshotIdInheritanceEnabled{
+ "compatibility.snapshot-id-inheritance.enabled", false};
+
Review Comment:
```suggestion
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org