You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by co...@apache.org on 2025/10/02 18:29:32 UTC
(polaris) 01/01: Made table properties constants and pulled out static utility method
This is an automated email from the ASF dual-hosted git repository.
collado pushed a commit to branch mcollado-table-metadata-properties
in repository https://gitbox.apache.org/repos/asf/polaris.git
commit e36e7ebc8aac5e92286f394f62968c6e73058a02
Author: Michael Collado <co...@gmail.com>
AuthorDate: Thu Oct 2 11:29:16 2025 -0700
Made table properties constants and pulled out static utility method
---
.../core/entity/table/IcebergTableLikeEntity.java | 41 ++++++++++++--
.../core/persistence/cache/EntityWeigherTest.java | 2 +-
.../service/catalog/iceberg/IcebergCatalog.java | 62 +++++++++++++---------
.../iceberg/AbstractIcebergCatalogTest.java | 25 +++++----
4 files changed, 90 insertions(+), 40 deletions(-)
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java
index ca6841b2c..f1b756779 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java
@@ -22,6 +22,9 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.common.base.Preconditions;
import jakarta.annotation.Nonnull;
import jakarta.annotation.Nullable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.rest.RESTUtil;
@@ -32,10 +35,6 @@ import org.apache.polaris.core.entity.PolarisEntityConstants;
import org.apache.polaris.core.entity.PolarisEntitySubType;
import org.apache.polaris.core.entity.PolarisEntityType;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-
/**
* An entity type for {@link TableLikeEntity} instances that conform to iceberg semantics around
* locations. This includes both Iceberg tables and Iceberg views.
@@ -51,6 +50,40 @@ public class IcebergTableLikeEntity extends TableLikeEntity {
public static final String LAST_ADMITTED_NOTIFICATION_TIMESTAMP_KEY =
"last-notification-timestamp";
+ /*
+ * The following constants are copied from the TableMetadataParser in Iceberg
+ * They represent the keys used in the table metadata JSON file.
+ */
+
+ public static final String FORMAT_VERSION = "format-version";
+ public static final String TABLE_UUID = "table-uuid";
+ public static final String LOCATION = "location";
+ public static final String LAST_SEQUENCE_NUMBER = "last-sequence-number";
+ public static final String LAST_UPDATED_MILLIS = "last-updated-ms";
+ public static final String LAST_COLUMN_ID = "last-column-id";
+ public static final String SCHEMA = "schema";
+ public static final String SCHEMAS = "schemas";
+ public static final String CURRENT_SCHEMA_ID = "current-schema-id";
+ public static final String PARTITION_SPEC = "partition-spec";
+ public static final String PARTITION_SPECS = "partition-specs";
+ public static final String DEFAULT_SPEC_ID = "default-spec-id";
+ public static final String LAST_PARTITION_ID = "last-partition-id";
+ public static final String DEFAULT_SORT_ORDER_ID = "default-sort-order-id";
+ public static final String SORT_ORDERS = "sort-orders";
+ public static final String PROPERTIES = "properties";
+ public static final String CURRENT_SNAPSHOT_ID = "current-snapshot-id";
+ public static final String REFS = "refs";
+ public static final String SNAPSHOTS = "snapshots";
+ public static final String SNAPSHOT_ID = "snapshot-id";
+ public static final String TIMESTAMP_MS = "timestamp-ms";
+ public static final String SNAPSHOT_LOG = "snapshot-log";
+ public static final String METADATA_FILE = "metadata-file";
+ public static final String METADATA_LOG = "metadata-log";
+ public static final String STATISTICS = "statistics";
+ public static final String PARTITION_STATISTICS = "partition-statistics";
+ public static final String ENCRYPTION_KEYS = "encryption-keys";
+ public static final String NEXT_ROW_ID = "next-row-id";
+
public IcebergTableLikeEntity(PolarisBaseEntity sourceEntity) {
super(sourceEntity);
PolarisEntitySubType subType = getSubType();
diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java
index fe965385b..780fbc944 100644
--- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java
+++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java
@@ -124,7 +124,7 @@ public class EntityWeigherTest {
"location",
"{\"a\": \"b\"}",
Optional.of("{\"c\": \"d\", \"e\": \"f\"}")));
- Assertions.assertThat(preciseWeight).isEqualTo(1255); // :( this is hard-coded
+ Assertions.assertThat(preciseWeight).isEqualTo(1183); // :( this is hard-coded
}
private static Map<String, String> getPropertiesMap(String properties) {
diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java
index 0d1d6979c..f02c61715 100644
--- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java
+++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java
@@ -1566,32 +1566,7 @@ public class IcebergCatalog extends BaseMetastoreViewCatalog
"Generic table with same name already exists: %s", tableIdentifier);
}
}
- Map<String, String> storedProperties = new HashMap<>();
- storedProperties.put("location", metadata.location());
- storedProperties.put("format-version", String.valueOf(metadata.formatVersion()));
- storedProperties.put("table-uuid", metadata.uuid());
- storedProperties.put("current-schema-id", String.valueOf(metadata.currentSchemaId()));
- if (metadata.currentSnapshot() != null) {
- storedProperties.put(
- "current-snapshot-id", String.valueOf(metadata.currentSnapshot().snapshotId()));
- }
- storedProperties.put("last-column-id", String.valueOf(metadata.lastColumnId()));
- storedProperties.put("next-row-id", String.valueOf(metadata.nextRowId()));
- storedProperties.put("last-sequence-number", String.valueOf(metadata.lastSequenceNumber()));
- storedProperties.put("last-updated-ms", String.valueOf(metadata.lastUpdatedMillis()));
- if (metadata.sortOrder() != null) {
- storedProperties.put(
- "default-sort-order-id", String.valueOf(metadata.defaultSortOrderId()));
- }
- if (metadata.spec() != null) {
- storedProperties.put("default-spec-id", String.valueOf(metadata.defaultSpecId()));
- storedProperties.put(
- "last-partition-id", String.valueOf(metadata.lastAssignedPartitionId()));
- }
- if (metadata.currentSnapshot() != null) {
- storedProperties.put(
- "current-snapshot-id", String.valueOf(metadata.currentSnapshot().snapshotId()));
- }
+ Map<String, String> storedProperties = buildTableMetadataPropertiesMap(metadata);
IcebergTableLikeEntity entity =
IcebergTableLikeEntity.of(resolvedPath == null ? null : resolvedPath.getRawLeafEntity());
String existingLocation;
@@ -1743,6 +1718,41 @@ public class IcebergCatalog extends BaseMetastoreViewCatalog
}
}
+ private static Map<String, String> buildTableMetadataPropertiesMap(TableMetadata metadata) {
+ Map<String, String> storedProperties = new HashMap<>();
+ storedProperties.put(IcebergTableLikeEntity.LOCATION, metadata.location());
+ storedProperties.put(
+ IcebergTableLikeEntity.FORMAT_VERSION, String.valueOf(metadata.formatVersion()));
+ storedProperties.put(IcebergTableLikeEntity.TABLE_UUID, metadata.uuid());
+ storedProperties.put(
+ IcebergTableLikeEntity.CURRENT_SCHEMA_ID, String.valueOf(metadata.currentSchemaId()));
+ if (metadata.currentSnapshot() != null) {
+ storedProperties.put(
+ IcebergTableLikeEntity.CURRENT_SNAPSHOT_ID,
+ String.valueOf(metadata.currentSnapshot().snapshotId()));
+ }
+ storedProperties.put(
+ IcebergTableLikeEntity.LAST_COLUMN_ID, String.valueOf(metadata.lastColumnId()));
+ storedProperties.put(IcebergTableLikeEntity.NEXT_ROW_ID, String.valueOf(metadata.nextRowId()));
+ storedProperties.put(
+ IcebergTableLikeEntity.LAST_SEQUENCE_NUMBER, String.valueOf(metadata.lastSequenceNumber()));
+ storedProperties.put(
+ IcebergTableLikeEntity.LAST_UPDATED_MILLIS, String.valueOf(metadata.lastUpdatedMillis()));
+ if (metadata.sortOrder() != null) {
+ storedProperties.put(
+ IcebergTableLikeEntity.DEFAULT_SORT_ORDER_ID,
+ String.valueOf(metadata.defaultSortOrderId()));
+ }
+ if (metadata.spec() != null) {
+ storedProperties.put(
+ IcebergTableLikeEntity.DEFAULT_SPEC_ID, String.valueOf(metadata.defaultSpecId()));
+ storedProperties.put(
+ IcebergTableLikeEntity.LAST_PARTITION_ID,
+ String.valueOf(metadata.lastAssignedPartitionId()));
+ }
+ return storedProperties;
+ }
+
/**
* An implementation of {@link ViewOperations} that integrates with {@link IcebergCatalog}. Much
* of this code was originally copied from {@link org.apache.iceberg.view.BaseViewOperations}.
diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogTest.java
index 5ce54760a..46d7f5518 100644
--- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogTest.java
+++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogTest.java
@@ -112,6 +112,7 @@ import org.apache.polaris.core.entity.PolarisEntitySubType;
import org.apache.polaris.core.entity.PolarisEntityType;
import org.apache.polaris.core.entity.PrincipalEntity;
import org.apache.polaris.core.entity.TaskEntity;
+import org.apache.polaris.core.entity.table.IcebergTableLikeEntity;
import org.apache.polaris.core.exceptions.CommitConflictException;
import org.apache.polaris.core.persistence.MetaStoreManagerFactory;
import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
@@ -2318,37 +2319,43 @@ public abstract class AbstractIcebergCatalogTest extends CatalogTests<IcebergCat
.asInstanceOf(InstanceOfAssertFactories.map(String.class, String.class))
.containsEntry(NamespaceEntity.PARENT_NAMESPACE_KEY, NS.toString())
.containsEntry(
- "current-snapshot-id", String.valueOf(afterAppend.currentSnapshot().snapshotId()))
- .containsEntry("location", afterAppend.location())
- .containsEntry("table-uuid", afterAppend.uuid().toString())
- .containsEntry("current-schema-id", String.valueOf(afterAppend.schema().schemaId()))
+ IcebergTableLikeEntity.CURRENT_SNAPSHOT_ID,
+ String.valueOf(afterAppend.currentSnapshot().snapshotId()))
+ .containsEntry(IcebergTableLikeEntity.LOCATION, afterAppend.location())
+ .containsEntry(IcebergTableLikeEntity.TABLE_UUID, afterAppend.uuid().toString())
.containsEntry(
- "last-column-id",
+ IcebergTableLikeEntity.CURRENT_SCHEMA_ID,
+ String.valueOf(afterAppend.schema().schemaId()))
+ .containsEntry(
+ IcebergTableLikeEntity.LAST_COLUMN_ID,
afterAppend.schema().columns().stream()
.max(Comparator.comparing(Types.NestedField::fieldId))
.map(Types.NestedField::fieldId)
.orElse(0)
.toString())
.containsEntry(
- "last-sequence-number", String.valueOf(afterAppend.currentSnapshot().sequenceNumber()));
+ IcebergTableLikeEntity.LAST_SEQUENCE_NUMBER,
+ String.valueOf(afterAppend.currentSnapshot().sequenceNumber()));
catalog.loadTable(TABLE).refresh();
catalog.loadTable(TABLE).newFastAppend().appendFile(FILE_B).commit();
validatePropertiesUpdated(
schemaResult,
- "current-snapshot-id",
+ IcebergTableLikeEntity.CURRENT_SNAPSHOT_ID,
tbl -> String.valueOf(tbl.currentSnapshot().snapshotId()));
catalog.loadTable(TABLE).refresh();
catalog.loadTable(TABLE).updateSchema().addColumn("new_col", Types.LongType.get()).commit();
validatePropertiesUpdated(
- schemaResult, "current-schema-id", tbl -> String.valueOf(tbl.schema().schemaId()));
+ schemaResult,
+ IcebergTableLikeEntity.CURRENT_SCHEMA_ID,
+ tbl -> String.valueOf(tbl.schema().schemaId()));
catalog.loadTable(TABLE).refresh();
catalog.loadTable(TABLE).replaceSortOrder().desc("new_col", NullOrder.NULLS_FIRST).commit();
validatePropertiesUpdated(
schemaResult,
- "default-sort-order-id",
+ IcebergTableLikeEntity.DEFAULT_SORT_ORDER_ID,
table -> String.valueOf(table.sortOrder().orderId()));
}