You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by hu...@apache.org on 2026/06/11 22:02:23 UTC
(iceberg) branch 1.11.x updated: Core: Fix SerializableTable.sortOrders() throwing on historical sort orders with dropped fields (#16519) (#16521) (#16779)
This is an automated email from the ASF dual-hosted git repository.
huaxingao pushed a commit to branch 1.11.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/1.11.x by this push:
new e706d55615 Core: Fix SerializableTable.sortOrders() throwing on historical sort orders with dropped fields (#16519) (#16521) (#16779)
e706d55615 is described below
commit e706d55615b0fe6d69137349bf2ea483f711780d
Author: Neelesh Salian <ns...@users.noreply.github.com>
AuthorDate: Thu Jun 11 15:02:15 2026 -0700
Core: Fix SerializableTable.sortOrders() throwing on historical sort orders with dropped fields (#16519) (#16521) (#16779)
Co-authored-by: Yong Zheng <yo...@gmail.com>
---
.../main/java/org/apache/iceberg/SerializableTable.java | 5 ++++-
.../main/java/org/apache/iceberg/SortOrderParser.java | 4 ++++
.../apache/iceberg/hadoop/TestTableSerialization.java | 16 ++++++++++++++++
3 files changed, 24 insertions(+), 1 deletion(-)
diff --git a/core/src/main/java/org/apache/iceberg/SerializableTable.java b/core/src/main/java/org/apache/iceberg/SerializableTable.java
index a26fff1fd5..5b4cd0e553 100644
--- a/core/src/main/java/org/apache/iceberg/SerializableTable.java
+++ b/core/src/main/java/org/apache/iceberg/SerializableTable.java
@@ -58,6 +58,7 @@ public class SerializableTable implements Table, HasTableOperations, Serializabl
private final int defaultSpecId;
private final Map<Integer, String> specAsJsonMap;
private final String sortOrderAsJson;
+ private final int defaultSortOrderId;
private final Map<Integer, String> sortOrderAsJsonMap;
private final FileIO io;
private final EncryptionManager encryption;
@@ -83,6 +84,7 @@ public class SerializableTable implements Table, HasTableOperations, Serializabl
Map<Integer, PartitionSpec> specs = table.specs();
specs.forEach((specId, spec) -> specAsJsonMap.put(specId, PartitionSpecParser.toJson(spec)));
this.sortOrderAsJson = SortOrderParser.toJson(table.sortOrder());
+ this.defaultSortOrderId = table.sortOrder().orderId();
this.sortOrderAsJsonMap = Maps.newHashMap();
table
.sortOrders()
@@ -253,7 +255,8 @@ public class SerializableTable implements Table, HasTableOperations, Serializabl
ImmutableMap.Builder<Integer, SortOrder> sortOrders =
ImmutableMap.builderWithExpectedSize(sortOrderAsJsonMap.size());
sortOrderAsJsonMap.forEach(
- (id, json) -> sortOrders.put(id, SortOrderParser.fromJson(schema(), json)));
+ (id, json) ->
+ sortOrders.put(id, SortOrderParser.fromJson(schema(), json, defaultSortOrderId)));
this.lazySortOrders = sortOrders.build();
} else if (lazySortOrders == null) {
this.lazySortOrders = lazyTable.sortOrders();
diff --git a/core/src/main/java/org/apache/iceberg/SortOrderParser.java b/core/src/main/java/org/apache/iceberg/SortOrderParser.java
index 31307cf9dc..53d7e5090c 100644
--- a/core/src/main/java/org/apache/iceberg/SortOrderParser.java
+++ b/core/src/main/java/org/apache/iceberg/SortOrderParser.java
@@ -112,6 +112,10 @@ public class SortOrderParser {
return fromJson(json).bind(schema);
}
+ public static SortOrder fromJson(Schema schema, String json, int defaultSortOrderId) {
+ return JsonUtil.parse(json, node -> fromJson(schema, node, defaultSortOrderId));
+ }
+
public static SortOrder fromJson(Schema schema, JsonNode json, int defaultSortOrderId) {
UnboundSortOrder unboundSortOrder = fromJson(json);
diff --git a/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java b/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java
index 5103e2e9be..ece9b24af3 100644
--- a/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java
+++ b/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java
@@ -79,6 +79,22 @@ public class TestTableSerialization extends HadoopTableTestBase {
.isEqualTo(TableUtil.metadataFileLocation(table));
}
+ @Test
+ public void testSerializableTableSortOrdersWithDroppedColumn()
+ throws IOException, ClassNotFoundException {
+ table.updateSchema().addColumn("ts", Types.LongType.get()).commit();
+ table.replaceSortOrder().asc("id").asc("ts").commit();
+
+ // historical sort order 1 still references "ts" after the column is dropped
+ table.replaceSortOrder().asc("id").commit();
+ table.updateSchema().deleteColumn("ts").commit();
+
+ TestHelpers.assertSerializedAndLoadedMetadata(table, TestHelpers.roundTripSerialize(table));
+ Table serializableTable = SerializableTable.copyOf(table);
+ TestHelpers.assertSerializedAndLoadedMetadata(
+ serializableTable, TestHelpers.KryoHelpers.roundTripSerialize(serializableTable));
+ }
+
@Test
public void testSerializableTableWithSnapshot() throws IOException, ClassNotFoundException {
table.newAppend().appendFile(FILE_A).commit();