You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by fo...@apache.org on 2025/05/07 07:05:48 UTC

(iceberg) branch main updated: Core: Disallow creation of invalid PartitionSpec (#12887)

This is an automated email from the ASF dual-hosted git repository.

fokko pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/main by this push:
     new 9fb80b7167 Core: Disallow creation of invalid PartitionSpec (#12887)
9fb80b7167 is described below

commit 9fb80b7167e58a2daef80f4a1a09f223b870c030
Author: Devin Smith <de...@deephaven.io>
AuthorDate: Wed May 7 00:05:42 2025 -0700

    Core: Disallow creation of invalid PartitionSpec (#12887)
    
    * Core: Disallow creation of invalid PartitionSpec
    
    Closes #12870
    
    * review response
---
 .../java/org/apache/iceberg/PartitionSpec.java     |  11 +++
 .../iceberg/TestPartitionSpecValidation.java       | 101 +++++++++++++++++++++
 2 files changed, 112 insertions(+)

diff --git a/api/src/main/java/org/apache/iceberg/PartitionSpec.java b/api/src/main/java/org/apache/iceberg/PartitionSpec.java
index 9b74893f18..80985dcb79 100644
--- a/api/src/main/java/org/apache/iceberg/PartitionSpec.java
+++ b/api/src/main/java/org/apache/iceberg/PartitionSpec.java
@@ -40,6 +40,7 @@ import org.apache.iceberg.transforms.Transform;
 import org.apache.iceberg.transforms.Transforms;
 import org.apache.iceberg.transforms.UnknownTransform;
 import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.TypeUtil;
 import org.apache.iceberg.types.Types;
 import org.apache.iceberg.types.Types.StructType;
 
@@ -624,6 +625,7 @@ public class PartitionSpec implements Serializable {
   }
 
   static void checkCompatibility(PartitionSpec spec, Schema schema) {
+    final Map<Integer, Integer> parents = TypeUtil.indexParents(schema.asStruct());
     for (PartitionField field : spec.fields) {
       Type sourceType = schema.findType(field.sourceId());
       Transform<?, ?> transform = field.transform();
@@ -644,6 +646,15 @@ public class PartitionSpec implements Serializable {
             "Invalid source type %s for transform: %s",
             sourceType,
             transform);
+        // The only valid parent types for a PartitionField are StructTypes. This must be checked
+        // recursively.
+        Integer parentId = parents.get(field.sourceId());
+        while (parentId != null) {
+          Type parentType = schema.findType(parentId);
+          ValidationException.check(
+              parentType.isStructType(), "Invalid partition field parent: %s", parentType);
+          parentId = parents.get(parentId);
+        }
       }
     }
   }
diff --git a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java
index ee71d39bb2..b8e16a9ee4 100644
--- a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java
+++ b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java
@@ -340,4 +340,105 @@ public class TestPartitionSpecValidation {
       {10, "unknown_partition1", "Invalid source type unknown for transform: bucket[5]"}
     };
   }
+
+  @Test
+  void testSourceIdNotFound() {
+    assertThatThrownBy(
+            () ->
+                PartitionSpec.builderFor(SCHEMA)
+                    .add(99, 1000, "Test", Transforms.identity())
+                    .build())
+        .isInstanceOf(ValidationException.class)
+        .hasMessage("Cannot find source column for partition field: 1000: Test: identity(99)");
+  }
+
+  @Test
+  void testPartitionFieldInStruct() {
+    final Schema schema =
+        new Schema(
+            NestedField.required(SCHEMA.highestFieldId() + 1, "MyStruct", SCHEMA.asStruct()));
+    PartitionSpec.builderFor(schema).identity("MyStruct.id").build();
+  }
+
+  @Test
+  void testPartitionFieldInStructInStruct() {
+    final Schema schema =
+        new Schema(
+            NestedField.optional(
+                SCHEMA.highestFieldId() + 2,
+                "Outer",
+                Types.StructType.of(
+                    NestedField.required(
+                        SCHEMA.highestFieldId() + 1, "Inner", SCHEMA.asStruct()))));
+    PartitionSpec.builderFor(schema).identity("Outer.Inner.id").build();
+  }
+
+  @Test
+  void testPartitionFieldInList() {
+    final Schema schema =
+        new Schema(
+            NestedField.required(
+                2, "MyList", Types.ListType.ofRequired(1, Types.IntegerType.get())));
+    assertThatThrownBy(() -> PartitionSpec.builderFor(schema).identity("MyList.element").build())
+        .isInstanceOf(ValidationException.class)
+        .hasMessage("Invalid partition field parent: list<int>");
+  }
+
+  @Test
+  void testPartitionFieldInStructInList() {
+    final Schema schema =
+        new Schema(
+            NestedField.required(
+                3,
+                "MyList",
+                Types.ListType.ofRequired(
+                    2,
+                    Types.StructType.of(NestedField.optional(1, "Foo", Types.IntegerType.get())))));
+    assertThatThrownBy(
+            () -> PartitionSpec.builderFor(schema).identity("MyList.element.Foo").build())
+        .isInstanceOf(ValidationException.class)
+        .hasMessage("Invalid partition field parent: list<struct<1: Foo: optional int>>");
+  }
+
+  @Test
+  void testPartitionFieldInMap() {
+    final Schema schema =
+        new Schema(
+            NestedField.required(
+                3,
+                "MyMap",
+                Types.MapType.ofRequired(1, 2, Types.IntegerType.get(), Types.IntegerType.get())));
+
+    assertThatThrownBy(() -> PartitionSpec.builderFor(schema).identity("MyMap.key").build())
+        .isInstanceOf(ValidationException.class)
+        .hasMessage("Invalid partition field parent: map<int, int>");
+
+    assertThatThrownBy(() -> PartitionSpec.builderFor(schema).identity("MyMap.value").build())
+        .isInstanceOf(ValidationException.class)
+        .hasMessage("Invalid partition field parent: map<int, int>");
+  }
+
+  @Test
+  void testPartitionFieldInStructInMap() {
+    final Schema schema =
+        new Schema(
+            NestedField.required(
+                5,
+                "MyMap",
+                Types.MapType.ofRequired(
+                    3,
+                    4,
+                    Types.StructType.of(NestedField.optional(1, "Foo", Types.IntegerType.get())),
+                    Types.StructType.of(NestedField.optional(2, "Bar", Types.IntegerType.get())))));
+
+    assertThatThrownBy(() -> PartitionSpec.builderFor(schema).identity("MyMap.key.Foo").build())
+        .isInstanceOf(ValidationException.class)
+        .hasMessage(
+            "Invalid partition field parent: map<struct<1: Foo: optional int>, struct<2: Bar: optional int>>");
+
+    assertThatThrownBy(() -> PartitionSpec.builderFor(schema).identity("MyMap.value.Bar").build())
+        .isInstanceOf(ValidationException.class)
+        .hasMessage(
+            "Invalid partition field parent: map<struct<1: Foo: optional int>, struct<2: Bar: optional int>>");
+  }
 }