You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/09/26 14:03:41 UTC

(spark) branch master updated: [SPARK-46679][SQL] Fix for SparkUnsupportedOperationException Not found an encoder of the type T, when using Parameterized class

This is an automated email from the ASF dual-hosted git repository.

wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 65b9da5f3083 [SPARK-46679][SQL] Fix for SparkUnsupportedOperationException Not found an encoder of the type T, when using Parameterized class
65b9da5f3083 is described below

commit 65b9da5f30839dc8569196a0a8b24557f49584dd
Author: Szehon Ho <sz...@gmail.com>
AuthorDate: Fri Sep 26 22:03:20 2025 +0800

    [SPARK-46679][SQL] Fix for SparkUnsupportedOperationException Not found an encoder of the type T, when using Parameterized class
    
    This pr is a rebase/continue of https://github.com/apache/spark/pull/48304/files, which was unexpectedly closed even though the issue is still there.
    
    ### What changes were proposed in this pull request?
    That pr fixes a bug in JavaTypeInference.encoderFor.  In the parameterizableType case, it overrides the existing map of parameterizable type -> real type.  So any nested parameterizable type will fail.
    
    ### Why are the changes needed?
    Fix nested parameterization cases for Java Encoder
    
    ### Does this PR introduce _any_ user-facing change?
    No
    
    ### How was this patch tested?
    The original pr adds unit test to JavaTypeInferenceSuite and JavaDatasetSuite.  The test from the original pr are simplified a little to make it obvious which one is wrapping.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    No
    
    Closes #52444 from szehon-ho/parameterized_encoders.
    
    Authored-by: Szehon Ho <sz...@gmail.com>
    Signed-off-by: Wenchen Fan <we...@databricks.com>
---
 .../spark/sql/catalyst/JavaTypeInference.scala     |  4 +-
 .../spark/sql/catalyst/JavaTypeInferenceBeans.java | 56 ++++++++++++++++++++++
 .../sql/catalyst/JavaTypeInferenceSuite.scala      | 24 +++++++++-
 .../org/apache/spark/sql/JavaDatasetSuite.java     | 30 ++++++++++--
 4 files changed, 108 insertions(+), 6 deletions(-)

diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
index 327ec5aa344c..906e6419b360 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
@@ -139,7 +139,9 @@ object JavaTypeInference {
       encoderFor(typeVariables(tv), seenTypeSet, typeVariables)
 
     case pt: ParameterizedType =>
-      encoderFor(pt.getRawType, seenTypeSet, JavaTypeUtils.getTypeArguments(pt).asScala.toMap)
+      val newTvs = JavaTypeUtils.getTypeArguments(pt).asScala.toMap
+      val allTvs = typeVariables ++ newTvs
+      encoderFor(pt.getRawType, seenTypeSet, allTvs)
 
     case c: Class[_] =>
       if (seenTypeSet.contains(c)) {
diff --git a/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/JavaTypeInferenceBeans.java b/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/JavaTypeInferenceBeans.java
index cc3540717ee7..3e4cade6c8d1 100644
--- a/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/JavaTypeInferenceBeans.java
+++ b/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/JavaTypeInferenceBeans.java
@@ -78,5 +78,61 @@ public class JavaTypeInferenceBeans {
   static class JavaBeanWithGenericHierarchy extends JavaBeanWithGenericsABC<Integer> {
 
   }
+
+  // SPARK-46679: Test classes for nested parameterized types with multi-level inheritance
+  static class Foo<T> {
+    private T t;
+
+    public T getT() {
+      return t;
+    }
+
+    public void setT(T t) {
+      this.t = t;
+    }
+  }
+
+  static class FooWrapper<U> {
+    private Foo<U> foo;
+
+    public Foo<U> getFoo() {
+      return foo;
+    }
+
+    public void setFoo(Foo<U> foo) {
+      this.foo = foo;
+    }
+  }
+
+  static class StringFooWrapper extends FooWrapper<String> {
+  }
+
+  // SPARK-46679: Additional test classes for same type variable names at different levels
+  static class StringBarWrapper extends BarWrapper<String> {
+  }
+
+  static class BarWrapper<T> {
+    private Bar<T> bar;
+
+    public Bar<T> getBar() {
+      return bar;
+    }
+
+    public void setBar(Bar<T> bar) {
+      this.bar = bar;
+    }
+  }
+
+  static class Bar<T> {
+    private T t;
+
+    public T getT() {
+      return t;
+    }
+
+    public void setT(T t) {
+      this.t = t;
+    }
+  }
 }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/JavaTypeInferenceSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/JavaTypeInferenceSuite.scala
index c785c71428ca..1b6f7defd3fe 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/JavaTypeInferenceSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/JavaTypeInferenceSuite.scala
@@ -24,7 +24,7 @@ import scala.beans.{BeanProperty, BooleanBeanProperty}
 import scala.reflect.{classTag, ClassTag}
 
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.JavaTypeInferenceBeans.{JavaBeanWithGenericBase, JavaBeanWithGenericHierarchy, JavaBeanWithGenericsABC}
+import org.apache.spark.sql.catalyst.JavaTypeInferenceBeans.{Bar, Foo, JavaBeanWithGenericBase, JavaBeanWithGenericHierarchy, JavaBeanWithGenericsABC, StringBarWrapper, StringFooWrapper}
 import org.apache.spark.sql.catalyst.encoders.{AgnosticEncoder, UDTCaseClass, UDTForCaseClass}
 import org.apache.spark.sql.catalyst.encoders.AgnosticEncoders._
 import org.apache.spark.sql.types.{DecimalType, MapType, Metadata, StringType, StructField, StructType}
@@ -279,4 +279,26 @@ class JavaTypeInferenceSuite extends SparkFunSuite {
       ))
     assert(encoder === expected)
   }
+
+  test("SPARK-46679: resolve generics with multi-level inheritance") {
+    val encoder = JavaTypeInference.encoderFor(classOf[StringFooWrapper])
+    val expected = JavaBeanEncoder(ClassTag(classOf[StringFooWrapper]), Seq(
+      encoderField("foo", JavaBeanEncoder(
+        ClassTag(classOf[Foo[String]]),
+        Seq(encoderField("t", StringEncoder))
+      ))
+    ))
+    assert(encoder === expected)
+  }
+
+  test("SPARK-46679: resolve generics with multi-level inheritance same type names") {
+    val encoder = JavaTypeInference.encoderFor(classOf[StringBarWrapper])
+    val expected = JavaBeanEncoder(ClassTag(classOf[StringBarWrapper]), Seq(
+      encoderField("bar", JavaBeanEncoder(
+        ClassTag(classOf[Bar[String]]),
+        Seq(encoderField("t", StringEncoder))
+      ))
+    ))
+    assert(encoder === expected)
+  }
 }
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java
index 33afafbf2560..83f477ecbe7b 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java
@@ -928,6 +928,7 @@ public class JavaDatasetSuite implements Serializable {
     private List<Long> f;
     private Map<Integer, String> g;
     private Map<List<Long>, Map<String, String>> h;
+    private List<List<Long>> i;
 
     public boolean isA() {
       return a;
@@ -993,6 +994,14 @@ public class JavaDatasetSuite implements Serializable {
       this.h = h;
     }
 
+    public List<List<Long>> getI() {
+      return i;
+    }
+
+    public void setI(List<List<Long>> i) {
+      this.i = i;
+    }
+
     @Override
     public boolean equals(Object o) {
       if (this == o) return true;
@@ -1007,7 +1016,8 @@ public class JavaDatasetSuite implements Serializable {
       if (!e.equals(that.e)) return false;
       if (!f.equals(that.f)) return false;
       if (!g.equals(that.g)) return false;
-      return h.equals(that.h);
+      if (!h.equals(that.h)) return false;
+      return i.equals(that.i);
 
     }
 
@@ -1021,6 +1031,7 @@ public class JavaDatasetSuite implements Serializable {
       result = 31 * result + f.hashCode();
       result = 31 * result + g.hashCode();
       result = 31 * result + h.hashCode();
+      result = 31 * result + i.hashCode();
       return result;
     }
   }
@@ -1110,6 +1121,10 @@ public class JavaDatasetSuite implements Serializable {
     Map<List<Long>, Map<String, String>> complexMap1 = new HashMap<>();
     complexMap1.put(Arrays.asList(1L, 2L), nestedMap1);
     obj1.setH(complexMap1);
+    List<Long> nestedList1 = List.of(1L, 2L, 3L);
+    List<Long> nestedList2 = List.of(4L, 5L, 6L);
+    List<List<Long>> complexList1 = List.of(nestedList1, nestedList2);
+    obj1.setI(complexList1);
 
     SimpleJavaBean obj2 = new SimpleJavaBean();
     obj2.setA(false);
@@ -1128,6 +1143,10 @@ public class JavaDatasetSuite implements Serializable {
     Map<List<Long>, Map<String, String>> complexMap2 = new HashMap<>();
     complexMap2.put(Arrays.asList(3L, 4L), nestedMap2);
     obj2.setH(complexMap2);
+    List<Long> nestedList3 = List.of(1L, 2L, 7L);
+    List<Long> nestedList4 = List.of(4L, 5L, 8L);
+    List<List<Long>> complexList2 = List.of(nestedList3, nestedList4);
+    obj2.setI(complexList2);
 
     List<SimpleJavaBean> data = Arrays.asList(obj1, obj2);
     Dataset<SimpleJavaBean> ds = spark.createDataset(data, Encoders.bean(SimpleJavaBean.class));
@@ -1148,7 +1167,8 @@ public class JavaDatasetSuite implements Serializable {
       Arrays.asList("a", "b"),
       Arrays.asList(100L, null, 200L),
       map1,
-      complexMap1});
+      complexMap1,
+      complexList1});
     Row row2 = new GenericRow(new Object[]{
       false,
       30,
@@ -1157,7 +1177,8 @@ public class JavaDatasetSuite implements Serializable {
       Arrays.asList("x", "y"),
       Arrays.asList(300L, null, 400L),
       map2,
-      complexMap2});
+      complexMap2,
+      complexList2});
     StructType schema = new StructType()
       .add("a", BooleanType, false)
       .add("b", IntegerType, false)
@@ -1166,7 +1187,8 @@ public class JavaDatasetSuite implements Serializable {
       .add("e", createArrayType(StringType))
       .add("f", createArrayType(LongType))
       .add("g", createMapType(IntegerType, StringType))
-      .add("h",createMapType(createArrayType(LongType), createMapType(StringType, StringType)));
+      .add("h", createMapType(createArrayType(LongType), createMapType(StringType, StringType)))
+      .add("i", createArrayType(createArrayType(LongType)));
     Dataset<SimpleJavaBean> ds3 = spark.createDataFrame(Arrays.asList(row1, row2), schema)
       .as(Encoders.bean(SimpleJavaBean.class));
     Assertions.assertEquals(data, ds3.collectAsList());


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org