You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/12/11 08:41:29 UTC
spark git commit: [SPARK-4791] [sql] Infer schema from case class
with multiple constructors
Repository: spark
Updated Branches:
refs/heads/master 57d37f9c7 -> 2a5b5fd4c
[SPARK-4791] [sql] Infer schema from case class with multiple constructors
Modified ScalaReflection.schemaFor to take primary constructor of Product when there are multiple constructors. Added test to suite which failed before but works now.
Needed for [https://github.com/apache/spark/pull/3637]
CC: marmbrus
Author: Joseph K. Bradley <jo...@databricks.com>
Closes #3646 from jkbradley/sql-reflection and squashes the following commits:
796b2e4 [Joseph K. Bradley] Modified ScalaReflection.schemaFor to take primary constructor of Product when there are multiple constructors. Added test to suite which failed before but works now.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2a5b5fd4
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2a5b5fd4
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2a5b5fd4
Branch: refs/heads/master
Commit: 2a5b5fd4ccf28fab5b7e32a54170be92d5d23ba6
Parents: 57d37f9
Author: Joseph K. Bradley <jo...@databricks.com>
Authored: Wed Dec 10 23:41:15 2014 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Wed Dec 10 23:41:15 2014 -0800
----------------------------------------------------------------------
.../apache/spark/sql/catalyst/ScalaReflection.scala | 14 +++++++++++++-
.../spark/sql/catalyst/ScalaReflectionSuite.scala | 14 ++++++++++++++
2 files changed, 27 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/2a5b5fd4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index 71034c2..2cf241d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -118,7 +118,19 @@ trait ScalaReflection {
case t if t <:< typeOf[Product] =>
val formalTypeArgs = t.typeSymbol.asClass.typeParams
val TypeRef(_, _, actualTypeArgs) = t
- val params = t.member(nme.CONSTRUCTOR).asMethod.paramss
+ val constructorSymbol = t.member(nme.CONSTRUCTOR)
+ val params = if (constructorSymbol.isMethod) {
+ constructorSymbol.asMethod.paramss
+ } else {
+ // Find the primary constructor, and use its parameter ordering.
+ val primaryConstructorSymbol: Option[Symbol] = constructorSymbol.asTerm.alternatives.find(
+ s => s.isMethod && s.asMethod.isPrimaryConstructor)
+ if (primaryConstructorSymbol.isEmpty) {
+ sys.error("Internal SQL error: Product object did not have a primary constructor.")
+ } else {
+ primaryConstructorSymbol.get.asMethod.paramss
+ }
+ }
Schema(StructType(
params.head.map { p =>
val Schema(dataType, nullable) =
http://git-wip-us.apache.org/repos/asf/spark/blob/2a5b5fd4/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
index ddc3d44..7be24be 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
@@ -68,6 +68,10 @@ case class ComplexData(
case class GenericData[A](
genericField: A)
+case class MultipleConstructorsData(a: Int, b: String, c: Double) {
+ def this(b: String, a: Int) = this(a, b, c = 1.0)
+}
+
class ScalaReflectionSuite extends FunSuite {
import ScalaReflection._
@@ -253,4 +257,14 @@ class ScalaReflectionSuite extends FunSuite {
Row(1, 1, 1, 1, 1, 1, true))
assert(convertToCatalyst(data, dataType) === convertedData)
}
+
+ test("infer schema from case class with multiple constructors") {
+ val dataType = schemaFor[MultipleConstructorsData].dataType
+ dataType match {
+ case s: StructType =>
+ // Schema should have order: a: Int, b: String, c: Double
+ assert(s.fieldNames === Seq("a", "b", "c"))
+ assert(s.fields.map(_.dataType) === Seq(IntegerType, StringType, DoubleType))
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org