You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/11/04 03:04:55 UTC
git commit: [SQL] Convert arguments to Scala UDFs
Repository: spark
Updated Branches:
refs/heads/master 28128150e -> 15b58a223
[SQL] Convert arguments to Scala UDFs
Author: Michael Armbrust <mi...@databricks.com>
Closes #3077 from marmbrus/udfsWithUdts and squashes the following commits:
34b5f27 [Michael Armbrust] style
504adef [Michael Armbrust] Convert arguments to Scala UDFs
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/15b58a22
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/15b58a22
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/15b58a22
Branch: refs/heads/master
Commit: 15b58a2234ab7ba30c9c0cbb536177a3c725e350
Parents: 2812815
Author: Michael Armbrust <mi...@databricks.com>
Authored: Mon Nov 3 18:04:51 2014 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Mon Nov 3 18:04:51 2014 -0800
----------------------------------------------------------------------
.../sql/catalyst/expressions/ScalaUdf.scala | 560 ++++++++++---------
.../apache/spark/sql/UserDefinedTypeSuite.scala | 18 +-
2 files changed, 316 insertions(+), 262 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/15b58a22/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUdf.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUdf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUdf.scala
index fa1786e..18c96da 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUdf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUdf.scala
@@ -34,320 +34,366 @@ case class ScalaUdf(function: AnyRef, dataType: DataType, children: Seq[Expressi
override def toString = s"scalaUDF(${children.mkString(",")})"
+ // scalastyle:off
+
/** This method has been generated by this script
(1 to 22).map { x =>
val anys = (1 to x).map(x => "Any").reduce(_ + ", " + _)
- val evals = (0 to x - 1).map(x => s"children($x).eval(input)").reduce(_ + ",\n " + _)
+ val evals = (0 to x - 1).map(x => s" ScalaReflection.convertToScala(children($x).eval(input), children($x).dataType)").reduce(_ + ",\n " + _)
s"""
case $x =>
function.asInstanceOf[($anys) => Any](
- $evals)
+ $evals)
"""
- }
+ }.foreach(println)
*/
- // scalastyle:off
override def eval(input: Row): Any = {
val result = children.size match {
case 0 => function.asInstanceOf[() => Any]()
- case 1 => function.asInstanceOf[(Any) => Any](children(0).eval(input))
+ case 1 =>
+ function.asInstanceOf[(Any) => Any](
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType))
+
+
case 2 =>
function.asInstanceOf[(Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType))
+
+
case 3 =>
function.asInstanceOf[(Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType))
+
+
case 4 =>
function.asInstanceOf[(Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType))
+
+
case 5 =>
function.asInstanceOf[(Any, Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input),
- children(4).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType),
+ ScalaReflection.convertToScala(children(4).eval(input), children(4).dataType))
+
+
case 6 =>
function.asInstanceOf[(Any, Any, Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input),
- children(4).eval(input),
- children(5).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType),
+ ScalaReflection.convertToScala(children(4).eval(input), children(4).dataType),
+ ScalaReflection.convertToScala(children(5).eval(input), children(5).dataType))
+
+
case 7 =>
function.asInstanceOf[(Any, Any, Any, Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input),
- children(4).eval(input),
- children(5).eval(input),
- children(6).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType),
+ ScalaReflection.convertToScala(children(4).eval(input), children(4).dataType),
+ ScalaReflection.convertToScala(children(5).eval(input), children(5).dataType),
+ ScalaReflection.convertToScala(children(6).eval(input), children(6).dataType))
+
+
case 8 =>
function.asInstanceOf[(Any, Any, Any, Any, Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input),
- children(4).eval(input),
- children(5).eval(input),
- children(6).eval(input),
- children(7).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType),
+ ScalaReflection.convertToScala(children(4).eval(input), children(4).dataType),
+ ScalaReflection.convertToScala(children(5).eval(input), children(5).dataType),
+ ScalaReflection.convertToScala(children(6).eval(input), children(6).dataType),
+ ScalaReflection.convertToScala(children(7).eval(input), children(7).dataType))
+
+
case 9 =>
function.asInstanceOf[(Any, Any, Any, Any, Any, Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input),
- children(4).eval(input),
- children(5).eval(input),
- children(6).eval(input),
- children(7).eval(input),
- children(8).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType),
+ ScalaReflection.convertToScala(children(4).eval(input), children(4).dataType),
+ ScalaReflection.convertToScala(children(5).eval(input), children(5).dataType),
+ ScalaReflection.convertToScala(children(6).eval(input), children(6).dataType),
+ ScalaReflection.convertToScala(children(7).eval(input), children(7).dataType),
+ ScalaReflection.convertToScala(children(8).eval(input), children(8).dataType))
+
+
case 10 =>
function.asInstanceOf[(Any, Any, Any, Any, Any, Any, Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input),
- children(4).eval(input),
- children(5).eval(input),
- children(6).eval(input),
- children(7).eval(input),
- children(8).eval(input),
- children(9).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType),
+ ScalaReflection.convertToScala(children(4).eval(input), children(4).dataType),
+ ScalaReflection.convertToScala(children(5).eval(input), children(5).dataType),
+ ScalaReflection.convertToScala(children(6).eval(input), children(6).dataType),
+ ScalaReflection.convertToScala(children(7).eval(input), children(7).dataType),
+ ScalaReflection.convertToScala(children(8).eval(input), children(8).dataType),
+ ScalaReflection.convertToScala(children(9).eval(input), children(9).dataType))
+
+
case 11 =>
function.asInstanceOf[(Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input),
- children(4).eval(input),
- children(5).eval(input),
- children(6).eval(input),
- children(7).eval(input),
- children(8).eval(input),
- children(9).eval(input),
- children(10).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType),
+ ScalaReflection.convertToScala(children(4).eval(input), children(4).dataType),
+ ScalaReflection.convertToScala(children(5).eval(input), children(5).dataType),
+ ScalaReflection.convertToScala(children(6).eval(input), children(6).dataType),
+ ScalaReflection.convertToScala(children(7).eval(input), children(7).dataType),
+ ScalaReflection.convertToScala(children(8).eval(input), children(8).dataType),
+ ScalaReflection.convertToScala(children(9).eval(input), children(9).dataType),
+ ScalaReflection.convertToScala(children(10).eval(input), children(10).dataType))
+
+
case 12 =>
function.asInstanceOf[(Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input),
- children(4).eval(input),
- children(5).eval(input),
- children(6).eval(input),
- children(7).eval(input),
- children(8).eval(input),
- children(9).eval(input),
- children(10).eval(input),
- children(11).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType),
+ ScalaReflection.convertToScala(children(4).eval(input), children(4).dataType),
+ ScalaReflection.convertToScala(children(5).eval(input), children(5).dataType),
+ ScalaReflection.convertToScala(children(6).eval(input), children(6).dataType),
+ ScalaReflection.convertToScala(children(7).eval(input), children(7).dataType),
+ ScalaReflection.convertToScala(children(8).eval(input), children(8).dataType),
+ ScalaReflection.convertToScala(children(9).eval(input), children(9).dataType),
+ ScalaReflection.convertToScala(children(10).eval(input), children(10).dataType),
+ ScalaReflection.convertToScala(children(11).eval(input), children(11).dataType))
+
+
case 13 =>
function.asInstanceOf[(Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input),
- children(4).eval(input),
- children(5).eval(input),
- children(6).eval(input),
- children(7).eval(input),
- children(8).eval(input),
- children(9).eval(input),
- children(10).eval(input),
- children(11).eval(input),
- children(12).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType),
+ ScalaReflection.convertToScala(children(4).eval(input), children(4).dataType),
+ ScalaReflection.convertToScala(children(5).eval(input), children(5).dataType),
+ ScalaReflection.convertToScala(children(6).eval(input), children(6).dataType),
+ ScalaReflection.convertToScala(children(7).eval(input), children(7).dataType),
+ ScalaReflection.convertToScala(children(8).eval(input), children(8).dataType),
+ ScalaReflection.convertToScala(children(9).eval(input), children(9).dataType),
+ ScalaReflection.convertToScala(children(10).eval(input), children(10).dataType),
+ ScalaReflection.convertToScala(children(11).eval(input), children(11).dataType),
+ ScalaReflection.convertToScala(children(12).eval(input), children(12).dataType))
+
+
case 14 =>
function.asInstanceOf[(Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input),
- children(4).eval(input),
- children(5).eval(input),
- children(6).eval(input),
- children(7).eval(input),
- children(8).eval(input),
- children(9).eval(input),
- children(10).eval(input),
- children(11).eval(input),
- children(12).eval(input),
- children(13).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType),
+ ScalaReflection.convertToScala(children(4).eval(input), children(4).dataType),
+ ScalaReflection.convertToScala(children(5).eval(input), children(5).dataType),
+ ScalaReflection.convertToScala(children(6).eval(input), children(6).dataType),
+ ScalaReflection.convertToScala(children(7).eval(input), children(7).dataType),
+ ScalaReflection.convertToScala(children(8).eval(input), children(8).dataType),
+ ScalaReflection.convertToScala(children(9).eval(input), children(9).dataType),
+ ScalaReflection.convertToScala(children(10).eval(input), children(10).dataType),
+ ScalaReflection.convertToScala(children(11).eval(input), children(11).dataType),
+ ScalaReflection.convertToScala(children(12).eval(input), children(12).dataType),
+ ScalaReflection.convertToScala(children(13).eval(input), children(13).dataType))
+
+
case 15 =>
function.asInstanceOf[(Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input),
- children(4).eval(input),
- children(5).eval(input),
- children(6).eval(input),
- children(7).eval(input),
- children(8).eval(input),
- children(9).eval(input),
- children(10).eval(input),
- children(11).eval(input),
- children(12).eval(input),
- children(13).eval(input),
- children(14).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType),
+ ScalaReflection.convertToScala(children(4).eval(input), children(4).dataType),
+ ScalaReflection.convertToScala(children(5).eval(input), children(5).dataType),
+ ScalaReflection.convertToScala(children(6).eval(input), children(6).dataType),
+ ScalaReflection.convertToScala(children(7).eval(input), children(7).dataType),
+ ScalaReflection.convertToScala(children(8).eval(input), children(8).dataType),
+ ScalaReflection.convertToScala(children(9).eval(input), children(9).dataType),
+ ScalaReflection.convertToScala(children(10).eval(input), children(10).dataType),
+ ScalaReflection.convertToScala(children(11).eval(input), children(11).dataType),
+ ScalaReflection.convertToScala(children(12).eval(input), children(12).dataType),
+ ScalaReflection.convertToScala(children(13).eval(input), children(13).dataType),
+ ScalaReflection.convertToScala(children(14).eval(input), children(14).dataType))
+
+
case 16 =>
function.asInstanceOf[(Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input),
- children(4).eval(input),
- children(5).eval(input),
- children(6).eval(input),
- children(7).eval(input),
- children(8).eval(input),
- children(9).eval(input),
- children(10).eval(input),
- children(11).eval(input),
- children(12).eval(input),
- children(13).eval(input),
- children(14).eval(input),
- children(15).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType),
+ ScalaReflection.convertToScala(children(4).eval(input), children(4).dataType),
+ ScalaReflection.convertToScala(children(5).eval(input), children(5).dataType),
+ ScalaReflection.convertToScala(children(6).eval(input), children(6).dataType),
+ ScalaReflection.convertToScala(children(7).eval(input), children(7).dataType),
+ ScalaReflection.convertToScala(children(8).eval(input), children(8).dataType),
+ ScalaReflection.convertToScala(children(9).eval(input), children(9).dataType),
+ ScalaReflection.convertToScala(children(10).eval(input), children(10).dataType),
+ ScalaReflection.convertToScala(children(11).eval(input), children(11).dataType),
+ ScalaReflection.convertToScala(children(12).eval(input), children(12).dataType),
+ ScalaReflection.convertToScala(children(13).eval(input), children(13).dataType),
+ ScalaReflection.convertToScala(children(14).eval(input), children(14).dataType),
+ ScalaReflection.convertToScala(children(15).eval(input), children(15).dataType))
+
+
case 17 =>
function.asInstanceOf[(Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input),
- children(4).eval(input),
- children(5).eval(input),
- children(6).eval(input),
- children(7).eval(input),
- children(8).eval(input),
- children(9).eval(input),
- children(10).eval(input),
- children(11).eval(input),
- children(12).eval(input),
- children(13).eval(input),
- children(14).eval(input),
- children(15).eval(input),
- children(16).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType),
+ ScalaReflection.convertToScala(children(4).eval(input), children(4).dataType),
+ ScalaReflection.convertToScala(children(5).eval(input), children(5).dataType),
+ ScalaReflection.convertToScala(children(6).eval(input), children(6).dataType),
+ ScalaReflection.convertToScala(children(7).eval(input), children(7).dataType),
+ ScalaReflection.convertToScala(children(8).eval(input), children(8).dataType),
+ ScalaReflection.convertToScala(children(9).eval(input), children(9).dataType),
+ ScalaReflection.convertToScala(children(10).eval(input), children(10).dataType),
+ ScalaReflection.convertToScala(children(11).eval(input), children(11).dataType),
+ ScalaReflection.convertToScala(children(12).eval(input), children(12).dataType),
+ ScalaReflection.convertToScala(children(13).eval(input), children(13).dataType),
+ ScalaReflection.convertToScala(children(14).eval(input), children(14).dataType),
+ ScalaReflection.convertToScala(children(15).eval(input), children(15).dataType),
+ ScalaReflection.convertToScala(children(16).eval(input), children(16).dataType))
+
+
case 18 =>
function.asInstanceOf[(Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input),
- children(4).eval(input),
- children(5).eval(input),
- children(6).eval(input),
- children(7).eval(input),
- children(8).eval(input),
- children(9).eval(input),
- children(10).eval(input),
- children(11).eval(input),
- children(12).eval(input),
- children(13).eval(input),
- children(14).eval(input),
- children(15).eval(input),
- children(16).eval(input),
- children(17).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType),
+ ScalaReflection.convertToScala(children(4).eval(input), children(4).dataType),
+ ScalaReflection.convertToScala(children(5).eval(input), children(5).dataType),
+ ScalaReflection.convertToScala(children(6).eval(input), children(6).dataType),
+ ScalaReflection.convertToScala(children(7).eval(input), children(7).dataType),
+ ScalaReflection.convertToScala(children(8).eval(input), children(8).dataType),
+ ScalaReflection.convertToScala(children(9).eval(input), children(9).dataType),
+ ScalaReflection.convertToScala(children(10).eval(input), children(10).dataType),
+ ScalaReflection.convertToScala(children(11).eval(input), children(11).dataType),
+ ScalaReflection.convertToScala(children(12).eval(input), children(12).dataType),
+ ScalaReflection.convertToScala(children(13).eval(input), children(13).dataType),
+ ScalaReflection.convertToScala(children(14).eval(input), children(14).dataType),
+ ScalaReflection.convertToScala(children(15).eval(input), children(15).dataType),
+ ScalaReflection.convertToScala(children(16).eval(input), children(16).dataType),
+ ScalaReflection.convertToScala(children(17).eval(input), children(17).dataType))
+
+
case 19 =>
function.asInstanceOf[(Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input),
- children(4).eval(input),
- children(5).eval(input),
- children(6).eval(input),
- children(7).eval(input),
- children(8).eval(input),
- children(9).eval(input),
- children(10).eval(input),
- children(11).eval(input),
- children(12).eval(input),
- children(13).eval(input),
- children(14).eval(input),
- children(15).eval(input),
- children(16).eval(input),
- children(17).eval(input),
- children(18).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType),
+ ScalaReflection.convertToScala(children(4).eval(input), children(4).dataType),
+ ScalaReflection.convertToScala(children(5).eval(input), children(5).dataType),
+ ScalaReflection.convertToScala(children(6).eval(input), children(6).dataType),
+ ScalaReflection.convertToScala(children(7).eval(input), children(7).dataType),
+ ScalaReflection.convertToScala(children(8).eval(input), children(8).dataType),
+ ScalaReflection.convertToScala(children(9).eval(input), children(9).dataType),
+ ScalaReflection.convertToScala(children(10).eval(input), children(10).dataType),
+ ScalaReflection.convertToScala(children(11).eval(input), children(11).dataType),
+ ScalaReflection.convertToScala(children(12).eval(input), children(12).dataType),
+ ScalaReflection.convertToScala(children(13).eval(input), children(13).dataType),
+ ScalaReflection.convertToScala(children(14).eval(input), children(14).dataType),
+ ScalaReflection.convertToScala(children(15).eval(input), children(15).dataType),
+ ScalaReflection.convertToScala(children(16).eval(input), children(16).dataType),
+ ScalaReflection.convertToScala(children(17).eval(input), children(17).dataType),
+ ScalaReflection.convertToScala(children(18).eval(input), children(18).dataType))
+
+
case 20 =>
function.asInstanceOf[(Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input),
- children(4).eval(input),
- children(5).eval(input),
- children(6).eval(input),
- children(7).eval(input),
- children(8).eval(input),
- children(9).eval(input),
- children(10).eval(input),
- children(11).eval(input),
- children(12).eval(input),
- children(13).eval(input),
- children(14).eval(input),
- children(15).eval(input),
- children(16).eval(input),
- children(17).eval(input),
- children(18).eval(input),
- children(19).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType),
+ ScalaReflection.convertToScala(children(4).eval(input), children(4).dataType),
+ ScalaReflection.convertToScala(children(5).eval(input), children(5).dataType),
+ ScalaReflection.convertToScala(children(6).eval(input), children(6).dataType),
+ ScalaReflection.convertToScala(children(7).eval(input), children(7).dataType),
+ ScalaReflection.convertToScala(children(8).eval(input), children(8).dataType),
+ ScalaReflection.convertToScala(children(9).eval(input), children(9).dataType),
+ ScalaReflection.convertToScala(children(10).eval(input), children(10).dataType),
+ ScalaReflection.convertToScala(children(11).eval(input), children(11).dataType),
+ ScalaReflection.convertToScala(children(12).eval(input), children(12).dataType),
+ ScalaReflection.convertToScala(children(13).eval(input), children(13).dataType),
+ ScalaReflection.convertToScala(children(14).eval(input), children(14).dataType),
+ ScalaReflection.convertToScala(children(15).eval(input), children(15).dataType),
+ ScalaReflection.convertToScala(children(16).eval(input), children(16).dataType),
+ ScalaReflection.convertToScala(children(17).eval(input), children(17).dataType),
+ ScalaReflection.convertToScala(children(18).eval(input), children(18).dataType),
+ ScalaReflection.convertToScala(children(19).eval(input), children(19).dataType))
+
+
case 21 =>
function.asInstanceOf[(Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input),
- children(4).eval(input),
- children(5).eval(input),
- children(6).eval(input),
- children(7).eval(input),
- children(8).eval(input),
- children(9).eval(input),
- children(10).eval(input),
- children(11).eval(input),
- children(12).eval(input),
- children(13).eval(input),
- children(14).eval(input),
- children(15).eval(input),
- children(16).eval(input),
- children(17).eval(input),
- children(18).eval(input),
- children(19).eval(input),
- children(20).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType),
+ ScalaReflection.convertToScala(children(4).eval(input), children(4).dataType),
+ ScalaReflection.convertToScala(children(5).eval(input), children(5).dataType),
+ ScalaReflection.convertToScala(children(6).eval(input), children(6).dataType),
+ ScalaReflection.convertToScala(children(7).eval(input), children(7).dataType),
+ ScalaReflection.convertToScala(children(8).eval(input), children(8).dataType),
+ ScalaReflection.convertToScala(children(9).eval(input), children(9).dataType),
+ ScalaReflection.convertToScala(children(10).eval(input), children(10).dataType),
+ ScalaReflection.convertToScala(children(11).eval(input), children(11).dataType),
+ ScalaReflection.convertToScala(children(12).eval(input), children(12).dataType),
+ ScalaReflection.convertToScala(children(13).eval(input), children(13).dataType),
+ ScalaReflection.convertToScala(children(14).eval(input), children(14).dataType),
+ ScalaReflection.convertToScala(children(15).eval(input), children(15).dataType),
+ ScalaReflection.convertToScala(children(16).eval(input), children(16).dataType),
+ ScalaReflection.convertToScala(children(17).eval(input), children(17).dataType),
+ ScalaReflection.convertToScala(children(18).eval(input), children(18).dataType),
+ ScalaReflection.convertToScala(children(19).eval(input), children(19).dataType),
+ ScalaReflection.convertToScala(children(20).eval(input), children(20).dataType))
+
+
case 22 =>
function.asInstanceOf[(Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any) => Any](
- children(0).eval(input),
- children(1).eval(input),
- children(2).eval(input),
- children(3).eval(input),
- children(4).eval(input),
- children(5).eval(input),
- children(6).eval(input),
- children(7).eval(input),
- children(8).eval(input),
- children(9).eval(input),
- children(10).eval(input),
- children(11).eval(input),
- children(12).eval(input),
- children(13).eval(input),
- children(14).eval(input),
- children(15).eval(input),
- children(16).eval(input),
- children(17).eval(input),
- children(18).eval(input),
- children(19).eval(input),
- children(20).eval(input),
- children(21).eval(input))
+ ScalaReflection.convertToScala(children(0).eval(input), children(0).dataType),
+ ScalaReflection.convertToScala(children(1).eval(input), children(1).dataType),
+ ScalaReflection.convertToScala(children(2).eval(input), children(2).dataType),
+ ScalaReflection.convertToScala(children(3).eval(input), children(3).dataType),
+ ScalaReflection.convertToScala(children(4).eval(input), children(4).dataType),
+ ScalaReflection.convertToScala(children(5).eval(input), children(5).dataType),
+ ScalaReflection.convertToScala(children(6).eval(input), children(6).dataType),
+ ScalaReflection.convertToScala(children(7).eval(input), children(7).dataType),
+ ScalaReflection.convertToScala(children(8).eval(input), children(8).dataType),
+ ScalaReflection.convertToScala(children(9).eval(input), children(9).dataType),
+ ScalaReflection.convertToScala(children(10).eval(input), children(10).dataType),
+ ScalaReflection.convertToScala(children(11).eval(input), children(11).dataType),
+ ScalaReflection.convertToScala(children(12).eval(input), children(12).dataType),
+ ScalaReflection.convertToScala(children(13).eval(input), children(13).dataType),
+ ScalaReflection.convertToScala(children(14).eval(input), children(14).dataType),
+ ScalaReflection.convertToScala(children(15).eval(input), children(15).dataType),
+ ScalaReflection.convertToScala(children(16).eval(input), children(16).dataType),
+ ScalaReflection.convertToScala(children(17).eval(input), children(17).dataType),
+ ScalaReflection.convertToScala(children(18).eval(input), children(18).dataType),
+ ScalaReflection.convertToScala(children(19).eval(input), children(19).dataType),
+ ScalaReflection.convertToScala(children(20).eval(input), children(20).dataType),
+ ScalaReflection.convertToScala(children(21).eval(input), children(21).dataType))
+
}
// scalastyle:on
http://git-wip-us.apache.org/repos/asf/spark/blob/15b58a22/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
index 666235e..1806a1d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
@@ -60,13 +60,13 @@ private[sql] class MyDenseVectorUDT extends UserDefinedType[MyDenseVector] {
}
class UserDefinedTypeSuite extends QueryTest {
+ val points = Seq(
+ MyLabeledPoint(1.0, new MyDenseVector(Array(0.1, 1.0))),
+ MyLabeledPoint(0.0, new MyDenseVector(Array(0.2, 2.0))))
+ val pointsRDD: RDD[MyLabeledPoint] = sparkContext.parallelize(points)
- test("register user type: MyDenseVector for MyLabeledPoint") {
- val points = Seq(
- MyLabeledPoint(1.0, new MyDenseVector(Array(0.1, 1.0))),
- MyLabeledPoint(0.0, new MyDenseVector(Array(0.2, 2.0))))
- val pointsRDD: RDD[MyLabeledPoint] = sparkContext.parallelize(points)
+ test("register user type: MyDenseVector for MyLabeledPoint") {
val labels: RDD[Double] = pointsRDD.select('label).map { case Row(v: Double) => v }
val labelsArrays: Array[Double] = labels.collect()
assert(labelsArrays.size === 2)
@@ -80,4 +80,12 @@ class UserDefinedTypeSuite extends QueryTest {
assert(featuresArrays.contains(new MyDenseVector(Array(0.1, 1.0))))
assert(featuresArrays.contains(new MyDenseVector(Array(0.2, 2.0))))
}
+
+ test("UDTs and UDFs") {
+ registerFunction("testType", (d: MyDenseVector) => d.isInstanceOf[MyDenseVector])
+ pointsRDD.registerTempTable("points")
+ checkAnswer(
+ sql("SELECT testType(features) from points"),
+ Seq(Row(true), Row(true)))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org