You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/02/04 05:07:48 UTC
[2/2] spark git commit: [SPARK-5578][SQL][DataFrame] Provide a
convenient way for Scala users to use UDFs
[SPARK-5578][SQL][DataFrame] Provide a convenient way for Scala users to use UDFs
A more convenient way to define user-defined functions.
Author: Reynold Xin <rx...@databricks.com>
Closes #4345 from rxin/defineUDF and squashes the following commits:
639c0f8 [Reynold Xin] udf tests.
0a0b339 [Reynold Xin] defineUDF -> udf.
b452b8d [Reynold Xin] Fix UDF registration.
d2e42c3 [Reynold Xin] SQLContext.udf.register() returns a UserDefinedFunction also.
4333605 [Reynold Xin] [SQL][DataFrame] defineUDF.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1077f2e1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1077f2e1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1077f2e1
Branch: refs/heads/master
Commit: 1077f2e1def6266aee6ad6f0640a8f46cd273e21
Parents: e380d2d
Author: Reynold Xin <rx...@databricks.com>
Authored: Tue Feb 3 20:07:46 2015 -0800
Committer: Reynold Xin <rx...@databricks.com>
Committed: Tue Feb 3 20:07:46 2015 -0800
----------------------------------------------------------------------
.../ml/classification/LogisticRegression.scala | 10 +-
.../spark/ml/feature/StandardScaler.scala | 6 +-
.../apache/spark/ml/recommendation/ALS.scala | 14 +-
.../scala/org/apache/spark/sql/DataFrame.scala | 4 +-
.../org/apache/spark/sql/DataFrameImpl.scala | 2 +-
.../main/scala/org/apache/spark/sql/Dsl.scala | 194 +++--
.../org/apache/spark/sql/UDFRegistration.scala | 571 ++++++++++++++
.../org/apache/spark/sql/UdfRegistration.scala | 774 -------------------
.../apache/spark/sql/UserDefinedFunction.scala | 39 +
.../org/apache/spark/sql/DataFrameSuite.scala | 6 +-
.../org/apache/spark/sql/SQLQuerySuite.scala | 6 +-
.../apache/spark/sql/UserDefinedTypeSuite.scala | 5 +-
12 files changed, 731 insertions(+), 900 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/1077f2e1/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
index 18be35a..df90078 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
@@ -132,14 +132,14 @@ class LogisticRegressionModel private[ml] (
override def transform(dataset: DataFrame, paramMap: ParamMap): DataFrame = {
transformSchema(dataset.schema, paramMap, logging = true)
val map = this.paramMap ++ paramMap
- val scoreFunction: Vector => Double = (v) => {
+ val scoreFunction = udf((v: Vector) => {
val margin = BLAS.dot(v, weights)
1.0 / (1.0 + math.exp(-margin))
- }
+ } : Double)
val t = map(threshold)
- val predictFunction: Double => Double = (score) => { if (score > t) 1.0 else 0.0 }
+ val predictFunction = udf((score: Double) => { if (score > t) 1.0 else 0.0 } : Double)
dataset
- .select($"*", callUDF(scoreFunction, col(map(featuresCol))).as(map(scoreCol)))
- .select($"*", callUDF(predictFunction, col(map(scoreCol))).as(map(predictionCol)))
+ .select($"*", scoreFunction(col(map(featuresCol))).as(map(scoreCol)))
+ .select($"*", predictFunction(col(map(scoreCol))).as(map(predictionCol)))
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/1077f2e1/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
index 01a4f5e..4745a7a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
@@ -81,10 +81,8 @@ class StandardScalerModel private[ml] (
override def transform(dataset: DataFrame, paramMap: ParamMap): DataFrame = {
transformSchema(dataset.schema, paramMap, logging = true)
val map = this.paramMap ++ paramMap
- val scale: (Vector) => Vector = (v) => {
- scaler.transform(v)
- }
- dataset.select($"*", callUDF(scale, col(map(inputCol))).as(map(outputCol)))
+ val scale = udf((v: Vector) => { scaler.transform(v) } : Vector)
+ dataset.select($"*", scale(col(map(inputCol))).as(map(outputCol)))
}
private[ml] override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {
http://git-wip-us.apache.org/repos/asf/spark/blob/1077f2e1/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
index 511cb2f..c7bec7a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
@@ -126,22 +126,20 @@ class ALSModel private[ml] (
val map = this.paramMap ++ paramMap
val users = userFactors.toDataFrame("id", "features")
val items = itemFactors.toDataFrame("id", "features")
- val predict: (Seq[Float], Seq[Float]) => Float = (userFeatures, itemFeatures) => {
+
+ // Register a UDF for DataFrame, and then
+ // create a new column named map(predictionCol) by running the predict UDF.
+ val predict = udf((userFeatures: Seq[Float], itemFeatures: Seq[Float]) => {
if (userFeatures != null && itemFeatures != null) {
blas.sdot(k, userFeatures.toArray, 1, itemFeatures.toArray, 1)
} else {
Float.NaN
}
- }
- val inputColumns = dataset.schema.fieldNames
- val prediction = callUDF(predict, users("features"), items("features")).as(map(predictionCol))
- val outputColumns = inputColumns.map(f => dataset(f)) :+ prediction
+ } : Float)
dataset
.join(users, dataset(map(userCol)) === users("id"), "left")
.join(items, dataset(map(itemCol)) === items("id"), "left")
- .select(outputColumns: _*)
- // TODO: Just use a dataset("*")
- // .select(dataset("*"), prediction)
+ .select(dataset("*"), predict(users("features"), items("features")).as(map(predictionCol)))
}
override private[ml] def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {
http://git-wip-us.apache.org/repos/asf/spark/blob/1077f2e1/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
index f3bc07a..732b685 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
@@ -54,10 +54,10 @@ private[sql] object DataFrame {
* }}}
*
* Note that the [[Column]] type can also be manipulated through its various functions.
- * {{
+ * {{{
* // The following creates a new column that increases everybody's age by 10.
* people("age") + 10 // in Scala
- * }}
+ * }}}
*
* A more concrete example:
* {{{
http://git-wip-us.apache.org/repos/asf/spark/blob/1077f2e1/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
index 0b0623d..a52bfa5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
@@ -329,7 +329,7 @@ private[sql] class DataFrameImpl protected[sql](
override def save(path: String): Unit = {
val dataSourceName = sqlContext.conf.defaultDataSourceName
- save(dataSourceName, ("path" -> path))
+ save(dataSourceName, "path" -> path)
}
override def save(
http://git-wip-us.apache.org/repos/asf/spark/blob/1077f2e1/sql/core/src/main/scala/org/apache/spark/sql/Dsl.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dsl.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dsl.scala
index 71365c7..8cf59f0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dsl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dsl.scala
@@ -186,15 +186,13 @@ object Dsl {
(0 to 22).map { x =>
val types = (1 to x).foldRight("RT")((i, s) => {s"A$i, $s"})
val typeTags = (1 to x).map(i => s"A$i: TypeTag").foldLeft("RT: TypeTag")(_ + ", " + _)
- val args = (1 to x).map(i => s"arg$i: Column").mkString(", ")
- val argsInUdf = (1 to x).map(i => s"arg$i.expr").mkString(", ")
println(s"""
/**
- * Call a Scala function of ${x} arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of ${x} arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[$typeTags](f: Function$x[$types]${if (args.length > 0) ", " + args else ""}): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq($argsInUdf))
+ def udf[$typeTags](f: Function$x[$types]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}""")
}
@@ -214,187 +212,187 @@ object Dsl {
}
*/
/**
- * Call a Scala function of 0 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 0 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag](f: Function0[RT]): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq())
+ def udf[RT: TypeTag](f: Function0[RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 1 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 1 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag](f: Function1[A1, RT], arg1: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr))
+ def udf[RT: TypeTag, A1: TypeTag](f: Function1[A1, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 2 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 2 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag](f: Function2[A1, A2, RT], arg1: Column, arg2: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag](f: Function2[A1, A2, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 3 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 3 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](f: Function3[A1, A2, A3, RT], arg1: Column, arg2: Column, arg3: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](f: Function3[A1, A2, A3, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 4 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 4 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](f: Function4[A1, A2, A3, A4, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](f: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 5 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 5 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](f: Function5[A1, A2, A3, A4, A5, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](f: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 6 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 6 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](f: Function6[A1, A2, A3, A4, A5, A6, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](f: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 7 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 7 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](f: Function7[A1, A2, A3, A4, A5, A6, A7, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](f: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 8 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 8 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](f: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](f: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 9 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 9 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](f: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](f: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 10 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 10 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](f: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](f: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 11 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 11 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag](f: Function11[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag](f: Function11[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 12 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 12 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag](f: Function12[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag](f: Function12[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 13 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 13 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag](f: Function13[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag](f: Function13[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 14 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 14 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag](f: Function14[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column, arg14: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr, arg14.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag](f: Function14[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 15 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 15 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag](f: Function15[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column, arg14: Column, arg15: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr, arg14.expr, arg15.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag](f: Function15[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 16 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 16 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag](f: Function16[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column, arg14: Column, arg15: Column, arg16: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr, arg14.expr, arg15.expr, arg16.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag](f: Function16[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 17 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 17 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag](f: Function17[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column, arg14: Column, arg15: Column, arg16: Column, arg17: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr, arg14.expr, arg15.expr, arg16.expr, arg17.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag](f: Function17[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 18 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 18 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag](f: Function18[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column, arg14: Column, arg15: Column, arg16: Column, arg17: Column, arg18: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr, arg14.expr, arg15.expr, arg16.expr, arg17.expr, arg18.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag](f: Function18[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 19 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 19 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag](f: Function19[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column, arg14: Column, arg15: Column, arg16: Column, arg17: Column, arg18: Column, arg19: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr, arg14.expr, arg15.expr, arg16.expr, arg17.expr, arg18.expr, arg19.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag](f: Function19[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 20 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 20 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag](f: Function20[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column, arg14: Column, arg15: Column, arg16: Column, arg17: Column, arg18: Column, arg19: Column, arg20: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr, arg14.expr, arg15.expr, arg16.expr, arg17.expr, arg18.expr, arg19.expr, arg20.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag](f: Function20[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 21 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 21 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag](f: Function21[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column, arg14: Column, arg15: Column, arg16: Column, arg17: Column, arg18: Column, arg19: Column, arg20: Column, arg21: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr, arg14.expr, arg15.expr, arg16.expr, arg17.expr, arg18.expr, arg19.expr, arg20.expr, arg21.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag](f: Function21[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
/**
- * Call a Scala function of 22 arguments as user-defined function (UDF), and automatically
- * infer the data types based on the function's signature.
+ * Defines a user-defined function of 22 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
*/
- def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag, A22: TypeTag](f: Function22[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, A22, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column, arg14: Column, arg15: Column, arg16: Column, arg17: Column, arg18: Column, arg19: Column, arg20: Column, arg21: Column, arg22: Column): Column = {
- ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr, arg14.expr, arg15.expr, arg16.expr, arg17.expr, arg18.expr, arg19.expr, arg20.expr, arg21.expr, arg22.expr))
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag, A22: TypeTag](f: Function22[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, A22, RT]): UserDefinedFunction = {
+ UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}
//////////////////////////////////////////////////////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/spark/blob/1077f2e1/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
new file mode 100644
index 0000000..d8b0a3b
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
@@ -0,0 +1,571 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import java.util.{List => JList, Map => JMap}
+
+import scala.reflect.runtime.universe.TypeTag
+
+import org.apache.spark.{Accumulator, Logging}
+import org.apache.spark.api.python.PythonBroadcast
+import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.sql.api.java._
+import org.apache.spark.sql.catalyst.ScalaReflection
+import org.apache.spark.sql.catalyst.expressions.{Expression, ScalaUdf}
+import org.apache.spark.sql.execution.PythonUDF
+import org.apache.spark.sql.types.DataType
+
+
+/**
+ * Functions for registering user-defined functions.
+ */
+class UDFRegistration(sqlContext: SQLContext) extends Logging {
+
+ private val functionRegistry = sqlContext.functionRegistry
+
+ protected[sql] def registerPython(
+ name: String,
+ command: Array[Byte],
+ envVars: JMap[String, String],
+ pythonIncludes: JList[String],
+ pythonExec: String,
+ broadcastVars: JList[Broadcast[PythonBroadcast]],
+ accumulator: Accumulator[JList[Array[Byte]]],
+ stringDataType: String): Unit = {
+ log.debug(
+ s"""
+ | Registering new PythonUDF:
+ | name: $name
+ | command: ${command.toSeq}
+ | envVars: $envVars
+ | pythonIncludes: $pythonIncludes
+ | pythonExec: $pythonExec
+ | dataType: $stringDataType
+ """.stripMargin)
+
+
+ val dataType = sqlContext.parseDataType(stringDataType)
+
+ def builder(e: Seq[Expression]) =
+ PythonUDF(
+ name,
+ command,
+ envVars,
+ pythonIncludes,
+ pythonExec,
+ broadcastVars,
+ accumulator,
+ dataType,
+ e)
+
+ functionRegistry.registerFunction(name, builder)
+ }
+
+ // scalastyle:off
+
+ /* register 0-22 were generated by this script
+
+ (0 to 22).map { x =>
+ val types = (1 to x).foldRight("RT")((i, s) => {s"A$i, $s"})
+ val typeTags = (1 to x).map(i => s"A${i}: TypeTag").foldLeft("RT: TypeTag")(_ + ", " + _)
+ println(s"""
+ /**
+ * Register a Scala closure of ${x} arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[$typeTags](name: String, func: Function$x[$types]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }""")
+ }
+
+ (1 to 22).foreach { i =>
+ val extTypeArgs = (1 to i).map(_ => "_").mkString(", ")
+ val anyTypeArgs = (1 to i).map(_ => "Any").mkString(", ")
+ val anyCast = s".asInstanceOf[UDF$i[$anyTypeArgs, Any]]"
+ val anyParams = (1 to i).map(_ => "_: Any").mkString(", ")
+ println(s"""
+ |/**
+ | * Register a user-defined function with ${i} arguments.
+ | */
+ |def register(name: String, f: UDF$i[$extTypeArgs, _], returnType: DataType) = {
+ | functionRegistry.registerFunction(
+ | name,
+ | (e: Seq[Expression]) => ScalaUdf(f$anyCast.call($anyParams), returnType, e))
+ |}""".stripMargin)
+ }
+ */
+
+ /**
+ * Register a Scala closure of 0 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag](name: String, func: Function0[RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 1 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag](name: String, func: Function1[A1, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 2 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag](name: String, func: Function2[A1, A2, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 3 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](name: String, func: Function3[A1, A2, A3, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 4 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](name: String, func: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 5 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](name: String, func: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 6 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](name: String, func: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 7 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](name: String, func: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 8 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](name: String, func: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 9 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](name: String, func: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 10 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](name: String, func: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 11 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag](name: String, func: Function11[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 12 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag](name: String, func: Function12[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 13 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag](name: String, func: Function13[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 14 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag](name: String, func: Function14[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 15 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag](name: String, func: Function15[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 16 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag](name: String, func: Function16[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 17 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag](name: String, func: Function17[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 18 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag](name: String, func: Function18[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 19 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag](name: String, func: Function19[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 20 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag](name: String, func: Function20[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 21 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag](name: String, func: Function21[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ /**
+ * Register a Scala closure of 22 arguments as user-defined function (UDF).
+ * @tparam RT return type of UDF.
+ */
+ def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag, A22: TypeTag](name: String, func: Function22[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, A22, RT]): UserDefinedFunction = {
+ val dataType = ScalaReflection.schemaFor[RT].dataType
+ def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
+ functionRegistry.registerFunction(name, builder)
+ UserDefinedFunction(func, dataType)
+ }
+
+ //////////////////////////////////////////////////////////////////////////////////////////////
+ //////////////////////////////////////////////////////////////////////////////////////////////
+
+ /**
+ * Register a user-defined function with 1 arguments.
+ */
+ def register(name: String, f: UDF1[_, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF1[Any, Any]].call(_: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 2 arguments.
+ */
+ def register(name: String, f: UDF2[_, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF2[Any, Any, Any]].call(_: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 3 arguments.
+ */
+ def register(name: String, f: UDF3[_, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF3[Any, Any, Any, Any]].call(_: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 4 arguments.
+ */
+ def register(name: String, f: UDF4[_, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF4[Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 5 arguments.
+ */
+ def register(name: String, f: UDF5[_, _, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF5[Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 6 arguments.
+ */
+ def register(name: String, f: UDF6[_, _, _, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF6[Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 7 arguments.
+ */
+ def register(name: String, f: UDF7[_, _, _, _, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF7[Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 8 arguments.
+ */
+ def register(name: String, f: UDF8[_, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF8[Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 9 arguments.
+ */
+ def register(name: String, f: UDF9[_, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF9[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 10 arguments.
+ */
+ def register(name: String, f: UDF10[_, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF10[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 11 arguments.
+ */
+ def register(name: String, f: UDF11[_, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF11[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 12 arguments.
+ */
+ def register(name: String, f: UDF12[_, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF12[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 13 arguments.
+ */
+ def register(name: String, f: UDF13[_, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF13[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 14 arguments.
+ */
+ def register(name: String, f: UDF14[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF14[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 15 arguments.
+ */
+ def register(name: String, f: UDF15[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF15[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 16 arguments.
+ */
+ def register(name: String, f: UDF16[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF16[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 17 arguments.
+ */
+ def register(name: String, f: UDF17[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF17[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 18 arguments.
+ */
+ def register(name: String, f: UDF18[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF18[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 19 arguments.
+ */
+ def register(name: String, f: UDF19[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF19[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 20 arguments.
+ */
+ def register(name: String, f: UDF20[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF20[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 21 arguments.
+ */
+ def register(name: String, f: UDF21[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF21[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ /**
+ * Register a user-defined function with 22 arguments.
+ */
+ def register(name: String, f: UDF22[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ functionRegistry.registerFunction(
+ name,
+ (e: Seq[Expression]) => ScalaUdf(f.asInstanceOf[UDF22[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
+ }
+
+ // scalastyle:on
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org