You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/12/29 05:43:11 UTC
spark git commit: [SPARK-12547][SQL] Tighten scala style checker
enforcement for UDF registration
Repository: spark
Updated Branches:
refs/heads/master 043135819 -> 1a91be807
[SPARK-12547][SQL] Tighten scala style checker enforcement for UDF registration
We use scalastyle:off to turn off style checks in certain places where it is not possible to follow the style guide. This is usually ok. However, in udf registration, we disable the checker for a large amount of code simply because some of them exceed 100 char line limit. It is better to just disable the line limit check rather than everything.
In this pull request, I only disabled line length check, and fixed a problem (lack explicit types for public methods).
Author: Reynold Xin <rx...@databricks.com>
Closes #10501 from rxin/SPARK-12547.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1a91be80
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1a91be80
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1a91be80
Branch: refs/heads/master
Commit: 1a91be807802ec88c068f1090dafe8fbfb1c6d5c
Parents: 0431358
Author: Reynold Xin <rx...@databricks.com>
Authored: Mon Dec 28 20:43:06 2015 -0800
Committer: Reynold Xin <rx...@databricks.com>
Committed: Mon Dec 28 20:43:06 2015 -0800
----------------------------------------------------------------------
.../org/apache/spark/sql/UDFRegistration.scala | 53 ++++++++++----------
.../scala/org/apache/spark/sql/functions.scala | 6 ++-
2 files changed, 30 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/1a91be80/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
index 051694c..42c373e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
@@ -17,8 +17,6 @@
package org.apache.spark.sql
-import java.util.{List => JList, Map => JMap}
-
import scala.reflect.runtime.universe.TypeTag
import scala.util.Try
@@ -69,7 +67,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
udaf
}
- // scalastyle:off
+ // scalastyle:off line.size.limit
/* register 0-22 were generated by this script
@@ -102,7 +100,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
| * Register a user-defined function with ${i} arguments.
| * @since 1.3.0
| */
- |def register(name: String, f: UDF$i[$extTypeArgs, _], returnType: DataType) = {
+ |def register(name: String, f: UDF$i[$extTypeArgs, _], returnType: DataType): Unit = {
| functionRegistry.registerFunction(
| name,
| (e: Seq[Expression]) => ScalaUDF(f$anyCast.call($anyParams), returnType, e))
@@ -416,7 +414,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 1 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF1[_, _], returnType: DataType) = {
+ def register(name: String, f: UDF1[_, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF1[Any, Any]].call(_: Any), returnType, e))
@@ -426,7 +424,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 2 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF2[_, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF2[_, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF2[Any, Any, Any]].call(_: Any, _: Any), returnType, e))
@@ -436,7 +434,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 3 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF3[_, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF3[_, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF3[Any, Any, Any, Any]].call(_: Any, _: Any, _: Any), returnType, e))
@@ -446,7 +444,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 4 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF4[_, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF4[_, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF4[Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -456,7 +454,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 5 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF5[_, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF5[_, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF5[Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -466,7 +464,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 6 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF6[_, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF6[_, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF6[Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -476,7 +474,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 7 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF7[_, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF7[_, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF7[Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -486,7 +484,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 8 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF8[_, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF8[_, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF8[Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -496,7 +494,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 9 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF9[_, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF9[_, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF9[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -506,7 +504,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 10 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF10[_, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF10[_, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF10[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -516,7 +514,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 11 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF11[_, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF11[_, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF11[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -526,7 +524,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 12 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF12[_, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF12[_, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF12[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -536,7 +534,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 13 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF13[_, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF13[_, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF13[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -546,7 +544,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 14 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF14[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF14[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF14[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -556,7 +554,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 15 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF15[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF15[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF15[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -566,7 +564,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 16 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF16[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF16[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF16[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -576,7 +574,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 17 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF17[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF17[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF17[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -586,7 +584,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 18 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF18[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF18[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF18[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -596,7 +594,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 19 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF19[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF19[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF19[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -606,7 +604,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 20 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF20[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF20[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF20[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -616,7 +614,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 21 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF21[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF21[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF21[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -626,11 +624,12 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 22 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF22[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF22[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF22[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
}
- // scalastyle:on
+ // scalastyle:on line.size.limit
+
}
http://git-wip-us.apache.org/repos/asf/spark/blob/1a91be80/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 65733dc..4871916 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -2512,7 +2512,8 @@ object functions extends LegacyFunctions {
//////////////////////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////////////////////
- // scalastyle:off
+ // scalastyle:off line.size.limit
+ // scalastyle:off parameter.number
/* Use the following code to generate:
(0 to 10).map { x =>
@@ -2839,7 +2840,8 @@ object functions extends LegacyFunctions {
ScalaUDF(f, returnType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr))
}
- // scalastyle:on
+ // scalastyle:on parameter.number
+ // scalastyle:on line.size.limit
/**
* Call an user-defined function.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org