You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2017/07/17 01:22:45 UTC
spark git commit: [SPARK-19810][SPARK-19810][MINOR][FOLLOW-UP]
Follow-ups from to remove Scala 2.10
Repository: spark
Updated Branches:
refs/heads/master 69e5282d3 -> fd52a747f
[SPARK-19810][SPARK-19810][MINOR][FOLLOW-UP] Follow-ups from to remove Scala 2.10
## What changes were proposed in this pull request?
Follow up to a few comments on https://github.com/apache/spark/pull/17150#issuecomment-315020196 that couldn't be addressed before it was merged.
## How was this patch tested?
Existing tests.
Author: Sean Owen <so...@cloudera.com>
Closes #18646 from srowen/SPARK-19810.2.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fd52a747
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fd52a747
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fd52a747
Branch: refs/heads/master
Commit: fd52a747fd7f5a0f9657b4fa16c797916aef9a95
Parents: 69e5282
Author: Sean Owen <so...@cloudera.com>
Authored: Mon Jul 17 09:22:42 2017 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Mon Jul 17 09:22:42 2017 +0800
----------------------------------------------------------------------
.../sql/catalyst/ScalaReflectionSuite.scala | 39 --------------------
.../execution/python/ExtractPythonUDFs.scala | 2 +-
2 files changed, 1 insertion(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/fd52a747/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
index ff2414b..a5b9855 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
@@ -17,17 +17,13 @@
package org.apache.spark.sql.catalyst
-import java.net.URLClassLoader
import java.sql.{Date, Timestamp}
-import scala.reflect.runtime.universe.typeOf
-
import org.apache.spark.SparkFunSuite
import org.apache.spark.sql.catalyst.expressions.{BoundReference, Literal, SpecificInternalRow}
import org.apache.spark.sql.catalyst.expressions.objects.NewInstance
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
-import org.apache.spark.util.Utils
case class PrimitiveData(
intField: Int,
@@ -339,39 +335,4 @@ class ScalaReflectionSuite extends SparkFunSuite {
assert(linkedHashMapDeserializer.dataType == ObjectType(classOf[LHMap[_, _]]))
}
- private val dataTypeForComplexData = dataTypeFor[ComplexData]
- private val typeOfComplexData = typeOf[ComplexData]
-
- Seq(
- ("mirror", () => mirror),
- ("dataTypeFor", () => dataTypeFor[ComplexData]),
- ("constructorFor", () => deserializerFor[ComplexData]),
- ("extractorsFor", {
- val inputObject = BoundReference(0, dataTypeForComplexData, nullable = false)
- () => serializerFor[ComplexData](inputObject)
- }),
- ("getConstructorParameters(cls)", () => getConstructorParameters(classOf[ComplexData])),
- ("getConstructorParameterNames", () => getConstructorParameterNames(classOf[ComplexData])),
- ("getClassFromType", () => getClassFromType(typeOfComplexData)),
- ("schemaFor", () => schemaFor[ComplexData]),
- ("localTypeOf", () => localTypeOf[ComplexData]),
- ("getClassNameFromType", () => getClassNameFromType(typeOfComplexData)),
- ("getParameterTypes", () => getParameterTypes(() => ())),
- ("getConstructorParameters(tpe)", () => getClassNameFromType(typeOfComplexData))).foreach {
- case (name, exec) =>
- test(s"SPARK-13640: thread safety of ${name}") {
- (0 until 100).foreach { _ =>
- val loader = new URLClassLoader(Array.empty, Utils.getContextOrSparkClassLoader)
- (0 until 10).par.foreach { _ =>
- val cl = Thread.currentThread.getContextClassLoader
- try {
- Thread.currentThread.setContextClassLoader(loader)
- exec()
- } finally {
- Thread.currentThread.setContextClassLoader(cl)
- }
- }
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/fd52a747/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala
index 2b3c5f0..1400452 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala
@@ -133,7 +133,7 @@ object ExtractPythonUDFs extends Rule[SparkPlan] with PredicateHelper {
val validUdfs = udfs.filter { udf =>
// Check to make sure that the UDF can be evaluated with only the input of this child.
udf.references.subsetOf(child.outputSet)
- }.toArray
+ }
if (validUdfs.nonEmpty) {
val resultAttrs = udfs.zipWithIndex.map { case (u, i) =>
AttributeReference(s"pythonUDF$i", u.dataType)()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org