You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2014/07/16 20:15:10 UTC
git commit: [SPARK-2517] Remove some compiler warnings.
Repository: spark
Updated Branches:
refs/heads/master cc965eea5 -> ef48222c1
[SPARK-2517] Remove some compiler warnings.
Author: Reynold Xin <rx...@apache.org>
Closes #1433 from rxin/compile-warning and squashes the following commits:
8d0b890 [Reynold Xin] Remove some compiler warnings.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ef48222c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ef48222c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ef48222c
Branch: refs/heads/master
Commit: ef48222c10be3d29a83dfc2329f455eba203cd38
Parents: cc965ee
Author: Reynold Xin <rx...@apache.org>
Authored: Wed Jul 16 11:15:07 2014 -0700
Committer: Reynold Xin <rx...@apache.org>
Committed: Wed Jul 16 11:15:07 2014 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/rdd/RDDSuite.scala | 3 ++
.../ProactiveClosureSerializationSuite.scala | 37 +++++++++-----------
.../apache/spark/util/FileAppenderSuite.scala | 3 +-
.../org/apache/spark/util/VectorSuite.scala | 1 +
.../spark/streaming/InputStreamsSuite.scala | 5 ++-
5 files changed, 25 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ef48222c/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala b/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala
index 0f9cbe2..6ea0451 100644
--- a/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala
@@ -379,6 +379,7 @@ class RDDSuite extends FunSuite with SharedSparkContext {
test("mapWith") {
import java.util.Random
val ones = sc.makeRDD(Array(1, 1, 1, 1, 1, 1), 2)
+ @deprecated("suppress compile time deprecation warning", "1.0.0")
val randoms = ones.mapWith(
(index: Int) => new Random(index + 42))
{(t: Int, prng: Random) => prng.nextDouble * t}.collect()
@@ -397,6 +398,7 @@ class RDDSuite extends FunSuite with SharedSparkContext {
test("flatMapWith") {
import java.util.Random
val ones = sc.makeRDD(Array(1, 1, 1, 1, 1, 1), 2)
+ @deprecated("suppress compile time deprecation warning", "1.0.0")
val randoms = ones.flatMapWith(
(index: Int) => new Random(index + 42))
{(t: Int, prng: Random) =>
@@ -418,6 +420,7 @@ class RDDSuite extends FunSuite with SharedSparkContext {
test("filterWith") {
import java.util.Random
val ints = sc.makeRDD(Array(1, 2, 3, 4, 5, 6), 2)
+ @deprecated("suppress compile time deprecation warning", "1.0.0")
val sample = ints.filterWith(
(index: Int) => new Random(index + 42))
{(t: Int, prng: Random) => prng.nextInt(3) == 0}.
http://git-wip-us.apache.org/repos/asf/spark/blob/ef48222c/core/src/test/scala/org/apache/spark/serializer/ProactiveClosureSerializationSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/serializer/ProactiveClosureSerializationSuite.scala b/core/src/test/scala/org/apache/spark/serializer/ProactiveClosureSerializationSuite.scala
index 5d15a68..aad6599 100644
--- a/core/src/test/scala/org/apache/spark/serializer/ProactiveClosureSerializationSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/ProactiveClosureSerializationSuite.scala
@@ -15,15 +15,12 @@
* limitations under the License.
*/
-package org.apache.spark.serializer;
-
-import java.io.NotSerializableException
+package org.apache.spark.serializer
import org.scalatest.FunSuite
+import org.apache.spark.{SharedSparkContext, SparkException}
import org.apache.spark.rdd.RDD
-import org.apache.spark.SparkException
-import org.apache.spark.SharedSparkContext
/* A trivial (but unserializable) container for trivial functions */
class UnserializableClass {
@@ -38,52 +35,50 @@ class ProactiveClosureSerializationSuite extends FunSuite with SharedSparkContex
test("throws expected serialization exceptions on actions") {
val (data, uc) = fixture
-
val ex = intercept[SparkException] {
- data.map(uc.op(_)).count
+ data.map(uc.op(_)).count()
}
-
assert(ex.getMessage.contains("Task not serializable"))
}
// There is probably a cleaner way to eliminate boilerplate here, but we're
// iterating over a map from transformation names to functions that perform that
// transformation on a given RDD, creating one test case for each
-
+
for (transformation <-
- Map("map" -> xmap _, "flatMap" -> xflatMap _, "filter" -> xfilter _,
- "mapWith" -> xmapWith _, "mapPartitions" -> xmapPartitions _,
+ Map("map" -> xmap _,
+ "flatMap" -> xflatMap _,
+ "filter" -> xfilter _,
+ "mapPartitions" -> xmapPartitions _,
"mapPartitionsWithIndex" -> xmapPartitionsWithIndex _,
- "mapPartitionsWithContext" -> xmapPartitionsWithContext _,
- "filterWith" -> xfilterWith _)) {
+ "mapPartitionsWithContext" -> xmapPartitionsWithContext _)) {
val (name, xf) = transformation
-
+
test(s"$name transformations throw proactive serialization exceptions") {
val (data, uc) = fixture
-
val ex = intercept[SparkException] {
xf(data, uc)
}
-
assert(ex.getMessage.contains("Task not serializable"),
s"RDD.$name doesn't proactively throw NotSerializableException")
}
}
-
+
private def xmap(x: RDD[String], uc: UnserializableClass): RDD[String] =
x.map(y=>uc.op(y))
- private def xmapWith(x: RDD[String], uc: UnserializableClass): RDD[String] =
- x.mapWith(x => x.toString)((x,y)=>x + uc.op(y))
+
private def xflatMap(x: RDD[String], uc: UnserializableClass): RDD[String] =
x.flatMap(y=>Seq(uc.op(y)))
+
private def xfilter(x: RDD[String], uc: UnserializableClass): RDD[String] =
x.filter(y=>uc.pred(y))
- private def xfilterWith(x: RDD[String], uc: UnserializableClass): RDD[String] =
- x.filterWith(x => x.toString)((x,y)=>uc.pred(y))
+
private def xmapPartitions(x: RDD[String], uc: UnserializableClass): RDD[String] =
x.mapPartitions(_.map(y=>uc.op(y)))
+
private def xmapPartitionsWithIndex(x: RDD[String], uc: UnserializableClass): RDD[String] =
x.mapPartitionsWithIndex((_, it) => it.map(y=>uc.op(y)))
+
private def xmapPartitionsWithContext(x: RDD[String], uc: UnserializableClass): RDD[String] =
x.mapPartitionsWithContext((_, it) => it.map(y=>uc.op(y)))
http://git-wip-us.apache.org/repos/asf/spark/blob/ef48222c/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala b/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala
index ca37d70..d2bee44 100644
--- a/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala
@@ -135,12 +135,11 @@ class FileAppenderSuite extends FunSuite with BeforeAndAfter with Logging {
val testOutputStream = new PipedOutputStream()
val testInputStream = new PipedInputStream(testOutputStream)
val appender = FileAppender(testInputStream, testFile, conf)
- assert(appender.isInstanceOf[ExpectedAppender])
+ //assert(appender.getClass === classTag[ExpectedAppender].getClass)
assert(appender.getClass.getSimpleName ===
classTag[ExpectedAppender].runtimeClass.getSimpleName)
if (appender.isInstanceOf[RollingFileAppender]) {
val rollingPolicy = appender.asInstanceOf[RollingFileAppender].rollingPolicy
- rollingPolicy.isInstanceOf[ExpectedRollingPolicy]
val policyParam = if (rollingPolicy.isInstanceOf[TimeBasedRollingPolicy]) {
rollingPolicy.asInstanceOf[TimeBasedRollingPolicy].rolloverIntervalMillis
} else {
http://git-wip-us.apache.org/repos/asf/spark/blob/ef48222c/core/src/test/scala/org/apache/spark/util/VectorSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/VectorSuite.scala b/core/src/test/scala/org/apache/spark/util/VectorSuite.scala
index 7006571..794a55d 100644
--- a/core/src/test/scala/org/apache/spark/util/VectorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/VectorSuite.scala
@@ -24,6 +24,7 @@ import org.scalatest.FunSuite
/**
* Tests org.apache.spark.util.Vector functionality
*/
+@deprecated("suppress compile time deprecation warning", "1.0.0")
class VectorSuite extends FunSuite {
def verifyVector(vector: Vector, expectedLength: Int) = {
http://git-wip-us.apache.org/repos/asf/spark/blob/ef48222c/streaming/src/test/scala/org/apache/spark/streaming/InputStreamsSuite.scala
----------------------------------------------------------------------
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/InputStreamsSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/InputStreamsSuite.scala
index cc4a650..952a74f 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/InputStreamsSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/InputStreamsSuite.scala
@@ -383,7 +383,10 @@ class TestActor(port: Int) extends Actor with ActorHelper {
def bytesToString(byteString: ByteString) = byteString.utf8String
- override def preStart = IOManager(context.system).connect(new InetSocketAddress(port))
+ override def preStart(): Unit = {
+ @deprecated("suppress compile time deprecation warning", "1.0.0")
+ val unit = IOManager(context.system).connect(new InetSocketAddress(port))
+ }
def receive = {
case IO.Read(socket, bytes) =>