You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by jo...@apache.org on 2014/12/30 18:33:37 UTC
spark git commit: Revert "[SPARK-4882] Register PythonBroadcast with
Kryo so that PySpark works with KryoSerializer"
Repository: spark
Updated Branches:
refs/heads/branch-1.1 822a0b42f -> d6b8d2c03
Revert "[SPARK-4882] Register PythonBroadcast with Kryo so that PySpark works with KryoSerializer"
This reverts commit 822a0b42f79acea2771d0b298e803c11c37aab81.
This fix does not apply to branch-1.1 or branch-1.0, since PythonBroadcast
is new in 1.2.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d6b8d2c0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d6b8d2c0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d6b8d2c0
Branch: refs/heads/branch-1.1
Commit: d6b8d2c0307b5cc2bacd0c07139003821055ca69
Parents: 822a0b4
Author: Josh Rosen <jo...@databricks.com>
Authored: Tue Dec 30 09:33:01 2014 -0800
Committer: Josh Rosen <jo...@databricks.com>
Committed: Tue Dec 30 09:33:07 2014 -0800
----------------------------------------------------------------------
.../spark/serializer/KryoSerializer.scala | 2 -
.../spark/api/python/PythonBroadcastSuite.scala | 60 --------------------
2 files changed, 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d6b8d2c0/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
index 0c8bd62..87ef9bb 100644
--- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -26,7 +26,6 @@ import com.esotericsoftware.kryo.serializers.{JavaSerializer => KryoJavaSerializ
import com.twitter.chill.{AllScalaRegistrar, EmptyScalaKryoInstantiator}
import org.apache.spark._
-import org.apache.spark.api.python.PythonBroadcast
import org.apache.spark.broadcast.HttpBroadcast
import org.apache.spark.scheduler.MapStatus
import org.apache.spark.storage._
@@ -80,7 +79,6 @@ class KryoSerializer(conf: SparkConf)
// Allow sending SerializableWritable
kryo.register(classOf[SerializableWritable[_]], new KryoJavaSerializer())
kryo.register(classOf[HttpBroadcast[_]], new KryoJavaSerializer())
- kryo.register(classOf[PythonBroadcast], new KryoJavaSerializer())
// Allow the user to register their own classes by setting spark.kryo.registrator
for (regCls <- registrator) {
http://git-wip-us.apache.org/repos/asf/spark/blob/d6b8d2c0/core/src/test/scala/org/apache/spark/api/python/PythonBroadcastSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/api/python/PythonBroadcastSuite.scala b/core/src/test/scala/org/apache/spark/api/python/PythonBroadcastSuite.scala
deleted file mode 100644
index 8959a84..0000000
--- a/core/src/test/scala/org/apache/spark/api/python/PythonBroadcastSuite.scala
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.api.python
-
-import scala.io.Source
-
-import java.io.{PrintWriter, File}
-
-import org.scalatest.{Matchers, FunSuite}
-
-import org.apache.spark.{SharedSparkContext, SparkConf}
-import org.apache.spark.serializer.KryoSerializer
-import org.apache.spark.util.Utils
-
-// This test suite uses SharedSparkContext because we need a SparkEnv in order to deserialize
-// a PythonBroadcast:
-class PythonBroadcastSuite extends FunSuite with Matchers with SharedSparkContext {
- test("PythonBroadcast can be serialized with Kryo (SPARK-4882)") {
- val tempDir = Utils.createTempDir()
- val broadcastedString = "Hello, world!"
- def assertBroadcastIsValid(broadcast: PythonBroadcast): Unit = {
- val source = Source.fromFile(broadcast.path)
- val contents = source.mkString
- source.close()
- contents should be (broadcastedString)
- }
- try {
- val broadcastDataFile: File = {
- val file = new File(tempDir, "broadcastData")
- val printWriter = new PrintWriter(file)
- printWriter.write(broadcastedString)
- printWriter.close()
- file
- }
- val broadcast = new PythonBroadcast(broadcastDataFile.getAbsolutePath)
- assertBroadcastIsValid(broadcast)
- val conf = new SparkConf().set("spark.kryo.registrationRequired", "true")
- val deserializedBroadcast =
- Utils.clone[PythonBroadcast](broadcast, new KryoSerializer(conf).newInstance())
- assertBroadcastIsValid(deserializedBroadcast)
- } finally {
- Utils.deleteRecursively(tempDir)
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org