You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2015/12/05 01:58:55 UTC
spark git commit: [SPARK-12080][CORE] Kryo - Support multiple user
registrators
Repository: spark
Updated Branches:
refs/heads/master bbfc16ec9 -> f30373f5e
[SPARK-12080][CORE] Kryo - Support multiple user registrators
Author: rotems <roter>
Closes #10078 from Botnaim/KryoMultipleCustomRegistrators.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f30373f5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f30373f5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f30373f5
Branch: refs/heads/master
Commit: f30373f5ee60f9892c28771e34b208e4f1f675a6
Parents: bbfc16e
Author: rotems <roter>
Authored: Fri Dec 4 16:58:31 2015 -0800
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Fri Dec 4 16:58:34 2015 -0800
----------------------------------------------------------------------
.../scala/org/apache/spark/serializer/KryoSerializer.scala | 6 ++++--
docs/configuration.md | 4 ++--
2 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/f30373f5/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
index d5ba690..7b77f78 100644
--- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -70,7 +70,9 @@ class KryoSerializer(conf: SparkConf)
private val referenceTracking = conf.getBoolean("spark.kryo.referenceTracking", true)
private val registrationRequired = conf.getBoolean("spark.kryo.registrationRequired", false)
- private val userRegistrator = conf.getOption("spark.kryo.registrator")
+ private val userRegistrators = conf.get("spark.kryo.registrator", "")
+ .split(',')
+ .filter(!_.isEmpty)
private val classesToRegister = conf.get("spark.kryo.classesToRegister", "")
.split(',')
.filter(!_.isEmpty)
@@ -119,7 +121,7 @@ class KryoSerializer(conf: SparkConf)
classesToRegister
.foreach { className => kryo.register(Class.forName(className, true, classLoader)) }
// Allow the user to register their own classes by setting spark.kryo.registrator.
- userRegistrator
+ userRegistrators
.map(Class.forName(_, true, classLoader).newInstance().asInstanceOf[KryoRegistrator])
.foreach { reg => reg.registerClasses(kryo) }
// scalastyle:on classforname
http://git-wip-us.apache.org/repos/asf/spark/blob/f30373f5/docs/configuration.md
----------------------------------------------------------------------
diff --git a/docs/configuration.md b/docs/configuration.md
index c39b489..fd61ddc 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -647,10 +647,10 @@ Apart from these, the following properties are also available, and may be useful
<td><code>spark.kryo.registrator</code></td>
<td>(none)</td>
<td>
- If you use Kryo serialization, set this class to register your custom classes with Kryo. This
+ If you use Kryo serialization, give a comma-separated list of classes that register your custom classes with Kryo. This
property is useful if you need to register your classes in a custom way, e.g. to specify a custom
field serializer. Otherwise <code>spark.kryo.classesToRegister</code> is simpler. It should be
- set to a class that extends
+ set to classes that extend
<a href="api/scala/index.html#org.apache.spark.serializer.KryoRegistrator">
<code>KryoRegistrator</code></a>.
See the <a href="tuning.html#data-serialization">tuning guide</a> for more details.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org