You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2013/12/04 19:30:07 UTC
[1/3] git commit: Mark partitioner, name,
and generator field in RDD as @transient.
Updated Branches:
refs/heads/master 8a3475aed -> d6e547387
Mark partitioner, name, and generator field in RDD as @transient.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/e34b4693
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/e34b4693
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/e34b4693
Branch: refs/heads/master
Commit: e34b4693d380c39d4a142515e416588e63d06297
Parents: 740922f
Author: Reynold Xin <rx...@apache.org>
Authored: Mon Dec 2 21:24:44 2013 -0800
Committer: Reynold Xin <rx...@apache.org>
Committed: Mon Dec 2 21:24:44 2013 -0800
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/rdd/RDD.scala | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/e34b4693/core/src/main/scala/org/apache/spark/rdd/RDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
index 5b12853..96e4841 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
@@ -101,7 +101,7 @@ abstract class RDD[T: ClassManifest](
protected def getPreferredLocations(split: Partition): Seq[String] = Nil
/** Optionally overridden by subclasses to specify how they are partitioned. */
- val partitioner: Option[Partitioner] = None
+ @transient val partitioner: Option[Partitioner] = None
// =======================================================================
// Methods and fields available on all RDDs
@@ -114,7 +114,7 @@ abstract class RDD[T: ClassManifest](
val id: Int = sc.newRddId()
/** A friendly name for this RDD */
- var name: String = null
+ @transient var name: String = null
/** Assign a name to this RDD */
def setName(_name: String) = {
@@ -123,7 +123,7 @@ abstract class RDD[T: ClassManifest](
}
/** User-defined generator of this RDD*/
- var generator = Utils.getCallSiteInfo.firstUserClass
+ @transient var generator = Utils.getCallSiteInfo.firstUserClass
/** Reset generator*/
def setGenerator(_generator: String) = {
[2/3] git commit: Marked doCheckpointCalled as transient.
Posted by ma...@apache.org.
Marked doCheckpointCalled as transient.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/974a69d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/974a69d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/974a69d7
Branch: refs/heads/master
Commit: 974a69d79c4bf64fd9f27b65c1c464d33e647e20
Parents: e34b469
Author: Reynold Xin <rx...@apache.org>
Authored: Tue Dec 3 11:34:38 2013 -0800
Committer: Reynold Xin <rx...@apache.org>
Committed: Tue Dec 3 11:34:38 2013 -0800
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/rdd/RDD.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/974a69d7/core/src/main/scala/org/apache/spark/rdd/RDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
index 96e4841..893708f 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
@@ -940,7 +940,7 @@ abstract class RDD[T: ClassManifest](
private var storageLevel: StorageLevel = StorageLevel.NONE
/** Record user function generating this RDD. */
- private[spark] val origin = Utils.formatSparkCallSite
+ @transient private[spark] val origin = Utils.formatSparkCallSite
private[spark] def elementClassManifest: ClassManifest[T] = classManifest[T]
@@ -955,7 +955,7 @@ abstract class RDD[T: ClassManifest](
def context = sc
// Avoid handling doCheckpoint multiple times to prevent excessive recursion
- private var doCheckpointCalled = false
+ @transient private var doCheckpointCalled = false
/**
* Performs the checkpointing of this RDD by saving this. It is called by the DAGScheduler
[3/3] git commit: Merge pull request #223 from rxin/transient
Posted by ma...@apache.org.
Merge pull request #223 from rxin/transient
Mark partitioner, name, and generator field in RDD as @transient.
As part of the effort to reduce serialized task size.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/d6e54738
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/d6e54738
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/d6e54738
Branch: refs/heads/master
Commit: d6e5473872f405a6f4e466705e33cf893af915c1
Parents: 8a3475a 974a69d
Author: Matei Zaharia <ma...@eecs.berkeley.edu>
Authored: Wed Dec 4 10:28:50 2013 -0800
Committer: Matei Zaharia <ma...@eecs.berkeley.edu>
Committed: Wed Dec 4 10:28:50 2013 -0800
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/rdd/RDD.scala | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------