You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2015/09/03 22:55:13 UTC
spark git commit: [SPARK-10430] [CORE] Added hashCode methods in
AccumulableInfo and RDDOperationScope
Repository: spark
Updated Branches:
refs/heads/master e62f4a46f -> 11ef32c5a
[SPARK-10430] [CORE] Added hashCode methods in AccumulableInfo and RDDOperationScope
Author: Vinod K C <vi...@huawei.com>
Closes #8581 from vinodkc/fix_RDDOperationScope_Hashcode.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/11ef32c5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/11ef32c5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/11ef32c5
Branch: refs/heads/master
Commit: 11ef32c5a1fad84574733ade1e9d50a94275842b
Parents: e62f4a4
Author: Vinod K C <vi...@huawei.com>
Authored: Thu Sep 3 13:54:58 2015 -0700
Committer: Andrew Or <an...@databricks.com>
Committed: Thu Sep 3 13:55:02 2015 -0700
----------------------------------------------------------------------
.../main/scala/org/apache/spark/rdd/RDDOperationScope.scala | 3 +++
.../scala/org/apache/spark/scheduler/AccumulableInfo.scala | 8 +++++++-
.../scala/org/apache/spark/rdd/RDDOperationScopeSuite.scala | 7 +++++++
.../org/apache/spark/scheduler/DAGSchedulerSuite.scala | 9 +++++++++
4 files changed, 26 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/11ef32c5/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
index 4466728..540cbd6 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
@@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.{JsonIgnore, JsonInclude, JsonPropertyOr
import com.fasterxml.jackson.annotation.JsonInclude.Include
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
+import com.google.common.base.Objects
import org.apache.spark.{Logging, SparkContext}
@@ -67,6 +68,8 @@ private[spark] class RDDOperationScope(
}
}
+ override def hashCode(): Int = Objects.hashCode(id, name, parent)
+
override def toString: String = toJson
}
http://git-wip-us.apache.org/repos/asf/spark/blob/11ef32c5/core/src/main/scala/org/apache/spark/scheduler/AccumulableInfo.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/AccumulableInfo.scala b/core/src/main/scala/org/apache/spark/scheduler/AccumulableInfo.scala
index 11d123e..b6bff64 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/AccumulableInfo.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/AccumulableInfo.scala
@@ -34,9 +34,15 @@ class AccumulableInfo private[spark] (
override def equals(other: Any): Boolean = other match {
case acc: AccumulableInfo =>
this.id == acc.id && this.name == acc.name &&
- this.update == acc.update && this.value == acc.value
+ this.update == acc.update && this.value == acc.value &&
+ this.internal == acc.internal
case _ => false
}
+
+ override def hashCode(): Int = {
+ val state = Seq(id, name, update, value, internal)
+ state.map(_.hashCode).reduceLeft(31 * _ + _)
+ }
}
object AccumulableInfo {
http://git-wip-us.apache.org/repos/asf/spark/blob/11ef32c5/core/src/test/scala/org/apache/spark/rdd/RDDOperationScopeSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/rdd/RDDOperationScopeSuite.scala b/core/src/test/scala/org/apache/spark/rdd/RDDOperationScopeSuite.scala
index f65349e..16a92f5 100644
--- a/core/src/test/scala/org/apache/spark/rdd/RDDOperationScopeSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/RDDOperationScopeSuite.scala
@@ -38,6 +38,13 @@ class RDDOperationScopeSuite extends SparkFunSuite with BeforeAndAfter {
sc.stop()
}
+ test("equals and hashCode") {
+ val opScope1 = new RDDOperationScope("scope1", id = "1")
+ val opScope2 = new RDDOperationScope("scope1", id = "1")
+ assert(opScope1 === opScope2)
+ assert(opScope1.hashCode() === opScope2.hashCode())
+ }
+
test("getAllScopes") {
assert(scope1.getAllScopes === Seq(scope1))
assert(scope2.getAllScopes === Seq(scope1, scope2))
http://git-wip-us.apache.org/repos/asf/spark/blob/11ef32c5/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
index 80f64de..2c3aa2c 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
@@ -285,6 +285,15 @@ class DAGSchedulerSuite
assertDataStructuresEmpty()
}
+ test("equals and hashCode AccumulableInfo") {
+ val accInfo1 = new AccumulableInfo(1, " Accumulable " + 1, Some("delta" + 1), "val" + 1, true)
+ val accInfo2 = new AccumulableInfo(1, " Accumulable " + 1, Some("delta" + 1), "val" + 1, false)
+ val accInfo3 = new AccumulableInfo(1, " Accumulable " + 1, Some("delta" + 1), "val" + 1, false)
+ assert(accInfo1 !== accInfo2)
+ assert(accInfo2 === accInfo3)
+ assert(accInfo2.hashCode() === accInfo3.hashCode())
+ }
+
test("cache location preferences w/ dependency") {
val baseRdd = new MyRDD(sc, 1, Nil).cache()
val finalRdd = new MyRDD(sc, 1, List(new OneToOneDependency(baseRdd)))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org