You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2014/06/08 23:18:56 UTC
git commit: SPARK-1628: Add missing hashCode methods in Partitioner
subclasses
Repository: spark
Updated Branches:
refs/heads/master ee96e9406 -> a71c6d1cf
SPARK-1628: Add missing hashCode methods in Partitioner subclasses
JIRA: https://issues.apache.org/jira/browse/SPARK-1628
Added `hashCode` in HashPartitioner, RangePartitioner, PythonPartitioner and PageRankUtils.CustomPartitioner.
Author: zsxwing <zs...@gmail.com>
Closes #549 from zsxwing/SPARK-1628 and squashes the following commits:
2620936 [zsxwing] SPARK-1628: Add missing hashCode methods in Partitioner subclasses
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a71c6d1c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a71c6d1c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a71c6d1c
Branch: refs/heads/master
Commit: a71c6d1cf0bbc027f633a65042191cf2201330d6
Parents: ee96e94
Author: zsxwing <zs...@gmail.com>
Authored: Sun Jun 8 14:18:52 2014 -0700
Committer: Reynold Xin <rx...@apache.org>
Committed: Sun Jun 8 14:18:52 2014 -0700
----------------------------------------------------------------------
.../main/scala/org/apache/spark/Partitioner.scala | 17 ++++++++++++++++-
.../spark/api/python/PythonPartitioner.scala | 2 ++
.../spark/examples/bagel/PageRankUtils.scala | 2 ++
3 files changed, 20 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/a71c6d1c/core/src/main/scala/org/apache/spark/Partitioner.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/Partitioner.scala b/core/src/main/scala/org/apache/spark/Partitioner.scala
index 9155159..01e918f 100644
--- a/core/src/main/scala/org/apache/spark/Partitioner.scala
+++ b/core/src/main/scala/org/apache/spark/Partitioner.scala
@@ -83,6 +83,8 @@ class HashPartitioner(partitions: Int) extends Partitioner {
case _ =>
false
}
+
+ override def hashCode: Int = numPartitions
}
/**
@@ -119,7 +121,7 @@ class RangePartitioner[K : Ordering : ClassTag, V](
}
}
- def numPartitions = partitions
+ def numPartitions = rangeBounds.length + 1
private val binarySearch: ((Array[K], K) => Int) = CollectionsUtils.makeBinarySearch[K]
@@ -155,4 +157,17 @@ class RangePartitioner[K : Ordering : ClassTag, V](
case _ =>
false
}
+
+
+ override def hashCode(): Int = {
+ val prime = 31
+ var result = 1
+ var i = 0
+ while (i < rangeBounds.length) {
+ result = prime * result + rangeBounds(i).hashCode
+ i += 1
+ }
+ result = prime * result + ascending.hashCode
+ result
+ }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a71c6d1c/core/src/main/scala/org/apache/spark/api/python/PythonPartitioner.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonPartitioner.scala b/core/src/main/scala/org/apache/spark/api/python/PythonPartitioner.scala
index 95bec50..e230d22 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonPartitioner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonPartitioner.scala
@@ -50,4 +50,6 @@ private[spark] class PythonPartitioner(
case _ =>
false
}
+
+ override def hashCode: Int = 31 * numPartitions + pyPartitionFunctionId.hashCode
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a71c6d1c/examples/src/main/scala/org/apache/spark/examples/bagel/PageRankUtils.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/examples/bagel/PageRankUtils.scala b/examples/src/main/scala/org/apache/spark/examples/bagel/PageRankUtils.scala
index b97cb8f..e06f4dc 100644
--- a/examples/src/main/scala/org/apache/spark/examples/bagel/PageRankUtils.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/bagel/PageRankUtils.scala
@@ -124,4 +124,6 @@ class CustomPartitioner(partitions: Int) extends Partitioner {
c.numPartitions == numPartitions
case _ => false
}
+
+ override def hashCode: Int = numPartitions
}