You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2015/06/19 11:58:09 UTC
spark git commit: [SPARK-7913] [CORE] Make AppendOnlyMap use the same
growth strategy of OpenHashSet and consistent exception message
Repository: spark
Updated Branches:
refs/heads/master 54557f353 -> 93360dc3c
[SPARK-7913] [CORE] Make AppendOnlyMap use the same growth strategy of OpenHashSet and consistent exception message
This is a follow up PR for #6456 to make AppendOnlyMap consistent with OpenHashSet.
/cc srowen andrewor14
Author: zsxwing <zs...@gmail.com>
Closes #6879 from zsxwing/append-only-map and squashes the following commits:
912c0ad [zsxwing] Fix the doc
dd4385b [zsxwing] Make AppendOnlyMap use the same growth strategy of OpenHashSet and consistent exception message
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/93360dc3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/93360dc3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/93360dc3
Branch: refs/heads/master
Commit: 93360dc3cd6186e9d33c762d153a829a5882b72b
Parents: 54557f3
Author: zsxwing <zs...@gmail.com>
Authored: Fri Jun 19 11:58:07 2015 +0200
Committer: Sean Owen <so...@cloudera.com>
Committed: Fri Jun 19 11:58:07 2015 +0200
----------------------------------------------------------------------
.../org/apache/spark/util/collection/AppendOnlyMap.scala | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/93360dc3/core/src/main/scala/org/apache/spark/util/collection/AppendOnlyMap.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/collection/AppendOnlyMap.scala b/core/src/main/scala/org/apache/spark/util/collection/AppendOnlyMap.scala
index d215ee4..4c1e161 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/AppendOnlyMap.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/AppendOnlyMap.scala
@@ -32,7 +32,7 @@ import org.apache.spark.annotation.DeveloperApi
* size, which is guaranteed to explore all spaces for each key (see
* http://en.wikipedia.org/wiki/Quadratic_probing).
*
- * The map can support up to `536870912 (2 ^ 29)` elements.
+ * The map can support up to `375809638 (0.7 * 2 ^ 29)` elements.
*
* TODO: Cache the hash values of each key? java.util.HashMap does that.
*/
@@ -199,11 +199,8 @@ class AppendOnlyMap[K, V](initialCapacity: Int = 64)
/** Increase table size by 1, rehashing if necessary */
private def incrementSize() {
- if (curSize == MAXIMUM_CAPACITY) {
- throw new IllegalStateException(s"Can't put more that ${MAXIMUM_CAPACITY} elements")
- }
curSize += 1
- if (curSize > growThreshold && capacity < MAXIMUM_CAPACITY) {
+ if (curSize > growThreshold) {
growTable()
}
}
@@ -216,7 +213,8 @@ class AppendOnlyMap[K, V](initialCapacity: Int = 64)
/** Double the table's size and re-hash everything */
protected def growTable() {
// capacity < MAXIMUM_CAPACITY (2 ^ 29) so capacity * 2 won't overflow
- val newCapacity = (capacity * 2).min(MAXIMUM_CAPACITY)
+ val newCapacity = capacity * 2
+ require(newCapacity <= MAXIMUM_CAPACITY, s"Can't contain more than ${growThreshold} elements")
val newData = new Array[AnyRef](2 * newCapacity)
val newMask = newCapacity - 1
// Insert all our old values into the new array. Note that because our old keys are
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org