You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2018/09/28 21:30:03 UTC
spark git commit: [SPARK-25542][CORE][TEST] Move flaky test in
OpenHashMapSuite to OpenHashSetSuite and make it against OpenHashSet
Repository: spark
Updated Branches:
refs/heads/master 0b33f0868 -> b7d80349b
[SPARK-25542][CORE][TEST] Move flaky test in OpenHashMapSuite to OpenHashSetSuite and make it against OpenHashSet
## What changes were proposed in this pull request?
The specified test in OpenHashMapSuite to test large items is somehow flaky to throw OOM.
By considering the original work #6763 that added this test, the test can be against OpenHashSetSuite. And by doing this should be to save memory because OpenHashMap allocates two more arrays when growing the map/set.
## How was this patch tested?
Existing tests.
Closes #22569 from viirya/SPARK-25542.
Authored-by: Liang-Chi Hsieh <vi...@gmail.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b7d80349
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b7d80349
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b7d80349
Branch: refs/heads/master
Commit: b7d80349b0e367d78cab238e62c2ec353f0f12b3
Parents: 0b33f08
Author: Liang-Chi Hsieh <vi...@gmail.com>
Authored: Fri Sep 28 14:29:56 2018 -0700
Committer: Dongjoon Hyun <do...@apache.org>
Committed: Fri Sep 28 14:29:56 2018 -0700
----------------------------------------------------------------------
.../spark/util/collection/OpenHashMapSuite.scala | 10 ----------
.../spark/util/collection/OpenHashSetSuite.scala | 13 +++++++++++++
2 files changed, 13 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b7d80349/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala
index 151235d..68bcc5e 100644
--- a/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala
@@ -185,16 +185,6 @@ class OpenHashMapSuite extends SparkFunSuite with Matchers {
assert(map.contains(null))
}
- test("support for more than 12M items") {
- val cnt = 12000000 // 12M
- val map = new OpenHashMap[Int, Int](cnt)
- for (i <- 0 until cnt) {
- map(i) = 1
- }
- val numInvalidValues = map.iterator.count(_._2 == 0)
- assertResult(0)(numInvalidValues)
- }
-
test("distinguish between the 0/0.0/0L and null") {
val specializedMap1 = new OpenHashMap[String, Long]
specializedMap1("a") = null.asInstanceOf[Long]
http://git-wip-us.apache.org/repos/asf/spark/blob/b7d80349/core/src/test/scala/org/apache/spark/util/collection/OpenHashSetSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/collection/OpenHashSetSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/OpenHashSetSuite.scala
index b887f93..44d2118 100644
--- a/core/src/test/scala/org/apache/spark/util/collection/OpenHashSetSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/collection/OpenHashSetSuite.scala
@@ -255,4 +255,17 @@ class OpenHashSetSuite extends SparkFunSuite with Matchers {
val set = new OpenHashSet[Long](0)
assert(set.size === 0)
}
+
+ test("support for more than 12M items") {
+ val cnt = 12000000 // 12M
+ val set = new OpenHashSet[Int](cnt)
+ for (i <- 0 until cnt) {
+ set.add(i)
+ assert(set.contains(i))
+
+ val pos1 = set.getPos(i)
+ val pos2 = set.addWithoutResize(i) & OpenHashSet.POSITION_MASK
+ assert(pos1 == pos2)
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org