You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2020/09/14 21:41:24 UTC
[spark] branch branch-2.4 updated: Revert "[SPARK-32872][CORE]
Prevent BytesToBytesMap at MAX_CAPACITY from exceeding growth threshold"
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 94b9d6f Revert "[SPARK-32872][CORE] Prevent BytesToBytesMap at MAX_CAPACITY from exceeding growth threshold"
94b9d6f is described below
commit 94b9d6f3c868689333ddc0ea20f1b80fb4e10a5e
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Mon Sep 14 14:22:08 2020 -0700
Revert "[SPARK-32872][CORE] Prevent BytesToBytesMap at MAX_CAPACITY from exceeding growth threshold"
This reverts commit ad61af01a98487176e7bc15ef31d697c85d863f3.
---
.../apache/spark/unsafe/map/BytesToBytesMap.java | 21 ++++++---------------
1 file changed, 6 insertions(+), 15 deletions(-)
diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
index 4339e92..5ab52cc 100644
--- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
+++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
@@ -756,21 +756,12 @@ public final class BytesToBytesMap extends MemoryConsumer {
longArray.set(pos * 2 + 1, keyHashcode);
isDefined = true;
- // If the map has reached its growth threshold, try to grow it.
- if (numKeys >= growthThreshold) {
- // We use two array entries per key, so the array size is twice the capacity.
- // We should compare the current capacity of the array, instead of its size.
- if (longArray.size() / 2 < MAX_CAPACITY) {
- try {
- growAndRehash();
- } catch (SparkOutOfMemoryError oom) {
- canGrowArray = false;
- }
- } else {
- // The map is already at MAX_CAPACITY and cannot grow. Instead, we prevent it from
- // accepting any more new elements to make sure we don't exceed the load factor. If we
- // need to spill later, this allows UnsafeKVExternalSorter to reuse the array for
- // sorting.
+ // We use two array entries per key, so the array size is twice the capacity.
+ // We should compare the current capacity of the array, instead of its size.
+ if (numKeys >= growthThreshold && longArray.size() / 2 < MAX_CAPACITY) {
+ try {
+ growAndRehash();
+ } catch (OutOfMemoryError oom) {
canGrowArray = false;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org