You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sh...@apache.org on 2015/07/30 19:46:31 UTC
spark git commit: [SPARK-9437] [CORE] avoid overflow in SizeEstimator
Repository: spark
Updated Branches:
refs/heads/master 520ec0ff9 -> 06b6a074f
[SPARK-9437] [CORE] avoid overflow in SizeEstimator
https://issues.apache.org/jira/browse/SPARK-9437
Author: Imran Rashid <ir...@cloudera.com>
Closes #7750 from squito/SPARK-9437_size_estimator_overflow and squashes the following commits:
29493f1 [Imran Rashid] prevent another potential overflow
bc1cb82 [Imran Rashid] avoid overflow
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/06b6a074
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/06b6a074
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/06b6a074
Branch: refs/heads/master
Commit: 06b6a074fb224b3fe23922bdc89fc5f7c2ffaaf6
Parents: 520ec0f
Author: Imran Rashid <ir...@cloudera.com>
Authored: Thu Jul 30 10:46:26 2015 -0700
Committer: Shivaram Venkataraman <sh...@cs.berkeley.edu>
Committed: Thu Jul 30 10:46:26 2015 -0700
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/util/SizeEstimator.scala | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/06b6a074/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala b/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
index 7d84468..14b1f2a 100644
--- a/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
+++ b/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
@@ -217,10 +217,10 @@ object SizeEstimator extends Logging {
var arrSize: Long = alignSize(objectSize + INT_SIZE)
if (elementClass.isPrimitive) {
- arrSize += alignSize(length * primitiveSize(elementClass))
+ arrSize += alignSize(length.toLong * primitiveSize(elementClass))
state.size += arrSize
} else {
- arrSize += alignSize(length * pointerSize)
+ arrSize += alignSize(length.toLong * pointerSize)
state.size += arrSize
if (length <= ARRAY_SIZE_FOR_SAMPLING) {
@@ -336,7 +336,7 @@ object SizeEstimator extends Logging {
// hg.openjdk.java.net/jdk8/jdk8/hotspot/file/tip/src/share/vm/classfile/classFileParser.cpp
var alignedSize = shellSize
for (size <- fieldSizes if sizeCount(size) > 0) {
- val count = sizeCount(size)
+ val count = sizeCount(size).toLong
// If there are internal gaps, smaller field can fit in.
alignedSize = math.max(alignedSize, alignSizeUp(shellSize, size) + size * count)
shellSize += size * count
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org