You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2015/05/13 22:01:47 UTC
spark git commit: [MINOR] Enhance SizeEstimator to detect IBM compressed refs and s390 …
Repository: spark
Updated Branches:
refs/heads/master e676fc0c6 -> 3cd9ad240
[MINOR] Enhance SizeEstimator to detect IBM compressed refs and s390 …
…arch.
- zSeries 64-bit Java reports its architecture as s390x, so enhance the 64-bit check to accommodate that value.
- SizeEstimator can detect whether IBM Java is using compressed object pointers using info in the "java.vm.info" property, so will do a better job than failing on the HotSpot MBean and guessing.
Author: Tim Ellison <t....@gmail.com>
Closes #6085 from tellison/SizeEstimator and squashes the following commits:
1b6ff6a [Tim Ellison] Merge branch 'master' of https://github.com/apache/spark into SizeEstimator
0968989 [Tim Ellison] [MINOR] Enhance SizeEstimator to detect IBM compressed refs and s390 arch.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3cd9ad24
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3cd9ad24
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3cd9ad24
Branch: refs/heads/master
Commit: 3cd9ad2406c59cd0ede6c9c8428a4ce4b805f8fa
Parents: e676fc0
Author: Tim Ellison <t....@gmail.com>
Authored: Wed May 13 21:01:42 2015 +0100
Committer: Sean Owen <so...@cloudera.com>
Committed: Wed May 13 21:01:42 2015 +0100
----------------------------------------------------------------------
.../scala/org/apache/spark/util/SizeEstimator.scala | 8 +++++++-
.../org/apache/spark/util/SizeEstimatorSuite.scala | 12 ++++++++++++
2 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/3cd9ad24/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala b/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
index d91c329..968a72d 100644
--- a/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
+++ b/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
@@ -75,7 +75,8 @@ private[spark] object SizeEstimator extends Logging {
// Sets object size, pointer size based on architecture and CompressedOops settings
// from the JVM.
private def initialize() {
- is64bit = System.getProperty("os.arch").contains("64")
+ val arch = System.getProperty("os.arch")
+ is64bit = arch.contains("64") || arch.contains("s390x")
isCompressedOops = getIsCompressedOops
objectSize = if (!is64bit) 8 else {
@@ -97,6 +98,11 @@ private[spark] object SizeEstimator extends Logging {
return System.getProperty("spark.test.useCompressedOops").toBoolean
}
+ // java.vm.info provides compressed ref info for IBM JDKs
+ if (System.getProperty("java.vendor").contains("IBM")) {
+ return System.getProperty("java.vm.info").contains("Compressed Ref")
+ }
+
try {
val hotSpotMBeanName = "com.sun.management:type=HotSpotDiagnostic"
val server = ManagementFactory.getPlatformMBeanServer()
http://git-wip-us.apache.org/repos/asf/spark/blob/3cd9ad24/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala b/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
index 133a76f..04f0f37 100644
--- a/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
@@ -45,6 +45,10 @@ class DummyClass6 extends DummyClass5 {
val y: Boolean = true
}
+class DummyClass7 {
+ val x: DummyClass1 = new DummyClass1
+}
+
object DummyString {
def apply(str: String) : DummyString = new DummyString(str.toArray)
}
@@ -197,4 +201,12 @@ class SizeEstimatorSuite
assertResult(24)(SizeEstimator.estimate(new DummyClass5))
assertResult(32)(SizeEstimator.estimate(new DummyClass6))
}
+
+ test("check 64-bit detection for s390x arch") {
+ System.setProperty("os.arch", "s390x")
+ val initialize = PrivateMethod[Unit]('initialize)
+ SizeEstimator invokePrivate initialize()
+ // Class should be 32 bytes on s390x if recognised as 64 bit platform
+ assertResult(32)(SizeEstimator.estimate(new DummyClass7))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org