You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/05/29 01:57:04 UTC
spark git commit: Remove SizeEstimator from o.a.spark package.
Repository: spark
Updated Branches:
refs/heads/master 7859ab659 -> 0077af22c
Remove SizeEstimator from o.a.spark package.
See comments on https://github.com/apache/spark/pull/3913
Author: Reynold Xin <rx...@databricks.com>
Closes #6471 from rxin/sizeestimator and squashes the following commits:
c057095 [Reynold Xin] Fixed import.
2da478b [Reynold Xin] Remove SizeEstimator from o.a.spark package.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0077af22
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0077af22
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0077af22
Branch: refs/heads/master
Commit: 0077af22ca5fcb2e50dcf7daa4f6804ae722bfbe
Parents: 7859ab6
Author: Reynold Xin <rx...@databricks.com>
Authored: Thu May 28 16:56:59 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Thu May 28 16:56:59 2015 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/SizeEstimator.scala | 44 --------------------
.../org/apache/spark/util/SizeEstimator.scala | 20 +++++++--
2 files changed, 17 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0077af22/core/src/main/scala/org/apache/spark/SizeEstimator.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/SizeEstimator.scala b/core/src/main/scala/org/apache/spark/SizeEstimator.scala
deleted file mode 100644
index 54fc3a8..0000000
--- a/core/src/main/scala/org/apache/spark/SizeEstimator.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark
-
-import org.apache.spark.annotation.DeveloperApi
-
-/**
- * Estimates the sizes of Java objects (number of bytes of memory they occupy), for use in
- * memory-aware caches.
- *
- * Based on the following JavaWorld article:
- * http://www.javaworld.com/javaworld/javaqa/2003-12/02-qa-1226-sizeof.html
- */
-@DeveloperApi
-object SizeEstimator {
- /**
- * :: DeveloperApi ::
- * Estimate the number of bytes that the given object takes up on the JVM heap. The estimate
- * includes space taken up by objects referenced by the given object, their references, and so on
- * and so forth.
- *
- * This is useful for determining the amount of heap space a broadcast variable will occupy on
- * each executor or the amount of space each object will take when caching objects in
- * deserialized form. This is not the same as the serialized size of the object, which will
- * typically be much smaller.
- */
- @DeveloperApi
- def estimate(obj: AnyRef): Long = org.apache.spark.util.SizeEstimator.estimate(obj)
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/0077af22/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala b/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
index 968a72d..f38949c 100644
--- a/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
+++ b/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
@@ -21,21 +21,37 @@ import java.lang.management.ManagementFactory
import java.lang.reflect.{Field, Modifier}
import java.util.{IdentityHashMap, Random}
import java.util.concurrent.ConcurrentHashMap
+
import scala.collection.mutable.ArrayBuffer
import scala.runtime.ScalaRunTime
import org.apache.spark.Logging
+import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.util.collection.OpenHashSet
/**
+ * :: DeveloperApi ::
* Estimates the sizes of Java objects (number of bytes of memory they occupy), for use in
* memory-aware caches.
*
* Based on the following JavaWorld article:
* http://www.javaworld.com/javaworld/javaqa/2003-12/02-qa-1226-sizeof.html
*/
-private[spark] object SizeEstimator extends Logging {
+@DeveloperApi
+object SizeEstimator extends Logging {
+
+ /**
+ * Estimate the number of bytes that the given object takes up on the JVM heap. The estimate
+ * includes space taken up by objects referenced by the given object, their references, and so on
+ * and so forth.
+ *
+ * This is useful for determining the amount of heap space a broadcast variable will occupy on
+ * each executor or the amount of space each object will take when caching objects in
+ * deserialized form. This is not the same as the serialized size of the object, which will
+ * typically be much smaller.
+ */
+ def estimate(obj: AnyRef): Long = estimate(obj, new IdentityHashMap[AnyRef, AnyRef])
// Sizes of primitive types
private val BYTE_SIZE = 1
@@ -161,8 +177,6 @@ private[spark] object SizeEstimator extends Logging {
val shellSize: Long,
val pointerFields: List[Field]) {}
- def estimate(obj: AnyRef): Long = estimate(obj, new IdentityHashMap[AnyRef, AnyRef])
-
private def estimate(obj: AnyRef, visited: IdentityHashMap[AnyRef, AnyRef]): Long = {
val state = new SearchState(visited)
state.enqueue(obj)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org