You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2013/11/05 02:54:25 UTC
[11/12] git commit: Minor cleanup in ShuffleBlockManager
Minor cleanup in ShuffleBlockManager
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/1ba11b1c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/1ba11b1c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/1ba11b1c
Branch: refs/heads/master
Commit: 1ba11b1c6aeda084cb158262ec0aa37a7b70fe32
Parents: 6201e5e
Author: Aaron Davidson <aa...@databricks.com>
Authored: Mon Nov 4 17:16:20 2013 -0800
Committer: Aaron Davidson <aa...@databricks.com>
Committed: Mon Nov 4 17:16:41 2013 -0800
----------------------------------------------------------------------
.../scala/org/apache/spark/storage/ShuffleBlockManager.scala | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/1ba11b1c/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala
index 6346db3..2f1b049 100644
--- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala
+++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala
@@ -23,7 +23,6 @@ import java.util.concurrent.atomic.AtomicInteger
import scala.collection.JavaConversions._
-import org.apache.spark.Logging
import org.apache.spark.serializer.Serializer
import org.apache.spark.util.{MetadataCleanerType, MetadataCleaner, TimeStampedHashMap}
import org.apache.spark.util.collection.{PrimitiveKeyOpenHashMap, PrimitiveVector}
@@ -59,7 +58,7 @@ private[spark] trait ShuffleWriterGroup {
* files within a ShuffleFileGroups associated with the block's reducer.
*/
private[spark]
-class ShuffleBlockManager(blockManager: BlockManager) extends Logging {
+class ShuffleBlockManager(blockManager: BlockManager) {
// Turning off shuffle file consolidation causes all shuffle Blocks to get their own file.
// TODO: Remove this once the shuffle file consolidation feature is stable.
val consolidateShuffleFiles =
@@ -83,7 +82,7 @@ class ShuffleBlockManager(blockManager: BlockManager) extends Logging {
private
val metadataCleaner = new MetadataCleaner(MetadataCleanerType.SHUFFLE_BLOCK_MANAGER, this.cleanup)
- def forMapTask(shuffleId: Int, mapId: Int, numBuckets: Int, serializer: Serializer) =
+ def forMapTask(shuffleId: Int, mapId: Int, numBuckets: Int, serializer: Serializer) = {
new ShuffleWriterGroup {
shuffleStates.putIfAbsent(shuffleId, new ShuffleState())
private val shuffleState = shuffleStates(shuffleId)
@@ -133,6 +132,7 @@ class ShuffleBlockManager(blockManager: BlockManager) extends Logging {
shuffleState.unusedFileGroups.add(group)
}
}
+ }
/**
* Returns the physical file segment in which the given BlockId is located.
@@ -177,7 +177,7 @@ object ShuffleBlockManager {
* Note: mapIdToIndex(mapId) returns the index of the mapper into the vector for every
* reducer.
*/
- private val blockOffsetsByReducer = Array.tabulate[PrimitiveVector[Long]](files.length) { _ =>
+ private val blockOffsetsByReducer = Array.fill[PrimitiveVector[Long]](files.length) {
new PrimitiveVector[Long]()
}