You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ir...@apache.org on 2017/11/09 21:56:21 UTC
[1/2] spark git commit: [SPARK-20647][CORE] Port StorageTab to the
new UI backend.
Repository: spark
Updated Branches:
refs/heads/master 6b19c0735 -> 6ae12715c
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/core/src/test/scala/org/apache/spark/ui/storage/StorageTabSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/ui/storage/StorageTabSuite.scala b/core/src/test/scala/org/apache/spark/ui/storage/StorageTabSuite.scala
deleted file mode 100644
index 79f02f2..0000000
--- a/core/src/test/scala/org/apache/spark/ui/storage/StorageTabSuite.scala
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.ui.storage
-
-import org.scalatest.BeforeAndAfter
-
-import org.apache.spark._
-import org.apache.spark.scheduler._
-import org.apache.spark.storage._
-
-/**
- * Test various functionality in the StorageListener that supports the StorageTab.
- */
-class StorageTabSuite extends SparkFunSuite with BeforeAndAfter {
- private var bus: SparkListenerBus = _
- private var storageStatusListener: StorageStatusListener = _
- private var storageListener: StorageListener = _
- private val memAndDisk = StorageLevel.MEMORY_AND_DISK
- private val memOnly = StorageLevel.MEMORY_ONLY
- private val none = StorageLevel.NONE
- private val taskInfo = new TaskInfo(0, 0, 0, 0, "big", "dog", TaskLocality.ANY, false)
- private val taskInfo1 = new TaskInfo(1, 1, 1, 1, "big", "cat", TaskLocality.ANY, false)
- private def rddInfo0 = new RDDInfo(0, "freedom", 100, memOnly, Seq(10))
- private def rddInfo1 = new RDDInfo(1, "hostage", 200, memOnly, Seq(10))
- private def rddInfo2 = new RDDInfo(2, "sanity", 300, memAndDisk, Seq(10))
- private def rddInfo3 = new RDDInfo(3, "grace", 400, memAndDisk, Seq(10))
- private val bm1 = BlockManagerId("big", "dog", 1)
-
- before {
- val conf = new SparkConf()
- bus = new ReplayListenerBus()
- storageStatusListener = new StorageStatusListener(conf)
- storageListener = new StorageListener(storageStatusListener)
- bus.addListener(storageStatusListener)
- bus.addListener(storageListener)
- }
-
- test("stage submitted / completed") {
- assert(storageListener._rddInfoMap.isEmpty)
- assert(storageListener.rddInfoList.isEmpty)
-
- // 2 RDDs are known, but none are cached
- val stageInfo0 = new StageInfo(0, 0, "0", 100, Seq(rddInfo0, rddInfo1), Seq.empty, "details")
- bus.postToAll(SparkListenerStageSubmitted(stageInfo0))
- assert(storageListener._rddInfoMap.size === 2)
- assert(storageListener.rddInfoList.isEmpty)
-
- // 4 RDDs are known, but only 2 are cached
- val rddInfo2Cached = rddInfo2
- val rddInfo3Cached = rddInfo3
- rddInfo2Cached.numCachedPartitions = 1
- rddInfo3Cached.numCachedPartitions = 1
- val stageInfo1 = new StageInfo(
- 1, 0, "0", 100, Seq(rddInfo2Cached, rddInfo3Cached), Seq.empty, "details")
- bus.postToAll(SparkListenerStageSubmitted(stageInfo1))
- assert(storageListener._rddInfoMap.size === 4)
- assert(storageListener.rddInfoList.size === 2)
-
- // Submitting RDDInfos with duplicate IDs does nothing
- val rddInfo0Cached = new RDDInfo(0, "freedom", 100, StorageLevel.MEMORY_ONLY, Seq(10))
- rddInfo0Cached.numCachedPartitions = 1
- val stageInfo0Cached = new StageInfo(0, 0, "0", 100, Seq(rddInfo0Cached), Seq.empty, "details")
- bus.postToAll(SparkListenerStageSubmitted(stageInfo0Cached))
- assert(storageListener._rddInfoMap.size === 4)
- assert(storageListener.rddInfoList.size === 2)
-
- // We only keep around the RDDs that are cached
- bus.postToAll(SparkListenerStageCompleted(stageInfo0))
- assert(storageListener._rddInfoMap.size === 2)
- assert(storageListener.rddInfoList.size === 2)
- }
-
- test("unpersist") {
- val rddInfo0Cached = rddInfo0
- val rddInfo1Cached = rddInfo1
- rddInfo0Cached.numCachedPartitions = 1
- rddInfo1Cached.numCachedPartitions = 1
- val stageInfo0 = new StageInfo(
- 0, 0, "0", 100, Seq(rddInfo0Cached, rddInfo1Cached), Seq.empty, "details")
- bus.postToAll(SparkListenerStageSubmitted(stageInfo0))
- assert(storageListener._rddInfoMap.size === 2)
- assert(storageListener.rddInfoList.size === 2)
- bus.postToAll(SparkListenerUnpersistRDD(0))
- assert(storageListener._rddInfoMap.size === 1)
- assert(storageListener.rddInfoList.size === 1)
- bus.postToAll(SparkListenerUnpersistRDD(4)) // doesn't exist
- assert(storageListener._rddInfoMap.size === 1)
- assert(storageListener.rddInfoList.size === 1)
- bus.postToAll(SparkListenerUnpersistRDD(1))
- assert(storageListener._rddInfoMap.size === 0)
- assert(storageListener.rddInfoList.size === 0)
- }
-
- test("block update") {
- val myRddInfo0 = rddInfo0
- val myRddInfo1 = rddInfo1
- val myRddInfo2 = rddInfo2
- val stageInfo0 = new StageInfo(
- 0, 0, "0", 100, Seq(myRddInfo0, myRddInfo1, myRddInfo2), Seq.empty, "details")
- bus.postToAll(SparkListenerBlockManagerAdded(1L, bm1, 1000L))
- bus.postToAll(SparkListenerStageSubmitted(stageInfo0))
- assert(storageListener._rddInfoMap.size === 3)
- assert(storageListener.rddInfoList.size === 0) // not cached
- assert(!storageListener._rddInfoMap(0).isCached)
- assert(!storageListener._rddInfoMap(1).isCached)
- assert(!storageListener._rddInfoMap(2).isCached)
-
- // Some blocks updated
- val blockUpdateInfos = Seq(
- BlockUpdatedInfo(bm1, RDDBlockId(0, 100), memAndDisk, 400L, 0L),
- BlockUpdatedInfo(bm1, RDDBlockId(0, 101), memAndDisk, 0L, 400L),
- BlockUpdatedInfo(bm1, RDDBlockId(1, 20), memAndDisk, 0L, 240L)
- )
- postUpdateBlocks(bus, blockUpdateInfos)
- assert(storageListener._rddInfoMap(0).memSize === 400L)
- assert(storageListener._rddInfoMap(0).diskSize === 400L)
- assert(storageListener._rddInfoMap(0).numCachedPartitions === 2)
- assert(storageListener._rddInfoMap(0).isCached)
- assert(storageListener._rddInfoMap(1).memSize === 0L)
- assert(storageListener._rddInfoMap(1).diskSize === 240L)
- assert(storageListener._rddInfoMap(1).numCachedPartitions === 1)
- assert(storageListener._rddInfoMap(1).isCached)
- assert(!storageListener._rddInfoMap(2).isCached)
- assert(storageListener._rddInfoMap(2).numCachedPartitions === 0)
-
- // Drop some blocks
- val blockUpdateInfos2 = Seq(
- BlockUpdatedInfo(bm1, RDDBlockId(0, 100), none, 0L, 0L),
- BlockUpdatedInfo(bm1, RDDBlockId(1, 20), none, 0L, 0L),
- BlockUpdatedInfo(bm1, RDDBlockId(2, 40), none, 0L, 0L), // doesn't actually exist
- BlockUpdatedInfo(bm1, RDDBlockId(4, 80), none, 0L, 0L) // doesn't actually exist
- )
- postUpdateBlocks(bus, blockUpdateInfos2)
- assert(storageListener._rddInfoMap(0).memSize === 0L)
- assert(storageListener._rddInfoMap(0).diskSize === 400L)
- assert(storageListener._rddInfoMap(0).numCachedPartitions === 1)
- assert(storageListener._rddInfoMap(0).isCached)
- assert(!storageListener._rddInfoMap(1).isCached)
- assert(storageListener._rddInfoMap(2).numCachedPartitions === 0)
- assert(!storageListener._rddInfoMap(2).isCached)
- assert(storageListener._rddInfoMap(2).numCachedPartitions === 0)
- }
-
- test("verify StorageTab contains all cached rdds") {
-
- val rddInfo0 = new RDDInfo(0, "rdd0", 1, memOnly, Seq(4))
- val rddInfo1 = new RDDInfo(1, "rdd1", 1, memOnly, Seq(4))
- val stageInfo0 = new StageInfo(0, 0, "stage0", 1, Seq(rddInfo0), Seq.empty, "details")
- val stageInfo1 = new StageInfo(1, 0, "stage1", 1, Seq(rddInfo1), Seq.empty, "details")
- val blockUpdateInfos1 = Seq(BlockUpdatedInfo(bm1, RDDBlockId(0, 1), memOnly, 100L, 0L))
- val blockUpdateInfos2 = Seq(BlockUpdatedInfo(bm1, RDDBlockId(1, 1), memOnly, 200L, 0L))
- bus.postToAll(SparkListenerBlockManagerAdded(1L, bm1, 1000L))
- bus.postToAll(SparkListenerStageSubmitted(stageInfo0))
- assert(storageListener.rddInfoList.size === 0)
- postUpdateBlocks(bus, blockUpdateInfos1)
- assert(storageListener.rddInfoList.size === 1)
- bus.postToAll(SparkListenerStageSubmitted(stageInfo1))
- assert(storageListener.rddInfoList.size === 1)
- bus.postToAll(SparkListenerStageCompleted(stageInfo0))
- assert(storageListener.rddInfoList.size === 1)
- postUpdateBlocks(bus, blockUpdateInfos2)
- assert(storageListener.rddInfoList.size === 2)
- bus.postToAll(SparkListenerStageCompleted(stageInfo1))
- assert(storageListener.rddInfoList.size === 2)
- }
-
- test("verify StorageTab still contains a renamed RDD") {
- val rddInfo = new RDDInfo(0, "original_name", 1, memOnly, Seq(4))
- val stageInfo0 = new StageInfo(0, 0, "stage0", 1, Seq(rddInfo), Seq.empty, "details")
- bus.postToAll(SparkListenerBlockManagerAdded(1L, bm1, 1000L))
- bus.postToAll(SparkListenerStageSubmitted(stageInfo0))
- val blockUpdateInfos1 = Seq(BlockUpdatedInfo(bm1, RDDBlockId(0, 1), memOnly, 100L, 0L))
- postUpdateBlocks(bus, blockUpdateInfos1)
- assert(storageListener.rddInfoList.size == 1)
-
- val newName = "new_name"
- val rddInfoRenamed = new RDDInfo(0, newName, 1, memOnly, Seq(4))
- val stageInfo1 = new StageInfo(1, 0, "stage1", 1, Seq(rddInfoRenamed), Seq.empty, "details")
- bus.postToAll(SparkListenerStageSubmitted(stageInfo1))
- assert(storageListener.rddInfoList.size == 1)
- assert(storageListener.rddInfoList.head.name == newName)
- }
-
- private def postUpdateBlocks(
- bus: SparkListenerBus, blockUpdateInfos: Seq[BlockUpdatedInfo]): Unit = {
- blockUpdateInfos.foreach { blockUpdateInfo =>
- bus.postToAll(SparkListenerBlockUpdated(blockUpdateInfo))
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/project/MimaExcludes.scala
----------------------------------------------------------------------
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index 0c31b2b..090375f 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -41,6 +41,8 @@ object MimaExcludes {
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.deploy.history.HistoryServer.getSparkUI"),
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.ui.env.EnvironmentListener"),
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.ui.exec.ExecutorsListener"),
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.ui.storage.StorageListener"),
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.storage.StorageStatusListener"),
// [SPARK-20495][SQL] Add StorageLevel to cacheTable API
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.cacheTable"),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[2/2] spark git commit: [SPARK-20647][CORE] Port StorageTab to the
new UI backend.
Posted by ir...@apache.org.
[SPARK-20647][CORE] Port StorageTab to the new UI backend.
This required adding information about StreamBlockId to the store,
which is not available yet via the API. So an internal type was added
until there's a need to expose that information in the API.
The UI only lists RDDs that have cached partitions, and that information
wasn't being correctly captured in the listener, so that's also fixed,
along with some minor (internal) API adjustments so that the UI can
get the correct data.
Because of the way partitions are cached, some optimizations w.r.t. how
often the data is flushed to the store could not be applied to this code;
because of that, some different ways to make the code more performant
were added to the data structures tracking RDD blocks, with the goal of
avoiding expensive copies when lots of blocks are being updated.
Tested with existing and updated unit tests.
Author: Marcelo Vanzin <va...@cloudera.com>
Closes #19679 from vanzin/SPARK-20647.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6ae12715
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6ae12715
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6ae12715
Branch: refs/heads/master
Commit: 6ae12715c7286abc87cc7a22b1fa955e844077d5
Parents: 6b19c07
Author: Marcelo Vanzin <va...@cloudera.com>
Authored: Thu Nov 9 15:46:16 2017 -0600
Committer: Imran Rashid <ir...@cloudera.com>
Committed: Thu Nov 9 15:46:16 2017 -0600
----------------------------------------------------------------------
.../apache/spark/status/AppStatusListener.scala | 97 ++++++---
.../apache/spark/status/AppStatusStore.scala | 10 +-
.../org/apache/spark/status/LiveEntity.scala | 170 +++++++++++----
.../spark/status/api/v1/AllRDDResource.scala | 81 +-------
.../spark/status/api/v1/OneRDDResource.scala | 10 +-
.../org/apache/spark/status/storeTypes.scala | 16 ++
.../spark/storage/BlockStatusListener.scala | 100 ---------
.../spark/storage/StorageStatusListener.scala | 111 ----------
.../scala/org/apache/spark/ui/SparkUI.scala | 17 +-
.../org/apache/spark/ui/storage/RDDPage.scala | 16 +-
.../apache/spark/ui/storage/StoragePage.scala | 78 ++++---
.../apache/spark/ui/storage/StorageTab.scala | 68 +-----
.../spark/status/AppStatusListenerSuite.scala | 186 ++++++++++++-----
.../apache/spark/status/LiveEntitySuite.scala | 68 ++++++
.../storage/BlockStatusListenerSuite.scala | 113 ----------
.../storage/StorageStatusListenerSuite.scala | 167 ---------------
.../spark/ui/storage/StoragePageSuite.scala | 128 +++++++-----
.../spark/ui/storage/StorageTabSuite.scala | 205 -------------------
project/MimaExcludes.scala | 2 +
19 files changed, 586 insertions(+), 1057 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
index 0469c87..7f2c00c 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
@@ -146,10 +146,19 @@ private[spark] class AppStatusListener(
override def onExecutorRemoved(event: SparkListenerExecutorRemoved): Unit = {
liveExecutors.remove(event.executorId).foreach { exec =>
+ val now = System.nanoTime()
exec.isActive = false
exec.removeTime = new Date(event.time)
exec.removeReason = event.reason
- update(exec, System.nanoTime())
+ update(exec, now)
+
+ // Remove all RDD distributions that reference the removed executor, in case there wasn't
+ // a corresponding event.
+ liveRDDs.values.foreach { rdd =>
+ if (rdd.removeDistribution(exec)) {
+ update(rdd, now)
+ }
+ }
}
}
@@ -465,7 +474,8 @@ private[spark] class AppStatusListener(
override def onBlockUpdated(event: SparkListenerBlockUpdated): Unit = {
event.blockUpdatedInfo.blockId match {
case block: RDDBlockId => updateRDDBlock(event, block)
- case _ => // TODO: API only covers RDD storage.
+ case stream: StreamBlockId => updateStreamBlock(event, stream)
+ case _ =>
}
}
@@ -502,29 +512,47 @@ private[spark] class AppStatusListener(
val maybeExec = liveExecutors.get(executorId)
var rddBlocksDelta = 0
+ // Update the executor stats first, since they are used to calculate the free memory
+ // on tracked RDD distributions.
+ maybeExec.foreach { exec =>
+ if (exec.hasMemoryInfo) {
+ if (storageLevel.useOffHeap) {
+ exec.usedOffHeap = newValue(exec.usedOffHeap, memoryDelta)
+ } else {
+ exec.usedOnHeap = newValue(exec.usedOnHeap, memoryDelta)
+ }
+ }
+ exec.memoryUsed = newValue(exec.memoryUsed, memoryDelta)
+ exec.diskUsed = newValue(exec.diskUsed, diskDelta)
+ }
+
// Update the block entry in the RDD info, keeping track of the deltas above so that we
// can update the executor information too.
liveRDDs.get(block.rddId).foreach { rdd =>
+ if (updatedStorageLevel.isDefined) {
+ rdd.storageLevel = updatedStorageLevel.get
+ }
+
val partition = rdd.partition(block.name)
val executors = if (updatedStorageLevel.isDefined) {
- if (!partition.executors.contains(executorId)) {
+ val current = partition.executors
+ if (current.contains(executorId)) {
+ current
+ } else {
rddBlocksDelta = 1
+ current :+ executorId
}
- partition.executors + executorId
} else {
rddBlocksDelta = -1
- partition.executors - executorId
+ partition.executors.filter(_ != executorId)
}
// Only update the partition if it's still stored in some executor, otherwise get rid of it.
if (executors.nonEmpty) {
- if (updatedStorageLevel.isDefined) {
- partition.storageLevel = updatedStorageLevel.get
- }
- partition.memoryUsed = newValue(partition.memoryUsed, memoryDelta)
- partition.diskUsed = newValue(partition.diskUsed, diskDelta)
- partition.executors = executors
+ partition.update(executors, rdd.storageLevel,
+ newValue(partition.memoryUsed, memoryDelta),
+ newValue(partition.diskUsed, diskDelta))
} else {
rdd.removePartition(block.name)
}
@@ -532,42 +560,39 @@ private[spark] class AppStatusListener(
maybeExec.foreach { exec =>
if (exec.rddBlocks + rddBlocksDelta > 0) {
val dist = rdd.distribution(exec)
- dist.memoryRemaining = newValue(dist.memoryRemaining, -memoryDelta)
dist.memoryUsed = newValue(dist.memoryUsed, memoryDelta)
dist.diskUsed = newValue(dist.diskUsed, diskDelta)
if (exec.hasMemoryInfo) {
if (storageLevel.useOffHeap) {
dist.offHeapUsed = newValue(dist.offHeapUsed, memoryDelta)
- dist.offHeapRemaining = newValue(dist.offHeapRemaining, -memoryDelta)
} else {
dist.onHeapUsed = newValue(dist.onHeapUsed, memoryDelta)
- dist.onHeapRemaining = newValue(dist.onHeapRemaining, -memoryDelta)
}
}
+ dist.lastUpdate = null
} else {
rdd.removeDistribution(exec)
}
- }
- if (updatedStorageLevel.isDefined) {
- rdd.storageLevel = updatedStorageLevel.get
+ // Trigger an update on other RDDs so that the free memory information is updated.
+ liveRDDs.values.foreach { otherRdd =>
+ if (otherRdd.info.id != block.rddId) {
+ otherRdd.distributionOpt(exec).foreach { dist =>
+ dist.lastUpdate = null
+ update(otherRdd, now)
+ }
+ }
+ }
}
+
rdd.memoryUsed = newValue(rdd.memoryUsed, memoryDelta)
rdd.diskUsed = newValue(rdd.diskUsed, diskDelta)
update(rdd, now)
}
+ // Finish updating the executor now that we know the delta in the number of blocks.
maybeExec.foreach { exec =>
- if (exec.hasMemoryInfo) {
- if (storageLevel.useOffHeap) {
- exec.usedOffHeap = newValue(exec.usedOffHeap, memoryDelta)
- } else {
- exec.usedOnHeap = newValue(exec.usedOnHeap, memoryDelta)
- }
- }
- exec.memoryUsed = newValue(exec.memoryUsed, memoryDelta)
- exec.diskUsed = newValue(exec.diskUsed, diskDelta)
exec.rddBlocks += rddBlocksDelta
maybeUpdate(exec, now)
}
@@ -577,6 +602,26 @@ private[spark] class AppStatusListener(
liveExecutors.getOrElseUpdate(executorId, new LiveExecutor(executorId, addTime))
}
+ private def updateStreamBlock(event: SparkListenerBlockUpdated, stream: StreamBlockId): Unit = {
+ val storageLevel = event.blockUpdatedInfo.storageLevel
+ if (storageLevel.isValid) {
+ val data = new StreamBlockData(
+ stream.name,
+ event.blockUpdatedInfo.blockManagerId.executorId,
+ event.blockUpdatedInfo.blockManagerId.hostPort,
+ storageLevel.description,
+ storageLevel.useMemory,
+ storageLevel.useDisk,
+ storageLevel.deserialized,
+ event.blockUpdatedInfo.memSize,
+ event.blockUpdatedInfo.diskSize)
+ kvstore.write(data)
+ } else {
+ kvstore.delete(classOf[StreamBlockData],
+ Array(stream.name, event.blockUpdatedInfo.blockManagerId.executorId))
+ }
+ }
+
private def getOrCreateStage(info: StageInfo): LiveStage = {
val stage = liveStages.getOrElseUpdate((info.stageId, info.attemptId), new LiveStage())
stage.info = info
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
index 3344078..80c8d7d 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
@@ -209,14 +209,20 @@ private[spark] class AppStatusStore(store: KVStore) {
indexed.skip(offset).max(length).asScala.map(_.info).toSeq
}
- def rddList(): Seq[v1.RDDStorageInfo] = {
- store.view(classOf[RDDStorageInfoWrapper]).asScala.map(_.info).toSeq
+ def rddList(cachedOnly: Boolean = true): Seq[v1.RDDStorageInfo] = {
+ store.view(classOf[RDDStorageInfoWrapper]).asScala.map(_.info).filter { rdd =>
+ !cachedOnly || rdd.numCachedPartitions > 0
+ }.toSeq
}
def rdd(rddId: Int): v1.RDDStorageInfo = {
store.read(classOf[RDDStorageInfoWrapper], rddId).info
}
+ def streamBlocksList(): Seq[StreamBlockData] = {
+ store.view(classOf[StreamBlockData]).asScala.toSeq
+ }
+
def close(): Unit = {
store.close()
}
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
index 8c48020..706d94c 100644
--- a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
+++ b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
@@ -420,79 +420,101 @@ private class LiveStage extends LiveEntity {
private class LiveRDDPartition(val blockName: String) {
- var executors = Set[String]()
- var storageLevel: String = null
- var memoryUsed = 0L
- var diskUsed = 0L
+ // Pointers used by RDDPartitionSeq.
+ @volatile var prev: LiveRDDPartition = null
+ @volatile var next: LiveRDDPartition = null
+
+ var value: v1.RDDPartitionInfo = null
+
+ def executors: Seq[String] = value.executors
+
+ def memoryUsed: Long = value.memoryUsed
+
+ def diskUsed: Long = value.diskUsed
- def toApi(): v1.RDDPartitionInfo = {
- new v1.RDDPartitionInfo(
+ def update(
+ executors: Seq[String],
+ storageLevel: String,
+ memoryUsed: Long,
+ diskUsed: Long): Unit = {
+ value = new v1.RDDPartitionInfo(
blockName,
storageLevel,
memoryUsed,
diskUsed,
- executors.toSeq.sorted)
+ executors)
}
}
-private class LiveRDDDistribution(val exec: LiveExecutor) {
+private class LiveRDDDistribution(exec: LiveExecutor) {
- var memoryRemaining = exec.maxMemory
+ val executorId = exec.executorId
var memoryUsed = 0L
var diskUsed = 0L
var onHeapUsed = 0L
var offHeapUsed = 0L
- var onHeapRemaining = 0L
- var offHeapRemaining = 0L
+
+ // Keep the last update handy. This avoids recomputing the API view when not needed.
+ var lastUpdate: v1.RDDDataDistribution = null
def toApi(): v1.RDDDataDistribution = {
- new v1.RDDDataDistribution(
- exec.hostPort,
- memoryUsed,
- memoryRemaining,
- diskUsed,
- if (exec.hasMemoryInfo) Some(onHeapUsed) else None,
- if (exec.hasMemoryInfo) Some(offHeapUsed) else None,
- if (exec.hasMemoryInfo) Some(onHeapRemaining) else None,
- if (exec.hasMemoryInfo) Some(offHeapRemaining) else None)
+ if (lastUpdate == null) {
+ lastUpdate = new v1.RDDDataDistribution(
+ exec.hostPort,
+ memoryUsed,
+ exec.maxMemory - exec.memoryUsed,
+ diskUsed,
+ if (exec.hasMemoryInfo) Some(onHeapUsed) else None,
+ if (exec.hasMemoryInfo) Some(offHeapUsed) else None,
+ if (exec.hasMemoryInfo) Some(exec.totalOnHeap - exec.usedOnHeap) else None,
+ if (exec.hasMemoryInfo) Some(exec.totalOffHeap - exec.usedOffHeap) else None)
+ }
+ lastUpdate
}
}
-private class LiveRDD(info: RDDInfo) extends LiveEntity {
+private class LiveRDD(val info: RDDInfo) extends LiveEntity {
var storageLevel: String = info.storageLevel.description
var memoryUsed = 0L
var diskUsed = 0L
private val partitions = new HashMap[String, LiveRDDPartition]()
+ private val partitionSeq = new RDDPartitionSeq()
+
private val distributions = new HashMap[String, LiveRDDDistribution]()
def partition(blockName: String): LiveRDDPartition = {
- partitions.getOrElseUpdate(blockName, new LiveRDDPartition(blockName))
+ partitions.getOrElseUpdate(blockName, {
+ val part = new LiveRDDPartition(blockName)
+ part.update(Nil, storageLevel, 0L, 0L)
+ partitionSeq.addPartition(part)
+ part
+ })
}
- def removePartition(blockName: String): Unit = partitions.remove(blockName)
+ def removePartition(blockName: String): Unit = {
+ partitions.remove(blockName).foreach(partitionSeq.removePartition)
+ }
def distribution(exec: LiveExecutor): LiveRDDDistribution = {
- distributions.getOrElseUpdate(exec.hostPort, new LiveRDDDistribution(exec))
+ distributions.getOrElseUpdate(exec.executorId, new LiveRDDDistribution(exec))
}
- def removeDistribution(exec: LiveExecutor): Unit = {
- distributions.remove(exec.hostPort)
+ def removeDistribution(exec: LiveExecutor): Boolean = {
+ distributions.remove(exec.executorId).isDefined
}
- override protected def doUpdate(): Any = {
- val parts = if (partitions.nonEmpty) {
- Some(partitions.values.toList.sortBy(_.blockName).map(_.toApi()))
- } else {
- None
- }
+ def distributionOpt(exec: LiveExecutor): Option[LiveRDDDistribution] = {
+ distributions.get(exec.executorId)
+ }
+ override protected def doUpdate(): Any = {
val dists = if (distributions.nonEmpty) {
- Some(distributions.values.toList.sortBy(_.exec.executorId).map(_.toApi()))
+ Some(distributions.values.map(_.toApi()).toSeq)
} else {
None
}
@@ -506,7 +528,7 @@ private class LiveRDD(info: RDDInfo) extends LiveEntity {
memoryUsed,
diskUsed,
dists,
- parts)
+ Some(partitionSeq))
new RDDStorageInfoWrapper(rdd)
}
@@ -526,7 +548,7 @@ private object LiveEntityHelpers {
.map { acc =>
new v1.AccumulableInfo(
acc.id,
- acc.name.map(_.intern()).orNull,
+ acc.name.orNull,
acc.update.map(_.toString()),
acc.value.map(_.toString()).orNull)
}
@@ -534,3 +556,81 @@ private object LiveEntityHelpers {
}
}
+
+/**
+ * A custom sequence of partitions based on a mutable linked list.
+ *
+ * The external interface is an immutable Seq, which is thread-safe for traversal. There are no
+ * guarantees about consistency though - iteration might return elements that have been removed
+ * or miss added elements.
+ *
+ * Internally, the sequence is mutable, and elements can modify the data they expose. Additions and
+ * removals are O(1). It is not safe to do multiple writes concurrently.
+ */
+private class RDDPartitionSeq extends Seq[v1.RDDPartitionInfo] {
+
+ @volatile private var _head: LiveRDDPartition = null
+ @volatile private var _tail: LiveRDDPartition = null
+ @volatile var count = 0
+
+ override def apply(idx: Int): v1.RDDPartitionInfo = {
+ var curr = 0
+ var e = _head
+ while (curr < idx && e != null) {
+ curr += 1
+ e = e.next
+ }
+ if (e != null) e.value else throw new IndexOutOfBoundsException(idx.toString)
+ }
+
+ override def iterator: Iterator[v1.RDDPartitionInfo] = {
+ new Iterator[v1.RDDPartitionInfo] {
+ var current = _head
+
+ override def hasNext: Boolean = current != null
+
+ override def next(): v1.RDDPartitionInfo = {
+ if (current != null) {
+ val tmp = current
+ current = tmp.next
+ tmp.value
+ } else {
+ throw new NoSuchElementException()
+ }
+ }
+ }
+ }
+
+ override def length: Int = count
+
+ def addPartition(part: LiveRDDPartition): Unit = {
+ part.prev = _tail
+ if (_tail != null) {
+ _tail.next = part
+ }
+ if (_head == null) {
+ _head = part
+ }
+ _tail = part
+ count += 1
+ }
+
+ def removePartition(part: LiveRDDPartition): Unit = {
+ count -= 1
+ // Remove the partition from the list, but leave the pointers unchanged. That ensures a best
+ // effort at returning existing elements when iterations still reference the removed partition.
+ if (part.prev != null) {
+ part.prev.next = part.next
+ }
+ if (part eq _head) {
+ _head = part.next
+ }
+ if (part.next != null) {
+ part.next.prev = part.prev
+ }
+ if (part eq _tail) {
+ _tail = part.prev
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/core/src/main/scala/org/apache/spark/status/api/v1/AllRDDResource.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/AllRDDResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/AllRDDResource.scala
index 1279b28..2189e1d 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/AllRDDResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/AllRDDResource.scala
@@ -21,90 +21,11 @@ import javax.ws.rs.core.MediaType
import org.apache.spark.storage.{RDDInfo, StorageStatus, StorageUtils}
import org.apache.spark.ui.SparkUI
-import org.apache.spark.ui.storage.StorageListener
@Produces(Array(MediaType.APPLICATION_JSON))
private[v1] class AllRDDResource(ui: SparkUI) {
@GET
- def rddList(): Seq[RDDStorageInfo] = {
- val storageStatusList = ui.storageListener.activeStorageStatusList
- val rddInfos = ui.storageListener.rddInfoList
- rddInfos.map{rddInfo =>
- AllRDDResource.getRDDStorageInfo(rddInfo.id, rddInfo, storageStatusList,
- includeDetails = false)
- }
- }
+ def rddList(): Seq[RDDStorageInfo] = ui.store.rddList()
}
-
-private[spark] object AllRDDResource {
-
- def getRDDStorageInfo(
- rddId: Int,
- listener: StorageListener,
- includeDetails: Boolean): Option[RDDStorageInfo] = {
- val storageStatusList = listener.activeStorageStatusList
- listener.rddInfoList.find { _.id == rddId }.map { rddInfo =>
- getRDDStorageInfo(rddId, rddInfo, storageStatusList, includeDetails)
- }
- }
-
- def getRDDStorageInfo(
- rddId: Int,
- rddInfo: RDDInfo,
- storageStatusList: Seq[StorageStatus],
- includeDetails: Boolean): RDDStorageInfo = {
- val workers = storageStatusList.map { (rddId, _) }
- val blockLocations = StorageUtils.getRddBlockLocations(rddId, storageStatusList)
- val blocks = storageStatusList
- .flatMap { _.rddBlocksById(rddId) }
- .sortWith { _._1.name < _._1.name }
- .map { case (blockId, status) =>
- (blockId, status, blockLocations.getOrElse(blockId, Seq[String]("Unknown")))
- }
-
- val dataDistribution = if (includeDetails) {
- Some(storageStatusList.map { status =>
- new RDDDataDistribution(
- address = status.blockManagerId.hostPort,
- memoryUsed = status.memUsedByRdd(rddId),
- memoryRemaining = status.memRemaining,
- diskUsed = status.diskUsedByRdd(rddId),
- onHeapMemoryUsed = Some(
- if (!rddInfo.storageLevel.useOffHeap) status.memUsedByRdd(rddId) else 0L),
- offHeapMemoryUsed = Some(
- if (rddInfo.storageLevel.useOffHeap) status.memUsedByRdd(rddId) else 0L),
- onHeapMemoryRemaining = status.onHeapMemRemaining,
- offHeapMemoryRemaining = status.offHeapMemRemaining
- ) } )
- } else {
- None
- }
- val partitions = if (includeDetails) {
- Some(blocks.map { case (id, block, locations) =>
- new RDDPartitionInfo(
- blockName = id.name,
- storageLevel = block.storageLevel.description,
- memoryUsed = block.memSize,
- diskUsed = block.diskSize,
- executors = locations
- )
- } )
- } else {
- None
- }
-
- new RDDStorageInfo(
- id = rddId,
- name = rddInfo.name,
- numPartitions = rddInfo.numPartitions,
- numCachedPartitions = rddInfo.numCachedPartitions,
- storageLevel = rddInfo.storageLevel.description,
- memoryUsed = rddInfo.memSize,
- diskUsed = rddInfo.diskSize,
- dataDistribution = dataDistribution,
- partitions = partitions
- )
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/core/src/main/scala/org/apache/spark/status/api/v1/OneRDDResource.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/OneRDDResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/OneRDDResource.scala
index 237aeac..ca9758c 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/OneRDDResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/OneRDDResource.scala
@@ -16,6 +16,7 @@
*/
package org.apache.spark.status.api.v1
+import java.util.NoSuchElementException
import javax.ws.rs.{GET, PathParam, Produces}
import javax.ws.rs.core.MediaType
@@ -26,9 +27,12 @@ private[v1] class OneRDDResource(ui: SparkUI) {
@GET
def rddData(@PathParam("rddId") rddId: Int): RDDStorageInfo = {
- AllRDDResource.getRDDStorageInfo(rddId, ui.storageListener, true).getOrElse(
- throw new NotFoundException(s"no rdd found w/ id $rddId")
- )
+ try {
+ ui.store.rdd(rddId)
+ } catch {
+ case _: NoSuchElementException =>
+ throw new NotFoundException(s"no rdd found w/ id $rddId")
+ }
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/core/src/main/scala/org/apache/spark/status/storeTypes.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/status/storeTypes.scala b/core/src/main/scala/org/apache/spark/status/storeTypes.scala
index 23e9a36..f44b793 100644
--- a/core/src/main/scala/org/apache/spark/status/storeTypes.scala
+++ b/core/src/main/scala/org/apache/spark/status/storeTypes.scala
@@ -131,3 +131,19 @@ private[spark] class ExecutorStageSummaryWrapper(
private[this] val stage: Array[Int] = Array(stageId, stageAttemptId)
}
+
+private[spark] class StreamBlockData(
+ val name: String,
+ val executorId: String,
+ val hostPort: String,
+ val storageLevel: String,
+ val useMemory: Boolean,
+ val useDisk: Boolean,
+ val deserialized: Boolean,
+ val memSize: Long,
+ val diskSize: Long) {
+
+ @JsonIgnore @KVIndex
+ def key: Array[String] = Array(name, executorId)
+
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/core/src/main/scala/org/apache/spark/storage/BlockStatusListener.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockStatusListener.scala b/core/src/main/scala/org/apache/spark/storage/BlockStatusListener.scala
deleted file mode 100644
index 0a14fca..0000000
--- a/core/src/main/scala/org/apache/spark/storage/BlockStatusListener.scala
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.storage
-
-import scala.collection.mutable
-
-import org.apache.spark.scheduler._
-
-private[spark] case class BlockUIData(
- blockId: BlockId,
- location: String,
- storageLevel: StorageLevel,
- memSize: Long,
- diskSize: Long)
-
-/**
- * The aggregated status of stream blocks in an executor
- */
-private[spark] case class ExecutorStreamBlockStatus(
- executorId: String,
- location: String,
- blocks: Seq[BlockUIData]) {
-
- def totalMemSize: Long = blocks.map(_.memSize).sum
-
- def totalDiskSize: Long = blocks.map(_.diskSize).sum
-
- def numStreamBlocks: Int = blocks.size
-
-}
-
-private[spark] class BlockStatusListener extends SparkListener {
-
- private val blockManagers =
- new mutable.HashMap[BlockManagerId, mutable.HashMap[BlockId, BlockUIData]]
-
- override def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit = {
- val blockId = blockUpdated.blockUpdatedInfo.blockId
- if (!blockId.isInstanceOf[StreamBlockId]) {
- // Now we only monitor StreamBlocks
- return
- }
- val blockManagerId = blockUpdated.blockUpdatedInfo.blockManagerId
- val storageLevel = blockUpdated.blockUpdatedInfo.storageLevel
- val memSize = blockUpdated.blockUpdatedInfo.memSize
- val diskSize = blockUpdated.blockUpdatedInfo.diskSize
-
- synchronized {
- // Drop the update info if the block manager is not registered
- blockManagers.get(blockManagerId).foreach { blocksInBlockManager =>
- if (storageLevel.isValid) {
- blocksInBlockManager.put(blockId,
- BlockUIData(
- blockId,
- blockManagerId.hostPort,
- storageLevel,
- memSize,
- diskSize)
- )
- } else {
- // If isValid is not true, it means we should drop the block.
- blocksInBlockManager -= blockId
- }
- }
- }
- }
-
- override def onBlockManagerAdded(blockManagerAdded: SparkListenerBlockManagerAdded): Unit = {
- synchronized {
- blockManagers.put(blockManagerAdded.blockManagerId, mutable.HashMap())
- }
- }
-
- override def onBlockManagerRemoved(
- blockManagerRemoved: SparkListenerBlockManagerRemoved): Unit = synchronized {
- blockManagers -= blockManagerRemoved.blockManagerId
- }
-
- def allExecutorStreamBlockStatus: Seq[ExecutorStreamBlockStatus] = synchronized {
- blockManagers.map { case (blockManagerId, blocks) =>
- ExecutorStreamBlockStatus(
- blockManagerId.executorId, blockManagerId.hostPort, blocks.values.toSeq)
- }.toSeq
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala b/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala
deleted file mode 100644
index ac60f79..0000000
--- a/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.storage
-
-import scala.collection.mutable
-
-import org.apache.spark.SparkConf
-import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.scheduler._
-
-/**
- * :: DeveloperApi ::
- * A SparkListener that maintains executor storage status.
- *
- * This class is thread-safe (unlike JobProgressListener)
- */
-@DeveloperApi
-@deprecated("This class will be removed in a future release.", "2.2.0")
-class StorageStatusListener(conf: SparkConf) extends SparkListener {
- // This maintains only blocks that are cached (i.e. storage level is not StorageLevel.NONE)
- private[storage] val executorIdToStorageStatus = mutable.Map[String, StorageStatus]()
- private[storage] val deadExecutorStorageStatus = new mutable.ListBuffer[StorageStatus]()
- private[this] val retainedDeadExecutors = conf.getInt("spark.ui.retainedDeadExecutors", 100)
-
- def storageStatusList: Seq[StorageStatus] = synchronized {
- executorIdToStorageStatus.values.toSeq
- }
-
- def deadStorageStatusList: Seq[StorageStatus] = synchronized {
- deadExecutorStorageStatus
- }
-
- /** Update storage status list to reflect updated block statuses */
- private def updateStorageStatus(execId: String, updatedBlocks: Seq[(BlockId, BlockStatus)]) {
- executorIdToStorageStatus.get(execId).foreach { storageStatus =>
- updatedBlocks.foreach { case (blockId, updatedStatus) =>
- if (updatedStatus.storageLevel == StorageLevel.NONE) {
- storageStatus.removeBlock(blockId)
- } else {
- storageStatus.updateBlock(blockId, updatedStatus)
- }
- }
- }
- }
-
- /** Update storage status list to reflect the removal of an RDD from the cache */
- private def updateStorageStatus(unpersistedRDDId: Int) {
- storageStatusList.foreach { storageStatus =>
- storageStatus.rddBlocksById(unpersistedRDDId).foreach { case (blockId, _) =>
- storageStatus.removeBlock(blockId)
- }
- }
- }
-
- override def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD): Unit = synchronized {
- updateStorageStatus(unpersistRDD.rddId)
- }
-
- override def onBlockManagerAdded(blockManagerAdded: SparkListenerBlockManagerAdded) {
- synchronized {
- val blockManagerId = blockManagerAdded.blockManagerId
- val executorId = blockManagerId.executorId
- // The onHeap and offHeap memory are always defined for new applications,
- // but they can be missing if we are replaying old event logs.
- val storageStatus = new StorageStatus(blockManagerId, blockManagerAdded.maxMem,
- blockManagerAdded.maxOnHeapMem, blockManagerAdded.maxOffHeapMem)
- executorIdToStorageStatus(executorId) = storageStatus
-
- // Try to remove the dead storage status if same executor register the block manager twice.
- deadExecutorStorageStatus.zipWithIndex.find(_._1.blockManagerId.executorId == executorId)
- .foreach(toRemoveExecutor => deadExecutorStorageStatus.remove(toRemoveExecutor._2))
- }
- }
-
- override def onBlockManagerRemoved(blockManagerRemoved: SparkListenerBlockManagerRemoved) {
- synchronized {
- val executorId = blockManagerRemoved.blockManagerId.executorId
- executorIdToStorageStatus.remove(executorId).foreach { status =>
- deadExecutorStorageStatus += status
- }
- if (deadExecutorStorageStatus.size > retainedDeadExecutors) {
- deadExecutorStorageStatus.trimStart(1)
- }
- }
- }
-
- override def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit = {
- val executorId = blockUpdated.blockUpdatedInfo.blockManagerId.executorId
- val blockId = blockUpdated.blockUpdatedInfo.blockId
- val storageLevel = blockUpdated.blockUpdatedInfo.storageLevel
- val memSize = blockUpdated.blockUpdatedInfo.memSize
- val diskSize = blockUpdated.blockUpdatedInfo.diskSize
- val blockStatus = BlockStatus(storageLevel, memSize, diskSize)
- updateStorageStatus(executorId, Seq((blockId, blockStatus)))
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
index 79d40b6..e93ade0 100644
--- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
@@ -26,13 +26,12 @@ import org.apache.spark.internal.Logging
import org.apache.spark.scheduler._
import org.apache.spark.status.AppStatusStore
import org.apache.spark.status.api.v1._
-import org.apache.spark.storage.StorageStatusListener
import org.apache.spark.ui.JettyUtils._
import org.apache.spark.ui.env.EnvironmentTab
import org.apache.spark.ui.exec.ExecutorsTab
import org.apache.spark.ui.jobs.{JobProgressListener, JobsTab, StagesTab}
import org.apache.spark.ui.scope.RDDOperationGraphListener
-import org.apache.spark.ui.storage.{StorageListener, StorageTab}
+import org.apache.spark.ui.storage.StorageTab
import org.apache.spark.util.Utils
/**
@@ -43,9 +42,7 @@ private[spark] class SparkUI private (
val sc: Option[SparkContext],
val conf: SparkConf,
securityManager: SecurityManager,
- val storageStatusListener: StorageStatusListener,
val jobProgressListener: JobProgressListener,
- val storageListener: StorageListener,
val operationGraphListener: RDDOperationGraphListener,
var appName: String,
val basePath: String,
@@ -69,7 +66,7 @@ private[spark] class SparkUI private (
attachTab(jobsTab)
val stagesTab = new StagesTab(this, store)
attachTab(stagesTab)
- attachTab(new StorageTab(this))
+ attachTab(new StorageTab(this, store))
attachTab(new EnvironmentTab(this, store))
attachTab(new ExecutorsTab(this))
attachHandler(createStaticHandler(SparkUI.STATIC_RESOURCE_DIR, "/static"))
@@ -186,18 +183,12 @@ private[spark] object SparkUI {
addListenerFn(listener)
listener
}
-
- val storageStatusListener = new StorageStatusListener(conf)
- val storageListener = new StorageListener(storageStatusListener)
val operationGraphListener = new RDDOperationGraphListener(conf)
- addListenerFn(storageStatusListener)
- addListenerFn(storageListener)
addListenerFn(operationGraphListener)
- new SparkUI(store, sc, conf, securityManager, storageStatusListener, jobProgressListener,
- storageListener, operationGraphListener, appName, basePath, lastUpdateTime, startTime,
- appSparkVersion)
+ new SparkUI(store, sc, conf, securityManager, jobProgressListener, operationGraphListener,
+ appName, basePath, lastUpdateTime, startTime, appSparkVersion)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala b/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala
index e8ff08f..02cee7f 100644
--- a/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala
@@ -22,13 +22,13 @@ import javax.servlet.http.HttpServletRequest
import scala.xml.{Node, Unparsed}
-import org.apache.spark.status.api.v1.{AllRDDResource, RDDDataDistribution, RDDPartitionInfo}
-import org.apache.spark.ui.{PagedDataSource, PagedTable, UIUtils, WebUIPage}
+import org.apache.spark.status.AppStatusStore
+import org.apache.spark.status.api.v1.{RDDDataDistribution, RDDPartitionInfo}
+import org.apache.spark.ui._
import org.apache.spark.util.Utils
/** Page showing storage details for a given RDD */
-private[ui] class RDDPage(parent: StorageTab) extends WebUIPage("rdd") {
- private val listener = parent.listener
+private[ui] class RDDPage(parent: SparkUITab, store: AppStatusStore) extends WebUIPage("rdd") {
def render(request: HttpServletRequest): Seq[Node] = {
// stripXSS is called first to remove suspicious characters used in XSS attacks
@@ -48,11 +48,13 @@ private[ui] class RDDPage(parent: StorageTab) extends WebUIPage("rdd") {
val blockPrevPageSize = Option(parameterBlockPrevPageSize).map(_.toInt).getOrElse(blockPageSize)
val rddId = parameterId.toInt
- val rddStorageInfo = AllRDDResource.getRDDStorageInfo(rddId, listener, includeDetails = true)
- .getOrElse {
+ val rddStorageInfo = try {
+ store.rdd(rddId)
+ } catch {
+ case _: NoSuchElementException =>
// Rather than crashing, render an "RDD Not Found" page
return UIUtils.headerSparkPage("RDD Not Found", Seq.empty[Node], parent)
- }
+ }
// Worker table
val workerTable = UIUtils.listingTable(workerHeader, workerRow,
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala b/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala
index b6c764d..b8aec98 100644
--- a/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala
@@ -19,23 +19,23 @@ package org.apache.spark.ui.storage
import javax.servlet.http.HttpServletRequest
+import scala.collection.SortedMap
import scala.xml.Node
-import org.apache.spark.storage._
-import org.apache.spark.ui.{UIUtils, WebUIPage}
+import org.apache.spark.status.{AppStatusStore, StreamBlockData}
+import org.apache.spark.status.api.v1
+import org.apache.spark.ui._
import org.apache.spark.util.Utils
/** Page showing list of RDD's currently stored in the cluster */
-private[ui] class StoragePage(parent: StorageTab) extends WebUIPage("") {
- private val listener = parent.listener
+private[ui] class StoragePage(parent: SparkUITab, store: AppStatusStore) extends WebUIPage("") {
def render(request: HttpServletRequest): Seq[Node] = {
- val content = rddTable(listener.rddInfoList) ++
- receiverBlockTables(listener.allExecutorStreamBlockStatus.sortBy(_.executorId))
+ val content = rddTable(store.rddList()) ++ receiverBlockTables(store.streamBlocksList())
UIUtils.headerSparkPage("Storage", content, parent)
}
- private[storage] def rddTable(rdds: Seq[RDDInfo]): Seq[Node] = {
+ private[storage] def rddTable(rdds: Seq[v1.RDDStorageInfo]): Seq[Node] = {
if (rdds.isEmpty) {
// Don't show the rdd table if there is no RDD persisted.
Nil
@@ -58,7 +58,7 @@ private[ui] class StoragePage(parent: StorageTab) extends WebUIPage("") {
"Size on Disk")
/** Render an HTML row representing an RDD */
- private def rddRow(rdd: RDDInfo): Seq[Node] = {
+ private def rddRow(rdd: v1.RDDStorageInfo): Seq[Node] = {
// scalastyle:off
<tr>
<td>{rdd.id}</td>
@@ -67,35 +67,40 @@ private[ui] class StoragePage(parent: StorageTab) extends WebUIPage("") {
{rdd.name}
</a>
</td>
- <td>{rdd.storageLevel.description}
+ <td>{rdd.storageLevel}
</td>
<td>{rdd.numCachedPartitions.toString}</td>
<td>{"%.0f%%".format(rdd.numCachedPartitions * 100.0 / rdd.numPartitions)}</td>
- <td sorttable_customkey={rdd.memSize.toString}>{Utils.bytesToString(rdd.memSize)}</td>
- <td sorttable_customkey={rdd.diskSize.toString} >{Utils.bytesToString(rdd.diskSize)}</td>
+ <td sorttable_customkey={rdd.memoryUsed.toString}>{Utils.bytesToString(rdd.memoryUsed)}</td>
+ <td sorttable_customkey={rdd.diskUsed.toString} >{Utils.bytesToString(rdd.diskUsed)}</td>
</tr>
// scalastyle:on
}
- private[storage] def receiverBlockTables(statuses: Seq[ExecutorStreamBlockStatus]): Seq[Node] = {
- if (statuses.map(_.numStreamBlocks).sum == 0) {
+ private[storage] def receiverBlockTables(blocks: Seq[StreamBlockData]): Seq[Node] = {
+ if (blocks.isEmpty) {
// Don't show the tables if there is no stream block
Nil
} else {
- val blocks = statuses.flatMap(_.blocks).groupBy(_.blockId).toSeq.sortBy(_._1.toString)
+ val sorted = blocks.groupBy(_.name).toSeq.sortBy(_._1.toString)
<div>
<h4>Receiver Blocks</h4>
- {executorMetricsTable(statuses)}
- {streamBlockTable(blocks)}
+ {executorMetricsTable(blocks)}
+ {streamBlockTable(sorted)}
</div>
}
}
- private def executorMetricsTable(statuses: Seq[ExecutorStreamBlockStatus]): Seq[Node] = {
+ private def executorMetricsTable(blocks: Seq[StreamBlockData]): Seq[Node] = {
+ val blockManagers = SortedMap(blocks.groupBy(_.executorId).toSeq: _*)
+ .map { case (id, blocks) =>
+ new ExecutorStreamSummary(blocks)
+ }
+
<div>
<h5>Aggregated Block Metrics by Executor</h5>
- {UIUtils.listingTable(executorMetricsTableHeader, executorMetricsTableRow, statuses,
+ {UIUtils.listingTable(executorMetricsTableHeader, executorMetricsTableRow, blockManagers,
id = Some("storage-by-executor-stream-blocks"))}
</div>
}
@@ -107,7 +112,7 @@ private[ui] class StoragePage(parent: StorageTab) extends WebUIPage("") {
"Total Size on Disk",
"Stream Blocks")
- private def executorMetricsTableRow(status: ExecutorStreamBlockStatus): Seq[Node] = {
+ private def executorMetricsTableRow(status: ExecutorStreamSummary): Seq[Node] = {
<tr>
<td>
{status.executorId}
@@ -127,7 +132,7 @@ private[ui] class StoragePage(parent: StorageTab) extends WebUIPage("") {
</tr>
}
- private def streamBlockTable(blocks: Seq[(BlockId, Seq[BlockUIData])]): Seq[Node] = {
+ private def streamBlockTable(blocks: Seq[(String, Seq[StreamBlockData])]): Seq[Node] = {
if (blocks.isEmpty) {
Nil
} else {
@@ -151,7 +156,7 @@ private[ui] class StoragePage(parent: StorageTab) extends WebUIPage("") {
"Size")
/** Render a stream block */
- private def streamBlockTableRow(block: (BlockId, Seq[BlockUIData])): Seq[Node] = {
+ private def streamBlockTableRow(block: (String, Seq[StreamBlockData])): Seq[Node] = {
val replications = block._2
assert(replications.nonEmpty) // This must be true because it's the result of "groupBy"
if (replications.size == 1) {
@@ -163,33 +168,36 @@ private[ui] class StoragePage(parent: StorageTab) extends WebUIPage("") {
}
private def streamBlockTableSubrow(
- blockId: BlockId, block: BlockUIData, replication: Int, firstSubrow: Boolean): Seq[Node] = {
+ blockId: String,
+ block: StreamBlockData,
+ replication: Int,
+ firstSubrow: Boolean): Seq[Node] = {
val (storageLevel, size) = streamBlockStorageLevelDescriptionAndSize(block)
<tr>
{
if (firstSubrow) {
<td rowspan={replication.toString}>
- {block.blockId.toString}
+ {block.name}
</td>
<td rowspan={replication.toString}>
{replication.toString}
</td>
}
}
- <td>{block.location}</td>
+ <td>{block.hostPort}</td>
<td>{storageLevel}</td>
<td>{Utils.bytesToString(size)}</td>
</tr>
}
private[storage] def streamBlockStorageLevelDescriptionAndSize(
- block: BlockUIData): (String, Long) = {
- if (block.storageLevel.useDisk) {
+ block: StreamBlockData): (String, Long) = {
+ if (block.useDisk) {
("Disk", block.diskSize)
- } else if (block.storageLevel.useMemory && block.storageLevel.deserialized) {
+ } else if (block.useMemory && block.deserialized) {
("Memory", block.memSize)
- } else if (block.storageLevel.useMemory && !block.storageLevel.deserialized) {
+ } else if (block.useMemory && !block.deserialized) {
("Memory Serialized", block.memSize)
} else {
throw new IllegalStateException(s"Invalid Storage Level: ${block.storageLevel}")
@@ -197,3 +205,17 @@ private[ui] class StoragePage(parent: StorageTab) extends WebUIPage("") {
}
}
+
+private class ExecutorStreamSummary(blocks: Seq[StreamBlockData]) {
+
+ def executorId: String = blocks.head.executorId
+
+ def location: String = blocks.head.hostPort
+
+ def totalMemSize: Long = blocks.map(_.memSize).sum
+
+ def totalDiskSize: Long = blocks.map(_.diskSize).sum
+
+ def numStreamBlocks: Int = blocks.size
+
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala b/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala
index 148efb1..688efa2 100644
--- a/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala
+++ b/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala
@@ -17,71 +17,13 @@
package org.apache.spark.ui.storage
-import scala.collection.mutable
-
-import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.scheduler._
-import org.apache.spark.storage._
+import org.apache.spark.status.AppStatusStore
import org.apache.spark.ui._
/** Web UI showing storage status of all RDD's in the given SparkContext. */
-private[ui] class StorageTab(parent: SparkUI) extends SparkUITab(parent, "storage") {
- val listener = parent.storageListener
-
- attachPage(new StoragePage(this))
- attachPage(new RDDPage(this))
-}
-
-/**
- * :: DeveloperApi ::
- * A SparkListener that prepares information to be displayed on the BlockManagerUI.
- *
- * This class is thread-safe (unlike JobProgressListener)
- */
-@DeveloperApi
-@deprecated("This class will be removed in a future release.", "2.2.0")
-class StorageListener(storageStatusListener: StorageStatusListener) extends BlockStatusListener {
-
- private[ui] val _rddInfoMap = mutable.Map[Int, RDDInfo]() // exposed for testing
-
- def activeStorageStatusList: Seq[StorageStatus] = storageStatusListener.storageStatusList
-
- /** Filter RDD info to include only those with cached partitions */
- def rddInfoList: Seq[RDDInfo] = synchronized {
- _rddInfoMap.values.filter(_.numCachedPartitions > 0).toSeq
- }
-
- /** Update the storage info of the RDDs whose blocks are among the given updated blocks */
- private def updateRDDInfo(updatedBlocks: Seq[(BlockId, BlockStatus)]): Unit = {
- val rddIdsToUpdate = updatedBlocks.flatMap { case (bid, _) => bid.asRDDId.map(_.rddId) }.toSet
- val rddInfosToUpdate = _rddInfoMap.values.toSeq.filter { s => rddIdsToUpdate.contains(s.id) }
- StorageUtils.updateRddInfo(rddInfosToUpdate, activeStorageStatusList)
- }
-
- override def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit = synchronized {
- val rddInfos = stageSubmitted.stageInfo.rddInfos
- rddInfos.foreach { info => _rddInfoMap.getOrElseUpdate(info.id, info).name = info.name }
- }
-
- override def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit = synchronized {
- // Remove all partitions that are no longer cached in current completed stage
- val completedRddIds = stageCompleted.stageInfo.rddInfos.map(r => r.id).toSet
- _rddInfoMap.retain { case (id, info) =>
- !completedRddIds.contains(id) || info.numCachedPartitions > 0
- }
- }
-
- override def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD): Unit = synchronized {
- _rddInfoMap.remove(unpersistRDD.rddId)
- }
+private[ui] class StorageTab(parent: SparkUI, store: AppStatusStore)
+ extends SparkUITab(parent, "storage") {
- override def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit = {
- super.onBlockUpdated(blockUpdated)
- val blockId = blockUpdated.blockUpdatedInfo.blockId
- val storageLevel = blockUpdated.blockUpdatedInfo.storageLevel
- val memSize = blockUpdated.blockUpdatedInfo.memSize
- val diskSize = blockUpdated.blockUpdatedInfo.diskSize
- val blockStatus = BlockStatus(storageLevel, memSize, diskSize)
- updateRDDInfo(Seq((blockId, blockStatus)))
- }
+ attachPage(new StoragePage(this, store))
+ attachPage(new RDDPage(this, store))
}
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
index 867d35f..ba082bc 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
@@ -578,145 +578,160 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
}
}
- val rdd1b1 = RDDBlockId(1, 1)
+ val rdd1b1 = RddBlock(1, 1, 1L, 2L)
+ val rdd1b2 = RddBlock(1, 2, 3L, 4L)
+ val rdd2b1 = RddBlock(2, 1, 5L, 6L)
val level = StorageLevel.MEMORY_AND_DISK
- // Submit a stage and make sure the RDD is recorded.
- val rddInfo = new RDDInfo(rdd1b1.rddId, "rdd1", 2, level, Nil)
- val stage = new StageInfo(1, 0, "stage1", 4, Seq(rddInfo), Nil, "details1")
+ // Submit a stage and make sure the RDDs are recorded.
+ val rdd1Info = new RDDInfo(rdd1b1.rddId, "rdd1", 2, level, Nil)
+ val rdd2Info = new RDDInfo(rdd2b1.rddId, "rdd2", 1, level, Nil)
+ val stage = new StageInfo(1, 0, "stage1", 4, Seq(rdd1Info, rdd2Info), Nil, "details1")
listener.onStageSubmitted(SparkListenerStageSubmitted(stage, new Properties()))
check[RDDStorageInfoWrapper](rdd1b1.rddId) { wrapper =>
- assert(wrapper.info.name === rddInfo.name)
- assert(wrapper.info.numPartitions === rddInfo.numPartitions)
- assert(wrapper.info.storageLevel === rddInfo.storageLevel.description)
+ assert(wrapper.info.name === rdd1Info.name)
+ assert(wrapper.info.numPartitions === rdd1Info.numPartitions)
+ assert(wrapper.info.storageLevel === rdd1Info.storageLevel.description)
}
// Add partition 1 replicated on two block managers.
- listener.onBlockUpdated(SparkListenerBlockUpdated(BlockUpdatedInfo(bm1, rdd1b1, level, 1L, 1L)))
+ listener.onBlockUpdated(SparkListenerBlockUpdated(
+ BlockUpdatedInfo(bm1, rdd1b1.blockId, level, rdd1b1.memSize, rdd1b1.diskSize)))
check[RDDStorageInfoWrapper](rdd1b1.rddId) { wrapper =>
- assert(wrapper.info.memoryUsed === 1L)
- assert(wrapper.info.diskUsed === 1L)
+ assert(wrapper.info.numCachedPartitions === 1L)
+ assert(wrapper.info.memoryUsed === rdd1b1.memSize)
+ assert(wrapper.info.diskUsed === rdd1b1.diskSize)
assert(wrapper.info.dataDistribution.isDefined)
assert(wrapper.info.dataDistribution.get.size === 1)
val dist = wrapper.info.dataDistribution.get.head
assert(dist.address === bm1.hostPort)
- assert(dist.memoryUsed === 1L)
- assert(dist.diskUsed === 1L)
+ assert(dist.memoryUsed === rdd1b1.memSize)
+ assert(dist.diskUsed === rdd1b1.diskSize)
assert(dist.memoryRemaining === maxMemory - dist.memoryUsed)
assert(wrapper.info.partitions.isDefined)
assert(wrapper.info.partitions.get.size === 1)
val part = wrapper.info.partitions.get.head
- assert(part.blockName === rdd1b1.name)
+ assert(part.blockName === rdd1b1.blockId.name)
assert(part.storageLevel === level.description)
- assert(part.memoryUsed === 1L)
- assert(part.diskUsed === 1L)
+ assert(part.memoryUsed === rdd1b1.memSize)
+ assert(part.diskUsed === rdd1b1.diskSize)
assert(part.executors === Seq(bm1.executorId))
}
check[ExecutorSummaryWrapper](bm1.executorId) { exec =>
assert(exec.info.rddBlocks === 1L)
- assert(exec.info.memoryUsed === 1L)
- assert(exec.info.diskUsed === 1L)
+ assert(exec.info.memoryUsed === rdd1b1.memSize)
+ assert(exec.info.diskUsed === rdd1b1.diskSize)
}
- listener.onBlockUpdated(SparkListenerBlockUpdated(BlockUpdatedInfo(bm2, rdd1b1, level, 1L, 1L)))
+ listener.onBlockUpdated(SparkListenerBlockUpdated(
+ BlockUpdatedInfo(bm2, rdd1b1.blockId, level, rdd1b1.memSize, rdd1b1.diskSize)))
check[RDDStorageInfoWrapper](rdd1b1.rddId) { wrapper =>
- assert(wrapper.info.memoryUsed === 2L)
- assert(wrapper.info.diskUsed === 2L)
+ assert(wrapper.info.numCachedPartitions === 1L)
+ assert(wrapper.info.memoryUsed === rdd1b1.memSize * 2)
+ assert(wrapper.info.diskUsed === rdd1b1.diskSize * 2)
assert(wrapper.info.dataDistribution.get.size === 2L)
assert(wrapper.info.partitions.get.size === 1L)
val dist = wrapper.info.dataDistribution.get.find(_.address == bm2.hostPort).get
- assert(dist.memoryUsed === 1L)
- assert(dist.diskUsed === 1L)
+ assert(dist.memoryUsed === rdd1b1.memSize)
+ assert(dist.diskUsed === rdd1b1.diskSize)
assert(dist.memoryRemaining === maxMemory - dist.memoryUsed)
val part = wrapper.info.partitions.get(0)
- assert(part.memoryUsed === 2L)
- assert(part.diskUsed === 2L)
+ assert(part.memoryUsed === rdd1b1.memSize * 2)
+ assert(part.diskUsed === rdd1b1.diskSize * 2)
assert(part.executors === Seq(bm1.executorId, bm2.executorId))
}
check[ExecutorSummaryWrapper](bm2.executorId) { exec =>
assert(exec.info.rddBlocks === 1L)
- assert(exec.info.memoryUsed === 1L)
- assert(exec.info.diskUsed === 1L)
+ assert(exec.info.memoryUsed === rdd1b1.memSize)
+ assert(exec.info.diskUsed === rdd1b1.diskSize)
}
// Add a second partition only to bm 1.
- val rdd1b2 = RDDBlockId(1, 2)
- listener.onBlockUpdated(SparkListenerBlockUpdated(BlockUpdatedInfo(bm1, rdd1b2, level,
- 3L, 3L)))
+ listener.onBlockUpdated(SparkListenerBlockUpdated(
+ BlockUpdatedInfo(bm1, rdd1b2.blockId, level, rdd1b2.memSize, rdd1b2.diskSize)))
check[RDDStorageInfoWrapper](rdd1b1.rddId) { wrapper =>
- assert(wrapper.info.memoryUsed === 5L)
- assert(wrapper.info.diskUsed === 5L)
+ assert(wrapper.info.numCachedPartitions === 2L)
+ assert(wrapper.info.memoryUsed === 2 * rdd1b1.memSize + rdd1b2.memSize)
+ assert(wrapper.info.diskUsed === 2 * rdd1b1.diskSize + rdd1b2.diskSize)
assert(wrapper.info.dataDistribution.get.size === 2L)
assert(wrapper.info.partitions.get.size === 2L)
val dist = wrapper.info.dataDistribution.get.find(_.address == bm1.hostPort).get
- assert(dist.memoryUsed === 4L)
- assert(dist.diskUsed === 4L)
+ assert(dist.memoryUsed === rdd1b1.memSize + rdd1b2.memSize)
+ assert(dist.diskUsed === rdd1b1.diskSize + rdd1b2.diskSize)
assert(dist.memoryRemaining === maxMemory - dist.memoryUsed)
- val part = wrapper.info.partitions.get.find(_.blockName === rdd1b2.name).get
+ val part = wrapper.info.partitions.get.find(_.blockName === rdd1b2.blockId.name).get
assert(part.storageLevel === level.description)
- assert(part.memoryUsed === 3L)
- assert(part.diskUsed === 3L)
+ assert(part.memoryUsed === rdd1b2.memSize)
+ assert(part.diskUsed === rdd1b2.diskSize)
assert(part.executors === Seq(bm1.executorId))
}
check[ExecutorSummaryWrapper](bm1.executorId) { exec =>
assert(exec.info.rddBlocks === 2L)
- assert(exec.info.memoryUsed === 4L)
- assert(exec.info.diskUsed === 4L)
+ assert(exec.info.memoryUsed === rdd1b1.memSize + rdd1b2.memSize)
+ assert(exec.info.diskUsed === rdd1b1.diskSize + rdd1b2.diskSize)
}
// Remove block 1 from bm 1.
- listener.onBlockUpdated(SparkListenerBlockUpdated(BlockUpdatedInfo(bm1, rdd1b1,
- StorageLevel.NONE, 1L, 1L)))
+ listener.onBlockUpdated(SparkListenerBlockUpdated(
+ BlockUpdatedInfo(bm1, rdd1b1.blockId, StorageLevel.NONE, rdd1b1.memSize, rdd1b1.diskSize)))
check[RDDStorageInfoWrapper](rdd1b1.rddId) { wrapper =>
- assert(wrapper.info.memoryUsed === 4L)
- assert(wrapper.info.diskUsed === 4L)
+ assert(wrapper.info.numCachedPartitions === 2L)
+ assert(wrapper.info.memoryUsed === rdd1b1.memSize + rdd1b2.memSize)
+ assert(wrapper.info.diskUsed === rdd1b1.diskSize + rdd1b2.diskSize)
assert(wrapper.info.dataDistribution.get.size === 2L)
assert(wrapper.info.partitions.get.size === 2L)
val dist = wrapper.info.dataDistribution.get.find(_.address == bm1.hostPort).get
- assert(dist.memoryUsed === 3L)
- assert(dist.diskUsed === 3L)
+ assert(dist.memoryUsed === rdd1b2.memSize)
+ assert(dist.diskUsed === rdd1b2.diskSize)
assert(dist.memoryRemaining === maxMemory - dist.memoryUsed)
- val part = wrapper.info.partitions.get.find(_.blockName === rdd1b1.name).get
+ val part = wrapper.info.partitions.get.find(_.blockName === rdd1b1.blockId.name).get
assert(part.storageLevel === level.description)
- assert(part.memoryUsed === 1L)
- assert(part.diskUsed === 1L)
+ assert(part.memoryUsed === rdd1b1.memSize)
+ assert(part.diskUsed === rdd1b1.diskSize)
assert(part.executors === Seq(bm2.executorId))
}
check[ExecutorSummaryWrapper](bm1.executorId) { exec =>
assert(exec.info.rddBlocks === 1L)
- assert(exec.info.memoryUsed === 3L)
- assert(exec.info.diskUsed === 3L)
+ assert(exec.info.memoryUsed === rdd1b2.memSize)
+ assert(exec.info.diskUsed === rdd1b2.diskSize)
}
- // Remove block 2 from bm 2. This should leave only block 2 info in the store.
- listener.onBlockUpdated(SparkListenerBlockUpdated(BlockUpdatedInfo(bm2, rdd1b1,
- StorageLevel.NONE, 1L, 1L)))
+ // Remove block 1 from bm 2. This should leave only block 2's info in the store.
+ listener.onBlockUpdated(SparkListenerBlockUpdated(
+ BlockUpdatedInfo(bm2, rdd1b1.blockId, StorageLevel.NONE, rdd1b1.memSize, rdd1b1.diskSize)))
check[RDDStorageInfoWrapper](rdd1b1.rddId) { wrapper =>
- assert(wrapper.info.memoryUsed === 3L)
- assert(wrapper.info.diskUsed === 3L)
+ assert(wrapper.info.numCachedPartitions === 1L)
+ assert(wrapper.info.memoryUsed === rdd1b2.memSize)
+ assert(wrapper.info.diskUsed === rdd1b2.diskSize)
assert(wrapper.info.dataDistribution.get.size === 1L)
assert(wrapper.info.partitions.get.size === 1L)
- assert(wrapper.info.partitions.get(0).blockName === rdd1b2.name)
+ assert(wrapper.info.partitions.get(0).blockName === rdd1b2.blockId.name)
+ }
+
+ check[ExecutorSummaryWrapper](bm1.executorId) { exec =>
+ assert(exec.info.rddBlocks === 1L)
+ assert(exec.info.memoryUsed === rdd1b2.memSize)
+ assert(exec.info.diskUsed === rdd1b2.diskSize)
}
check[ExecutorSummaryWrapper](bm2.executorId) { exec =>
@@ -725,12 +740,61 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
assert(exec.info.diskUsed === 0L)
}
+ // Add a block from a different RDD. Verify the executor is updated correctly and also that
+ // the distribution data for both rdds is updated to match the remaining memory.
+ listener.onBlockUpdated(SparkListenerBlockUpdated(
+ BlockUpdatedInfo(bm1, rdd2b1.blockId, level, rdd2b1.memSize, rdd2b1.diskSize)))
+
+ check[ExecutorSummaryWrapper](bm1.executorId) { exec =>
+ assert(exec.info.rddBlocks === 2L)
+ assert(exec.info.memoryUsed === rdd1b2.memSize + rdd2b1.memSize)
+ assert(exec.info.diskUsed === rdd1b2.diskSize + rdd2b1.diskSize)
+ }
+
+ check[RDDStorageInfoWrapper](rdd1b2.rddId) { wrapper =>
+ assert(wrapper.info.dataDistribution.get.size === 1L)
+ val dist = wrapper.info.dataDistribution.get(0)
+ assert(dist.memoryRemaining === maxMemory - rdd2b1.memSize - rdd1b2.memSize )
+ }
+
+ check[RDDStorageInfoWrapper](rdd2b1.rddId) { wrapper =>
+ assert(wrapper.info.dataDistribution.get.size === 1L)
+
+ val dist = wrapper.info.dataDistribution.get(0)
+ assert(dist.memoryUsed === rdd2b1.memSize)
+ assert(dist.diskUsed === rdd2b1.diskSize)
+ assert(dist.memoryRemaining === maxMemory - rdd2b1.memSize - rdd1b2.memSize )
+ }
+
// Unpersist RDD1.
listener.onUnpersistRDD(SparkListenerUnpersistRDD(rdd1b1.rddId))
- intercept[NoSuchElementException] {
+ intercept[NoSuchElementException] {
check[RDDStorageInfoWrapper](rdd1b1.rddId) { _ => () }
}
+ // Update a StreamBlock.
+ val stream1 = StreamBlockId(1, 1L)
+ listener.onBlockUpdated(SparkListenerBlockUpdated(
+ BlockUpdatedInfo(bm1, stream1, level, 1L, 1L)))
+
+ check[StreamBlockData](Array(stream1.name, bm1.executorId)) { stream =>
+ assert(stream.name === stream1.name)
+ assert(stream.executorId === bm1.executorId)
+ assert(stream.hostPort === bm1.hostPort)
+ assert(stream.storageLevel === level.description)
+ assert(stream.useMemory === level.useMemory)
+ assert(stream.useDisk === level.useDisk)
+ assert(stream.deserialized === level.deserialized)
+ assert(stream.memSize === 1L)
+ assert(stream.diskSize === 1L)
+ }
+
+ // Drop a StreamBlock.
+ listener.onBlockUpdated(SparkListenerBlockUpdated(
+ BlockUpdatedInfo(bm1, stream1, StorageLevel.NONE, 0L, 0L)))
+ intercept[NoSuchElementException] {
+ check[StreamBlockData](stream1.name) { _ => () }
+ }
}
private def key(stage: StageInfo): Array[Int] = Array(stage.stageId, stage.attemptId)
@@ -740,4 +804,14 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
fn(value)
}
+ private case class RddBlock(
+ rddId: Int,
+ partId: Int,
+ memSize: Long,
+ diskSize: Long) {
+
+ def blockId: BlockId = RDDBlockId(rddId, partId)
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/core/src/test/scala/org/apache/spark/status/LiveEntitySuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/status/LiveEntitySuite.scala b/core/src/test/scala/org/apache/spark/status/LiveEntitySuite.scala
new file mode 100644
index 0000000..bb2d263
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/status/LiveEntitySuite.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.status
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.status.api.v1.RDDPartitionInfo
+
+class LiveEntitySuite extends SparkFunSuite {
+
+ test("partition seq") {
+ val seq = new RDDPartitionSeq()
+ val items = (1 to 10).map { i =>
+ val part = newPartition(i)
+ seq.addPartition(part)
+ part
+ }.toList
+
+ checkSize(seq, 10)
+
+ val added = newPartition(11)
+ seq.addPartition(added)
+ checkSize(seq, 11)
+ assert(seq.last.blockName === added.blockName)
+
+ seq.removePartition(items(0))
+ assert(seq.head.blockName === items(1).blockName)
+ assert(!seq.exists(_.blockName == items(0).blockName))
+ checkSize(seq, 10)
+
+ seq.removePartition(added)
+ assert(seq.last.blockName === items.last.blockName)
+ assert(!seq.exists(_.blockName == added.blockName))
+ checkSize(seq, 9)
+
+ seq.removePartition(items(5))
+ checkSize(seq, 8)
+ assert(!seq.exists(_.blockName == items(5).blockName))
+ }
+
+ private def checkSize(seq: Seq[_], expected: Int): Unit = {
+ assert(seq.length === expected)
+ var count = 0
+ seq.iterator.foreach { _ => count += 1 }
+ assert(count === expected)
+ }
+
+ private def newPartition(i: Int): LiveRDDPartition = {
+ val part = new LiveRDDPartition(i.toString)
+ part.update(Seq(i.toString), i.toString, i, i)
+ part
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/core/src/test/scala/org/apache/spark/storage/BlockStatusListenerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockStatusListenerSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockStatusListenerSuite.scala
deleted file mode 100644
index 06acca3..0000000
--- a/core/src/test/scala/org/apache/spark/storage/BlockStatusListenerSuite.scala
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.storage
-
-import org.apache.spark.SparkFunSuite
-import org.apache.spark.scheduler._
-
-class BlockStatusListenerSuite extends SparkFunSuite {
-
- test("basic functions") {
- val blockManagerId = BlockManagerId("0", "localhost", 10000)
- val listener = new BlockStatusListener()
-
- // Add a block manager and a new block status
- listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(0, blockManagerId, 0))
- listener.onBlockUpdated(SparkListenerBlockUpdated(
- BlockUpdatedInfo(
- blockManagerId,
- StreamBlockId(0, 100),
- StorageLevel.MEMORY_AND_DISK,
- memSize = 100,
- diskSize = 100)))
- // The new block status should be added to the listener
- val expectedBlock = BlockUIData(
- StreamBlockId(0, 100),
- "localhost:10000",
- StorageLevel.MEMORY_AND_DISK,
- memSize = 100,
- diskSize = 100
- )
- val expectedExecutorStreamBlockStatus = Seq(
- ExecutorStreamBlockStatus("0", "localhost:10000", Seq(expectedBlock))
- )
- assert(listener.allExecutorStreamBlockStatus === expectedExecutorStreamBlockStatus)
-
- // Add the second block manager
- val blockManagerId2 = BlockManagerId("1", "localhost", 10001)
- listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(0, blockManagerId2, 0))
- // Add a new replication of the same block id from the second manager
- listener.onBlockUpdated(SparkListenerBlockUpdated(
- BlockUpdatedInfo(
- blockManagerId2,
- StreamBlockId(0, 100),
- StorageLevel.MEMORY_AND_DISK,
- memSize = 100,
- diskSize = 100)))
- val expectedBlock2 = BlockUIData(
- StreamBlockId(0, 100),
- "localhost:10001",
- StorageLevel.MEMORY_AND_DISK,
- memSize = 100,
- diskSize = 100
- )
- // Each block manager should contain one block
- val expectedExecutorStreamBlockStatus2 = Set(
- ExecutorStreamBlockStatus("0", "localhost:10000", Seq(expectedBlock)),
- ExecutorStreamBlockStatus("1", "localhost:10001", Seq(expectedBlock2))
- )
- assert(listener.allExecutorStreamBlockStatus.toSet === expectedExecutorStreamBlockStatus2)
-
- // Remove a replication of the same block
- listener.onBlockUpdated(SparkListenerBlockUpdated(
- BlockUpdatedInfo(
- blockManagerId2,
- StreamBlockId(0, 100),
- StorageLevel.NONE, // StorageLevel.NONE means removing it
- memSize = 0,
- diskSize = 0)))
- // Only the first block manager contains a block
- val expectedExecutorStreamBlockStatus3 = Set(
- ExecutorStreamBlockStatus("0", "localhost:10000", Seq(expectedBlock)),
- ExecutorStreamBlockStatus("1", "localhost:10001", Seq.empty)
- )
- assert(listener.allExecutorStreamBlockStatus.toSet === expectedExecutorStreamBlockStatus3)
-
- // Remove the second block manager at first but add a new block status
- // from this removed block manager
- listener.onBlockManagerRemoved(SparkListenerBlockManagerRemoved(0, blockManagerId2))
- listener.onBlockUpdated(SparkListenerBlockUpdated(
- BlockUpdatedInfo(
- blockManagerId2,
- StreamBlockId(0, 100),
- StorageLevel.MEMORY_AND_DISK,
- memSize = 100,
- diskSize = 100)))
- // The second block manager is removed so we should not see the new block
- val expectedExecutorStreamBlockStatus4 = Seq(
- ExecutorStreamBlockStatus("0", "localhost:10000", Seq(expectedBlock))
- )
- assert(listener.allExecutorStreamBlockStatus === expectedExecutorStreamBlockStatus4)
-
- // Remove the last block manager
- listener.onBlockManagerRemoved(SparkListenerBlockManagerRemoved(0, blockManagerId))
- // No block manager now so we should dop all block managers
- assert(listener.allExecutorStreamBlockStatus.isEmpty)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/core/src/test/scala/org/apache/spark/storage/StorageStatusListenerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/storage/StorageStatusListenerSuite.scala b/core/src/test/scala/org/apache/spark/storage/StorageStatusListenerSuite.scala
deleted file mode 100644
index 9835f11..0000000
--- a/core/src/test/scala/org/apache/spark/storage/StorageStatusListenerSuite.scala
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.storage
-
-import org.apache.spark.{SparkConf, SparkFunSuite, Success}
-import org.apache.spark.executor.TaskMetrics
-import org.apache.spark.scheduler._
-
-/**
- * Test the behavior of StorageStatusListener in response to all relevant events.
- */
-class StorageStatusListenerSuite extends SparkFunSuite {
- private val bm1 = BlockManagerId("big", "dog", 1)
- private val bm2 = BlockManagerId("fat", "duck", 2)
- private val taskInfo1 = new TaskInfo(0, 0, 0, 0, "big", "dog", TaskLocality.ANY, false)
- private val taskInfo2 = new TaskInfo(0, 0, 0, 0, "fat", "duck", TaskLocality.ANY, false)
- private val conf = new SparkConf()
-
- test("block manager added/removed") {
- conf.set("spark.ui.retainedDeadExecutors", "1")
- val listener = new StorageStatusListener(conf)
-
- // Block manager add
- assert(listener.executorIdToStorageStatus.size === 0)
- listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(1L, bm1, 1000L))
- assert(listener.executorIdToStorageStatus.size === 1)
- assert(listener.executorIdToStorageStatus.get("big").isDefined)
- assert(listener.executorIdToStorageStatus("big").blockManagerId === bm1)
- assert(listener.executorIdToStorageStatus("big").maxMem === 1000L)
- assert(listener.executorIdToStorageStatus("big").numBlocks === 0)
- listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(1L, bm2, 2000L))
- assert(listener.executorIdToStorageStatus.size === 2)
- assert(listener.executorIdToStorageStatus.get("fat").isDefined)
- assert(listener.executorIdToStorageStatus("fat").blockManagerId === bm2)
- assert(listener.executorIdToStorageStatus("fat").maxMem === 2000L)
- assert(listener.executorIdToStorageStatus("fat").numBlocks === 0)
-
- // Block manager remove
- listener.onBlockManagerRemoved(SparkListenerBlockManagerRemoved(1L, bm1))
- assert(listener.executorIdToStorageStatus.size === 1)
- assert(!listener.executorIdToStorageStatus.get("big").isDefined)
- assert(listener.executorIdToStorageStatus.get("fat").isDefined)
- assert(listener.deadExecutorStorageStatus.size === 1)
- assert(listener.deadExecutorStorageStatus(0).blockManagerId.executorId.equals("big"))
- listener.onBlockManagerRemoved(SparkListenerBlockManagerRemoved(1L, bm2))
- assert(listener.executorIdToStorageStatus.size === 0)
- assert(!listener.executorIdToStorageStatus.get("big").isDefined)
- assert(!listener.executorIdToStorageStatus.get("fat").isDefined)
- assert(listener.deadExecutorStorageStatus.size === 1)
- assert(listener.deadExecutorStorageStatus(0).blockManagerId.executorId.equals("fat"))
- }
-
- test("task end without updated blocks") {
- val listener = new StorageStatusListener(conf)
- listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(1L, bm1, 1000L))
- listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(1L, bm2, 2000L))
- val taskMetrics = new TaskMetrics
-
- // Task end with no updated blocks
- assert(listener.executorIdToStorageStatus("big").numBlocks === 0)
- assert(listener.executorIdToStorageStatus("fat").numBlocks === 0)
- listener.onTaskEnd(SparkListenerTaskEnd(1, 0, "obliteration", Success, taskInfo1, taskMetrics))
- assert(listener.executorIdToStorageStatus("big").numBlocks === 0)
- assert(listener.executorIdToStorageStatus("fat").numBlocks === 0)
- listener.onTaskEnd(SparkListenerTaskEnd(1, 0, "obliteration", Success, taskInfo2, taskMetrics))
- assert(listener.executorIdToStorageStatus("big").numBlocks === 0)
- assert(listener.executorIdToStorageStatus("fat").numBlocks === 0)
- }
-
- test("updated blocks") {
- val listener = new StorageStatusListener(conf)
- listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(1L, bm1, 1000L))
- listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(1L, bm2, 2000L))
-
- val blockUpdateInfos1 = Seq(
- BlockUpdatedInfo(bm1, RDDBlockId(1, 1), StorageLevel.DISK_ONLY, 0L, 100L),
- BlockUpdatedInfo(bm1, RDDBlockId(1, 2), StorageLevel.DISK_ONLY, 0L, 200L)
- )
- val blockUpdateInfos2 =
- Seq(BlockUpdatedInfo(bm2, RDDBlockId(4, 0), StorageLevel.DISK_ONLY, 0L, 300L))
-
- // Add some new blocks
- assert(listener.executorIdToStorageStatus("big").numBlocks === 0)
- assert(listener.executorIdToStorageStatus("fat").numBlocks === 0)
- postUpdateBlock(listener, blockUpdateInfos1)
- assert(listener.executorIdToStorageStatus("big").numBlocks === 2)
- assert(listener.executorIdToStorageStatus("fat").numBlocks === 0)
- assert(listener.executorIdToStorageStatus("big").containsBlock(RDDBlockId(1, 1)))
- assert(listener.executorIdToStorageStatus("big").containsBlock(RDDBlockId(1, 2)))
- assert(listener.executorIdToStorageStatus("fat").numBlocks === 0)
- postUpdateBlock(listener, blockUpdateInfos2)
- assert(listener.executorIdToStorageStatus("big").numBlocks === 2)
- assert(listener.executorIdToStorageStatus("fat").numBlocks === 1)
- assert(listener.executorIdToStorageStatus("big").containsBlock(RDDBlockId(1, 1)))
- assert(listener.executorIdToStorageStatus("big").containsBlock(RDDBlockId(1, 2)))
- assert(listener.executorIdToStorageStatus("fat").containsBlock(RDDBlockId(4, 0)))
-
- // Dropped the blocks
- val droppedBlockInfo1 = Seq(
- BlockUpdatedInfo(bm1, RDDBlockId(1, 1), StorageLevel.NONE, 0L, 0L),
- BlockUpdatedInfo(bm1, RDDBlockId(4, 0), StorageLevel.NONE, 0L, 0L)
- )
- val droppedBlockInfo2 = Seq(
- BlockUpdatedInfo(bm2, RDDBlockId(1, 2), StorageLevel.NONE, 0L, 0L),
- BlockUpdatedInfo(bm2, RDDBlockId(4, 0), StorageLevel.NONE, 0L, 0L)
- )
-
- postUpdateBlock(listener, droppedBlockInfo1)
- assert(listener.executorIdToStorageStatus("big").numBlocks === 1)
- assert(listener.executorIdToStorageStatus("fat").numBlocks === 1)
- assert(!listener.executorIdToStorageStatus("big").containsBlock(RDDBlockId(1, 1)))
- assert(listener.executorIdToStorageStatus("big").containsBlock(RDDBlockId(1, 2)))
- assert(listener.executorIdToStorageStatus("fat").containsBlock(RDDBlockId(4, 0)))
- postUpdateBlock(listener, droppedBlockInfo2)
- assert(listener.executorIdToStorageStatus("big").numBlocks === 1)
- assert(listener.executorIdToStorageStatus("fat").numBlocks === 0)
- assert(!listener.executorIdToStorageStatus("big").containsBlock(RDDBlockId(1, 1)))
- assert(listener.executorIdToStorageStatus("big").containsBlock(RDDBlockId(1, 2)))
- assert(listener.executorIdToStorageStatus("fat").numBlocks === 0)
- }
-
- test("unpersist RDD") {
- val listener = new StorageStatusListener(conf)
- listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(1L, bm1, 1000L))
- val blockUpdateInfos1 = Seq(
- BlockUpdatedInfo(bm1, RDDBlockId(1, 1), StorageLevel.DISK_ONLY, 0L, 100L),
- BlockUpdatedInfo(bm1, RDDBlockId(1, 2), StorageLevel.DISK_ONLY, 0L, 200L)
- )
- val blockUpdateInfos2 =
- Seq(BlockUpdatedInfo(bm1, RDDBlockId(4, 0), StorageLevel.DISK_ONLY, 0L, 300L))
- postUpdateBlock(listener, blockUpdateInfos1)
- postUpdateBlock(listener, blockUpdateInfos2)
- assert(listener.executorIdToStorageStatus("big").numBlocks === 3)
-
- // Unpersist RDD
- listener.onUnpersistRDD(SparkListenerUnpersistRDD(9090))
- assert(listener.executorIdToStorageStatus("big").numBlocks === 3)
- listener.onUnpersistRDD(SparkListenerUnpersistRDD(4))
- assert(listener.executorIdToStorageStatus("big").numBlocks === 2)
- assert(listener.executorIdToStorageStatus("big").containsBlock(RDDBlockId(1, 1)))
- assert(listener.executorIdToStorageStatus("big").containsBlock(RDDBlockId(1, 2)))
- listener.onUnpersistRDD(SparkListenerUnpersistRDD(1))
- assert(listener.executorIdToStorageStatus("big").numBlocks === 0)
- }
-
- private def postUpdateBlock(
- listener: StorageStatusListener, updateBlockInfos: Seq[BlockUpdatedInfo]): Unit = {
- updateBlockInfos.foreach { updateBlockInfo =>
- listener.onBlockUpdated(SparkListenerBlockUpdated(updateBlockInfo))
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/6ae12715/core/src/test/scala/org/apache/spark/ui/storage/StoragePageSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/ui/storage/StoragePageSuite.scala b/core/src/test/scala/org/apache/spark/ui/storage/StoragePageSuite.scala
index 4a48b3c..a71521c 100644
--- a/core/src/test/scala/org/apache/spark/ui/storage/StoragePageSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/storage/StoragePageSuite.scala
@@ -20,39 +20,46 @@ package org.apache.spark.ui.storage
import org.mockito.Mockito._
import org.apache.spark.SparkFunSuite
+import org.apache.spark.status.StreamBlockData
+import org.apache.spark.status.api.v1.RDDStorageInfo
import org.apache.spark.storage._
class StoragePageSuite extends SparkFunSuite {
val storageTab = mock(classOf[StorageTab])
when(storageTab.basePath).thenReturn("http://localhost:4040")
- val storagePage = new StoragePage(storageTab)
+ val storagePage = new StoragePage(storageTab, null)
test("rddTable") {
- val rdd1 = new RDDInfo(1,
+ val rdd1 = new RDDStorageInfo(1,
"rdd1",
10,
- StorageLevel.MEMORY_ONLY,
- Seq.empty)
- rdd1.memSize = 100
- rdd1.numCachedPartitions = 10
+ 10,
+ StorageLevel.MEMORY_ONLY.description,
+ 100L,
+ 0L,
+ None,
+ None)
- val rdd2 = new RDDInfo(2,
+ val rdd2 = new RDDStorageInfo(2,
"rdd2",
10,
- StorageLevel.DISK_ONLY,
- Seq.empty)
- rdd2.diskSize = 200
- rdd2.numCachedPartitions = 5
-
- val rdd3 = new RDDInfo(3,
+ 5,
+ StorageLevel.DISK_ONLY.description,
+ 0L,
+ 200L,
+ None,
+ None)
+
+ val rdd3 = new RDDStorageInfo(3,
"rdd3",
10,
- StorageLevel.MEMORY_AND_DISK_SER,
- Seq.empty)
- rdd3.memSize = 400
- rdd3.diskSize = 500
- rdd3.numCachedPartitions = 10
+ 10,
+ StorageLevel.MEMORY_AND_DISK_SER.description,
+ 400L,
+ 500L,
+ None,
+ None)
val xmlNodes = storagePage.rddTable(Seq(rdd1, rdd2, rdd3))
@@ -91,58 +98,85 @@ class StoragePageSuite extends SparkFunSuite {
}
test("streamBlockStorageLevelDescriptionAndSize") {
- val memoryBlock = BlockUIData(StreamBlockId(0, 0),
+ val memoryBlock = new StreamBlockData("0",
+ "0",
"localhost:1111",
- StorageLevel.MEMORY_ONLY,
- memSize = 100,
- diskSize = 0)
+ StorageLevel.MEMORY_ONLY.description,
+ true,
+ false,
+ true,
+ 100,
+ 0)
assert(("Memory", 100) === storagePage.streamBlockStorageLevelDescriptionAndSize(memoryBlock))
- val memorySerializedBlock = BlockUIData(StreamBlockId(0, 0),
+ val memorySerializedBlock = new StreamBlockData("0",
+ "0",
"localhost:1111",
- StorageLevel.MEMORY_ONLY_SER,
+ StorageLevel.MEMORY_ONLY_SER.description,
+ true,
+ false,
+ false,
memSize = 100,
diskSize = 0)
assert(("Memory Serialized", 100) ===
storagePage.streamBlockStorageLevelDescriptionAndSize(memorySerializedBlock))
- val diskBlock = BlockUIData(StreamBlockId(0, 0),
+ val diskBlock = new StreamBlockData("0",
+ "0",
"localhost:1111",
- StorageLevel.DISK_ONLY,
- memSize = 0,
- diskSize = 100)
+ StorageLevel.DISK_ONLY.description,
+ false,
+ true,
+ false,
+ 0,
+ 100)
assert(("Disk", 100) === storagePage.streamBlockStorageLevelDescriptionAndSize(diskBlock))
}
test("receiverBlockTables") {
val blocksForExecutor0 = Seq(
- BlockUIData(StreamBlockId(0, 0),
+ new StreamBlockData(StreamBlockId(0, 0).name,
+ "0",
"localhost:10000",
- StorageLevel.MEMORY_ONLY,
- memSize = 100,
- diskSize = 0),
- BlockUIData(StreamBlockId(1, 1),
+ StorageLevel.MEMORY_ONLY.description,
+ true,
+ false,
+ true,
+ 100,
+ 0),
+ new StreamBlockData(StreamBlockId(1, 1).name,
+ "0",
"localhost:10000",
- StorageLevel.DISK_ONLY,
- memSize = 0,
- diskSize = 100)
+ StorageLevel.DISK_ONLY.description,
+ false,
+ true,
+ false,
+ 0,
+ 100)
)
- val executor0 = ExecutorStreamBlockStatus("0", "localhost:10000", blocksForExecutor0)
val blocksForExecutor1 = Seq(
- BlockUIData(StreamBlockId(0, 0),
+ new StreamBlockData(StreamBlockId(0, 0).name,
+ "1",
"localhost:10001",
- StorageLevel.MEMORY_ONLY,
+ StorageLevel.MEMORY_ONLY.description,
+ true,
+ false,
+ true,
memSize = 100,
diskSize = 0),
- BlockUIData(StreamBlockId(1, 1),
+ new StreamBlockData(StreamBlockId(1, 1).name,
+ "1",
"localhost:10001",
- StorageLevel.MEMORY_ONLY_SER,
- memSize = 100,
- diskSize = 0)
+ StorageLevel.MEMORY_ONLY_SER.description,
+ true,
+ false,
+ false,
+ 100,
+ 0)
)
- val executor1 = ExecutorStreamBlockStatus("1", "localhost:10001", blocksForExecutor1)
- val xmlNodes = storagePage.receiverBlockTables(Seq(executor0, executor1))
+
+ val xmlNodes = storagePage.receiverBlockTables(blocksForExecutor0 ++ blocksForExecutor1)
val executorTable = (xmlNodes \\ "table")(0)
val executorHeaders = Seq(
@@ -190,8 +224,6 @@ class StoragePageSuite extends SparkFunSuite {
test("empty receiverBlockTables") {
assert(storagePage.receiverBlockTables(Seq.empty).isEmpty)
- val executor0 = ExecutorStreamBlockStatus("0", "localhost:10000", Seq.empty)
- val executor1 = ExecutorStreamBlockStatus("1", "localhost:10001", Seq.empty)
- assert(storagePage.receiverBlockTables(Seq(executor0, executor1)).isEmpty)
}
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org