You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2017/03/30 15:15:44 UTC
spark git commit: [SPARK-20127][CORE] few warning have been fixed
which Intellij IDEA reported Intellij IDEA
Repository: spark
Updated Branches:
refs/heads/master 669a11b61 -> 5e00a5de1
[SPARK-20127][CORE] few warning have been fixed which Intellij IDEA reported Intellij IDEA
## What changes were proposed in this pull request?
Few changes related to Intellij IDEA inspection.
## How was this patch tested?
Changes were tested by existing unit tests
Author: Denis Bolshakov <de...@onefactor.com>
Closes #17458 from dbolshak/SPARK-20127.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5e00a5de
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5e00a5de
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5e00a5de
Branch: refs/heads/master
Commit: 5e00a5de14ae2d80471c6f38c30cc6fe63e05163
Parents: 669a11b
Author: Denis Bolshakov <de...@onefactor.com>
Authored: Thu Mar 30 16:15:40 2017 +0100
Committer: Sean Owen <so...@cloudera.com>
Committed: Thu Mar 30 16:15:40 2017 +0100
----------------------------------------------------------------------
.../java/org/apache/spark/memory/TaskMemoryManager.java | 6 +-----
.../java/org/apache/spark/status/api/v1/TaskSorting.java | 5 ++---
.../main/scala/org/apache/spark/io/CompressionCodec.scala | 3 +--
core/src/main/scala/org/apache/spark/ui/WebUI.scala | 2 +-
.../org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala | 2 +-
.../scala/org/apache/spark/ui/exec/ExecutorsPage.scala | 3 +--
.../scala/org/apache/spark/ui/exec/ExecutorsTab.scala | 4 ++--
.../scala/org/apache/spark/ui/jobs/AllStagesPage.scala | 4 ++--
.../scala/org/apache/spark/ui/jobs/ExecutorTable.scala | 4 ++--
.../org/apache/spark/ui/jobs/JobProgressListener.scala | 4 ++--
.../main/scala/org/apache/spark/ui/jobs/StagePage.scala | 10 +++++-----
.../main/scala/org/apache/spark/ui/jobs/StageTable.scala | 2 +-
.../scala/org/apache/spark/ui/storage/StoragePage.scala | 2 +-
13 files changed, 22 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/5e00a5de/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java b/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java
index 39fb3b2..aa0b373 100644
--- a/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java
+++ b/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java
@@ -155,11 +155,7 @@ public class TaskMemoryManager {
for (MemoryConsumer c: consumers) {
if (c != consumer && c.getUsed() > 0 && c.getMode() == mode) {
long key = c.getUsed();
- List<MemoryConsumer> list = sortedConsumers.get(key);
- if (list == null) {
- list = new ArrayList<>(1);
- sortedConsumers.put(key, list);
- }
+ List<MemoryConsumer> list = sortedConsumers.computeIfAbsent(key, k -> new ArrayList<>(1));
list.add(c);
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/5e00a5de/core/src/main/java/org/apache/spark/status/api/v1/TaskSorting.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/spark/status/api/v1/TaskSorting.java b/core/src/main/java/org/apache/spark/status/api/v1/TaskSorting.java
index 9307eb9..b38639e 100644
--- a/core/src/main/java/org/apache/spark/status/api/v1/TaskSorting.java
+++ b/core/src/main/java/org/apache/spark/status/api/v1/TaskSorting.java
@@ -19,6 +19,7 @@ package org.apache.spark.status.api.v1;
import org.apache.spark.util.EnumUtil;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
@@ -30,9 +31,7 @@ public enum TaskSorting {
private final Set<String> alternateNames;
TaskSorting(String... names) {
alternateNames = new HashSet<>();
- for (String n: names) {
- alternateNames.add(n);
- }
+ Collections.addAll(alternateNames, names);
}
public static TaskSorting fromString(String str) {
http://git-wip-us.apache.org/repos/asf/spark/blob/5e00a5de/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
index 2e991ce..c216fe4 100644
--- a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
+++ b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
@@ -71,8 +71,7 @@ private[spark] object CompressionCodec {
val ctor = Utils.classForName(codecClass).getConstructor(classOf[SparkConf])
Some(ctor.newInstance(conf).asInstanceOf[CompressionCodec])
} catch {
- case e: ClassNotFoundException => None
- case e: IllegalArgumentException => None
+ case _: ClassNotFoundException | _: IllegalArgumentException => None
}
codec.getOrElse(throw new IllegalArgumentException(s"Codec [$codecName] is not available. " +
s"Consider setting $configKey=$FALLBACK_COMPRESSION_CODEC"))
http://git-wip-us.apache.org/repos/asf/spark/blob/5e00a5de/core/src/main/scala/org/apache/spark/ui/WebUI.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/WebUI.scala b/core/src/main/scala/org/apache/spark/ui/WebUI.scala
index a9480cc..8b75f5d 100644
--- a/core/src/main/scala/org/apache/spark/ui/WebUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/WebUI.scala
@@ -124,7 +124,7 @@ private[spark] abstract class WebUI(
/** Bind to the HTTP server behind this web interface. */
def bind(): Unit = {
- assert(!serverInfo.isDefined, s"Attempted to bind $className more than once!")
+ assert(serverInfo.isEmpty, s"Attempted to bind $className more than once!")
try {
val host = Option(conf.getenv("SPARK_LOCAL_IP")).getOrElse("0.0.0.0")
serverInfo = Some(startJettyServer(host, port, sslOptions, handlers, conf, name))
http://git-wip-us.apache.org/repos/asf/spark/blob/5e00a5de/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala
index c6a0744..dbcc640 100644
--- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala
@@ -49,7 +49,7 @@ private[ui] class ExecutorThreadDumpPage(parent: ExecutorsTab) extends WebUIPage
}.map { thread =>
val threadId = thread.threadId
val blockedBy = thread.blockedByThreadId match {
- case Some(blockedByThreadId) =>
+ case Some(_) =>
<div>
Blocked by <a href={s"#${thread.blockedByThreadId}_td_id"}>
Thread {thread.blockedByThreadId} {thread.blockedByLock}</a>
http://git-wip-us.apache.org/repos/asf/spark/blob/5e00a5de/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
index 2d1691e..d849ce7 100644
--- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
@@ -48,7 +48,6 @@ private[ui] class ExecutorsPage(
parent: ExecutorsTab,
threadDumpEnabled: Boolean)
extends WebUIPage("") {
- private val listener = parent.listener
def render(request: HttpServletRequest): Seq[Node] = {
val content =
@@ -59,7 +58,7 @@ private[ui] class ExecutorsPage(
<script src={UIUtils.prependBaseUri("/static/executorspage.js")}></script> ++
<script>setThreadDumpEnabled({threadDumpEnabled})</script>
}
- </div>;
+ </div>
UIUtils.headerSparkPage("Executors", content, parent, useDataTables = true)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/5e00a5de/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala
index 8ae712f..0385129 100644
--- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala
+++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala
@@ -64,7 +64,7 @@ private[ui] case class ExecutorTaskSummary(
@DeveloperApi
class ExecutorsListener(storageStatusListener: StorageStatusListener, conf: SparkConf)
extends SparkListener {
- var executorToTaskSummary = LinkedHashMap[String, ExecutorTaskSummary]()
+ val executorToTaskSummary = LinkedHashMap[String, ExecutorTaskSummary]()
var executorEvents = new ListBuffer[SparkListenerEvent]()
private val maxTimelineExecutors = conf.getInt("spark.ui.timeline.executors.maximum", 1000)
@@ -137,7 +137,7 @@ class ExecutorsListener(storageStatusListener: StorageStatusListener, conf: Spar
// could have failed half-way through. The correct fix would be to keep track of the
// metrics added by each attempt, but this is much more complicated.
return
- case e: ExceptionFailure =>
+ case _: ExceptionFailure =>
taskSummary.tasksFailed += 1
case _ =>
taskSummary.tasksComplete += 1
http://git-wip-us.apache.org/repos/asf/spark/blob/5e00a5de/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala
index fe6ca10..2b0816e 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala
@@ -34,9 +34,9 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
listener.synchronized {
val activeStages = listener.activeStages.values.toSeq
val pendingStages = listener.pendingStages.values.toSeq
- val completedStages = listener.completedStages.reverse.toSeq
+ val completedStages = listener.completedStages.reverse
val numCompletedStages = listener.numCompletedStages
- val failedStages = listener.failedStages.reverse.toSeq
+ val failedStages = listener.failedStages.reverse
val numFailedStages = listener.numFailedStages
val subPath = "stages"
http://git-wip-us.apache.org/repos/asf/spark/blob/5e00a5de/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
index 52f4129..382a6f9 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
@@ -133,9 +133,9 @@ private[ui] class ExecutorTable(stageId: Int, stageAttemptId: Int, parent: Stage
</td>
<td>{executorIdToAddress.getOrElse(k, "CANNOT FIND ADDRESS")}</td>
<td sorttable_customkey={v.taskTime.toString}>{UIUtils.formatDuration(v.taskTime)}</td>
- <td>{v.failedTasks + v.succeededTasks + v.reasonToNumKilled.map(_._2).sum}</td>
+ <td>{v.failedTasks + v.succeededTasks + v.reasonToNumKilled.values.sum}</td>
<td>{v.failedTasks}</td>
- <td>{v.reasonToNumKilled.map(_._2).sum}</td>
+ <td>{v.reasonToNumKilled.values.sum}</td>
<td>{v.succeededTasks}</td>
{if (stageData.hasInput) {
<td sorttable_customkey={v.inputBytes.toString}>
http://git-wip-us.apache.org/repos/asf/spark/blob/5e00a5de/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
index 1cf03e1..f78db5a 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
@@ -226,7 +226,7 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
trimJobsIfNecessary(completedJobs)
jobData.status = JobExecutionStatus.SUCCEEDED
numCompletedJobs += 1
- case JobFailed(exception) =>
+ case JobFailed(_) =>
failedJobs += jobData
trimJobsIfNecessary(failedJobs)
jobData.status = JobExecutionStatus.FAILED
@@ -284,7 +284,7 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
) {
jobData.numActiveStages -= 1
if (stage.failureReason.isEmpty) {
- if (!stage.submissionTime.isEmpty) {
+ if (stage.submissionTime.isDefined) {
jobData.completedStageIndices.add(stage.stageId)
}
} else {
http://git-wip-us.apache.org/repos/asf/spark/blob/5e00a5de/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index ff17775..19325a2 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -142,7 +142,7 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
val allAccumulables = progressListener.stageIdToData((stageId, stageAttemptId)).accumulables
val externalAccumulables = allAccumulables.values.filter { acc => !acc.internal }
- val hasAccumulators = externalAccumulables.size > 0
+ val hasAccumulators = externalAccumulables.nonEmpty
val summary =
<div>
@@ -339,7 +339,7 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
val validTasks = tasks.filter(t => t.taskInfo.status == "SUCCESS" && t.metrics.isDefined)
val summaryTable: Option[Seq[Node]] =
- if (validTasks.size == 0) {
+ if (validTasks.isEmpty) {
None
}
else {
@@ -786,8 +786,8 @@ private[ui] object StagePage {
info: TaskInfo, metrics: TaskMetricsUIData, currentTime: Long): Long = {
if (info.finished) {
val totalExecutionTime = info.finishTime - info.launchTime
- val executorOverhead = (metrics.executorDeserializeTime +
- metrics.resultSerializationTime)
+ val executorOverhead = metrics.executorDeserializeTime +
+ metrics.resultSerializationTime
math.max(
0,
totalExecutionTime - metrics.executorRunTime - executorOverhead -
@@ -872,7 +872,7 @@ private[ui] class TaskDataSource(
// so that we can avoid creating duplicate contents during sorting the data
private val data = tasks.map(taskRow).sorted(ordering(sortColumn, desc))
- private var _slicedTaskIds: Set[Long] = null
+ private var _slicedTaskIds: Set[Long] = _
override def dataSize: Int = data.size
http://git-wip-us.apache.org/repos/asf/spark/blob/5e00a5de/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
index f4caad0..256b726 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
@@ -412,7 +412,7 @@ private[ui] class StageDataSource(
// so that we can avoid creating duplicate contents during sorting the data
private val data = stages.map(stageRow).sorted(ordering(sortColumn, desc))
- private var _slicedStageIds: Set[Int] = null
+ private var _slicedStageIds: Set[Int] = _
override def dataSize: Int = data.size
http://git-wip-us.apache.org/repos/asf/spark/blob/5e00a5de/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala b/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala
index 76d7c6d..aa84788 100644
--- a/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala
@@ -151,7 +151,7 @@ private[ui] class StoragePage(parent: StorageTab) extends WebUIPage("") {
/** Render a stream block */
private def streamBlockTableRow(block: (BlockId, Seq[BlockUIData])): Seq[Node] = {
val replications = block._2
- assert(replications.size > 0) // This must be true because it's the result of "groupBy"
+ assert(replications.nonEmpty) // This must be true because it's the result of "groupBy"
if (replications.size == 1) {
streamBlockTableSubrow(block._1, replications.head, replications.size, true)
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org