You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by hu...@apache.org on 2022/03/02 20:03:55 UTC
[spark] branch master updated: [SPARK-38269][CORE][SQL][SS][ML][MLLIB][MESOS][YARN][K8S][EXAMPLES] Clean up redundant type cast
This is an automated email from the ASF dual-hosted git repository.
huaxingao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 226bdec [SPARK-38269][CORE][SQL][SS][ML][MLLIB][MESOS][YARN][K8S][EXAMPLES] Clean up redundant type cast
226bdec is described below
commit 226bdec8d99c51a58018f0bd085a51f1907c1e1a
Author: yangjie01 <ya...@baidu.com>
AuthorDate: Wed Mar 2 12:02:27 2022 -0800
[SPARK-38269][CORE][SQL][SS][ML][MLLIB][MESOS][YARN][K8S][EXAMPLES] Clean up redundant type cast
### What changes were proposed in this pull request?
This pr aims to clean up redundant type cast in Spark code.
### Why are the changes needed?
Code simplification
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
- Pass GA
- Manually build a client, check `org.apache.spark.examples.DriverSubmissionTest` and `org.apache.spark.examples.mllib.LDAExample` passed
Closes #35592 from LuciferYang/redundant-cast.
Authored-by: yangjie01 <ya...@baidu.com>
Signed-off-by: huaxingao <hu...@apple.com>
---
.../main/java/org/apache/spark/util/kvstore/ArrayWrappers.java | 2 +-
.../src/main/java/org/apache/spark/unsafe/types/ByteArray.java | 2 +-
.../main/java/org/apache/spark/unsafe/types/UTF8String.java | 4 ++--
.../main/java/org/apache/spark/io/ReadAheadInputStream.java | 2 +-
.../org/apache/spark/deploy/history/FsHistoryProvider.scala | 6 +++---
.../scala/org/apache/spark/deploy/master/ui/MasterPage.scala | 8 ++++----
.../main/scala/org/apache/spark/metrics/MetricsConfig.scala | 6 +++---
.../scala/org/apache/spark/rdd/ReliableRDDCheckpointData.scala | 2 +-
.../main/scala/org/apache/spark/resource/ResourceProfile.scala | 2 +-
core/src/main/scala/org/apache/spark/ui/GraphUIData.scala | 2 +-
.../main/scala/org/apache/spark/ui/storage/StoragePage.scala | 2 +-
core/src/main/scala/org/apache/spark/util/JsonProtocol.scala | 2 +-
core/src/main/scala/org/apache/spark/util/SizeEstimator.scala | 4 ++--
.../scala/org/apache/spark/examples/DriverSubmissionTest.scala | 2 +-
.../scala/org/apache/spark/examples/mllib/LDAExample.scala | 2 +-
.../org/apache/spark/sql/kafka010/KafkaSourceProvider.scala | 2 +-
.../apache/spark/ml/classification/LogisticRegression.scala | 2 +-
.../scala/org/apache/spark/ml/classification/NaiveBayes.scala | 2 +-
mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala | 2 +-
.../spark/ml/regression/GeneralizedLinearRegression.scala | 2 +-
.../scala/org/apache/spark/ml/tree/impl/RandomForest.scala | 4 ++--
mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala | 8 ++++----
.../scala/org/apache/spark/mllib/clustering/LDAModel.scala | 4 ++--
.../org/apache/spark/mllib/evaluation/MulticlassMetrics.scala | 10 +++++-----
.../spark/mllib/linalg/distributed/IndexedRowMatrix.scala | 4 ++--
.../cluster/k8s/KubernetesClusterSchedulerBackend.scala | 4 ++--
.../cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala | 2 +-
.../org/apache/spark/deploy/yarn/ResourceRequestHelper.scala | 4 ++--
.../main/scala/org/apache/spark/deploy/yarn/YarnRMClient.scala | 2 +-
.../spark/sql/catalyst/rules/QueryExecutionMetering.scala | 2 +-
.../datasources/parquet/VectorizedParquetRecordReader.java | 2 +-
.../spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala | 2 +-
.../spark/sql/execution/datasources/jdbc/JDBCOptions.scala | 2 +-
.../spark/sql/execution/ui/StreamingQueryStatusStore.scala | 2 +-
.../org/apache/spark/sql/streaming/StreamingQueryStatus.scala | 2 +-
.../spark/sql/streaming/ui/StreamingQueryStatusListener.scala | 2 +-
.../sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala | 4 ++--
.../spark/sql/hive/thriftserver/ui/ThriftServerPage.scala | 2 +-
.../main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala | 2 +-
39 files changed, 61 insertions(+), 61 deletions(-)
diff --git a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/ArrayWrappers.java b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/ArrayWrappers.java
index 825355e..6f94873 100644
--- a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/ArrayWrappers.java
+++ b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/ArrayWrappers.java
@@ -200,7 +200,7 @@ class ArrayWrappers {
public int compareTo(ComparableObjectArray other) {
int len = Math.min(array.length, other.array.length);
for (int i = 0; i < len; i++) {
- int diff = ((Comparable<Object>) array[i]).compareTo((Comparable<Object>) other.array[i]);
+ int diff = ((Comparable<Object>) array[i]).compareTo(other.array[i]);
if (diff != 0) {
return diff;
}
diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/ByteArray.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/ByteArray.java
index 4126cf5..aae47aa 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/ByteArray.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/ByteArray.java
@@ -178,7 +178,7 @@ public final class ByteArray {
for (int pos = firstPos; pos < beyondPos; pos += pad.length) {
final int jMax = Math.min(pad.length, beyondPos - pos);
for (int j = 0; j < jMax; ++j) {
- result[pos + j] = (byte) pad[j];
+ result[pos + j] = pad[j];
}
}
}
diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
index 04c69f8..98c61cf 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
@@ -369,7 +369,7 @@ public final class UTF8String implements Comparable<UTF8String>, Externalizable,
// fallback
return toUpperCaseSlow();
}
- int upper = Character.toUpperCase((int) b);
+ int upper = Character.toUpperCase(b);
if (upper > 127) {
// fallback
return toUpperCaseSlow();
@@ -399,7 +399,7 @@ public final class UTF8String implements Comparable<UTF8String>, Externalizable,
// fallback
return toLowerCaseSlow();
}
- int lower = Character.toLowerCase((int) b);
+ int lower = Character.toLowerCase(b);
if (lower > 127) {
// fallback
return toLowerCaseSlow();
diff --git a/core/src/main/java/org/apache/spark/io/ReadAheadInputStream.java b/core/src/main/java/org/apache/spark/io/ReadAheadInputStream.java
index 2e18715..011fecb 100644
--- a/core/src/main/java/org/apache/spark/io/ReadAheadInputStream.java
+++ b/core/src/main/java/org/apache/spark/io/ReadAheadInputStream.java
@@ -302,7 +302,7 @@ public class ReadAheadInputStream extends InputStream {
stateChangeLock.lock();
// Make sure we have no integer overflow.
try {
- return (int) Math.min((long) Integer.MAX_VALUE,
+ return (int) Math.min(Integer.MAX_VALUE,
(long) activeBuffer.remaining() + readAheadBuffer.remaining());
} finally {
stateChangeLock.unlock();
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index faa7033..a2494eb 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -144,8 +144,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
val dbPath = Files.createDirectories(new File(path, dir).toPath()).toFile()
Utils.chmod700(dbPath)
- val metadata = new FsHistoryProviderMetadata(CURRENT_LISTING_VERSION,
- AppStatusStore.CURRENT_VERSION, logDir.toString())
+ val metadata = FsHistoryProviderMetadata(CURRENT_LISTING_VERSION,
+ AppStatusStore.CURRENT_VERSION, logDir)
try {
open(dbPath, metadata, conf)
@@ -414,7 +414,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
} else {
Map()
}
- Map("Event log directory" -> logDir.toString) ++ safeMode
+ Map("Event log directory" -> logDir) ++ safeMode
}
override def start(): Unit = {
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
index 2b4d860..a71eb33 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
@@ -277,7 +277,7 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
s"if (window.confirm('Are you sure you want to kill application ${app.id} ?')) " +
"{ this.parentNode.submit(); return true; } else { return false; }"
<form action="app/kill/" method="POST" style="display:inline">
- <input type="hidden" name="id" value={app.id.toString}/>
+ <input type="hidden" name="id" value={app.id}/>
<input type="hidden" name="terminate" value="true"/>
<a href="#" onclick={confirm} class="kill-link">(kill)</a>
</form>
@@ -328,7 +328,7 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
s"if (window.confirm('Are you sure you want to kill driver ${driver.id} ?')) " +
"{ this.parentNode.submit(); return true; } else { return false; }"
<form action="driver/kill/" method="POST" style="display:inline">
- <input type="hidden" name="id" value={driver.id.toString}/>
+ <input type="hidden" name="id" value={driver.id}/>
<input type="hidden" name="terminate" value="true"/>
<a href="#" onclick={confirm} class="kill-link">(kill)</a>
</form>
@@ -339,10 +339,10 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
<td>{driver.worker.map(w =>
if (w.isAlive()) {
<a href={UIUtils.makeHref(parent.master.reverseProxy, w.id, w.webUiAddress)}>
- {w.id.toString}
+ {w.id}
</a>
} else {
- w.id.toString
+ w.id
}).getOrElse("None")}
</td>
<td>{driver.state}</td>
diff --git a/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala b/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala
index bddd18a..4b53aad 100644
--- a/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala
@@ -107,9 +107,9 @@ private[spark] class MetricsConfig(conf: SparkConf) extends Logging {
def subProperties(prop: Properties, regex: Regex): mutable.HashMap[String, Properties] = {
val subProperties = new mutable.HashMap[String, Properties]
prop.asScala.foreach { kv =>
- if (regex.findPrefixOf(kv._1.toString).isDefined) {
- val regex(prefix, suffix) = kv._1.toString
- subProperties.getOrElseUpdate(prefix, new Properties).setProperty(suffix, kv._2.toString)
+ if (regex.findPrefixOf(kv._1).isDefined) {
+ val regex(prefix, suffix) = kv._1
+ subProperties.getOrElseUpdate(prefix, new Properties).setProperty(suffix, kv._2)
}
}
subProperties
diff --git a/core/src/main/scala/org/apache/spark/rdd/ReliableRDDCheckpointData.scala b/core/src/main/scala/org/apache/spark/rdd/ReliableRDDCheckpointData.scala
index 0a26b7b..0d1bc14 100644
--- a/core/src/main/scala/org/apache/spark/rdd/ReliableRDDCheckpointData.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/ReliableRDDCheckpointData.scala
@@ -46,7 +46,7 @@ private[spark] class ReliableRDDCheckpointData[T: ClassTag](@transient private v
*/
def getCheckpointDir: Option[String] = RDDCheckpointData.synchronized {
if (isCheckpointed) {
- Some(cpDir.toString)
+ Some(cpDir)
} else {
None
}
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala b/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
index 3398701..087897f 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
@@ -87,7 +87,7 @@ class ResourceProfile(
}
private[spark] def getPySparkMemory: Option[Long] = {
- executorResources.get(ResourceProfile.PYSPARK_MEM).map(_.amount.toLong)
+ executorResources.get(ResourceProfile.PYSPARK_MEM).map(_.amount)
}
/*
diff --git a/core/src/main/scala/org/apache/spark/ui/GraphUIData.scala b/core/src/main/scala/org/apache/spark/ui/GraphUIData.scala
index 87ff677..ab8757f 100644
--- a/core/src/main/scala/org/apache/spark/ui/GraphUIData.scala
+++ b/core/src/main/scala/org/apache/spark/ui/GraphUIData.scala
@@ -102,7 +102,7 @@ private[spark] class GraphUIData(
val jsForLabels = operationLabels.toSeq.sorted.mkString("[\"", "\",\"", "\"]")
val (maxX, minX, maxY, minY) = if (values != null && values.length > 0) {
- val xValues = values.map(_._1.toLong)
+ val xValues = values.map(_._1)
val yValues = values.map(_._2.asScala.toSeq.map(_._2.toLong).sum)
(xValues.max, xValues.min, yValues.max, yValues.min)
} else {
diff --git a/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala b/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala
index fb43af3..c1708c3 100644
--- a/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala
@@ -110,7 +110,7 @@ private[ui] class StoragePage(parent: SparkUITab, store: AppStatusStore) extends
// Don't show the tables if there is no stream block
Nil
} else {
- val sorted = blocks.groupBy(_.name).toSeq.sortBy(_._1.toString)
+ val sorted = blocks.groupBy(_.name).toSeq.sortBy(_._1)
<div>
<h4>Receiver Blocks</h4>
diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
index 4e68ee0..f9b6ed3 100644
--- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
@@ -463,7 +463,7 @@ private[spark] object JsonProtocol {
case ExecutorLostFailure(executorId, exitCausedByApp, reason) =>
("Executor ID" -> executorId) ~
("Exit Caused By App" -> exitCausedByApp) ~
- ("Loss Reason" -> reason.map(_.toString))
+ ("Loss Reason" -> reason)
case taskKilled: TaskKilled =>
val accumUpdates = JArray(taskKilled.accumUpdates.map(accumulableInfoToJson).toList)
("Kill Reason" -> taskKilled.reason) ~
diff --git a/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala b/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
index 9ec9307..55d1380 100644
--- a/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
+++ b/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
@@ -262,7 +262,7 @@ object SizeEstimator extends Logging {
val s2 = sampleArray(array, state, rand, drawn, length)
val size = math.min(s1, s2)
state.size += math.max(s1, s2) +
- (size * ((length - ARRAY_SAMPLE_SIZE) / (ARRAY_SAMPLE_SIZE))).toLong
+ (size * ((length - ARRAY_SAMPLE_SIZE) / ARRAY_SAMPLE_SIZE))
}
}
}
@@ -282,7 +282,7 @@ object SizeEstimator extends Logging {
drawn.add(index)
val obj = ScalaRunTime.array_apply(array, index).asInstanceOf[AnyRef]
if (obj != null) {
- size += SizeEstimator.estimate(obj, state.visited).toLong
+ size += SizeEstimator.estimate(obj, state.visited)
}
}
size
diff --git a/examples/src/main/scala/org/apache/spark/examples/DriverSubmissionTest.scala b/examples/src/main/scala/org/apache/spark/examples/DriverSubmissionTest.scala
index ed56108..94fc755 100644
--- a/examples/src/main/scala/org/apache/spark/examples/DriverSubmissionTest.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/DriverSubmissionTest.scala
@@ -41,7 +41,7 @@ object DriverSubmissionTest {
env.asScala.filter { case (k, _) => k.contains("SPARK_TEST")}.foreach(println)
println("System properties containing spark.test:")
- properties.filter { case (k, _) => k.toString.contains("spark.test") }.foreach(println)
+ properties.filter { case (k, _) => k.contains("spark.test") }.foreach(println)
for (i <- 1 until numSecondsToSleep) {
println(s"Alive for $i out of $numSecondsToSleep seconds")
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/LDAExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/LDAExample.scala
index afd529c..d80f54d 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/LDAExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/LDAExample.scala
@@ -169,7 +169,7 @@ object LDAExample {
// Print the topics, showing the top-weighted terms for each topic.
val topicIndices = ldaModel.describeTopics(maxTermsPerTopic = 10)
val topics = topicIndices.map { case (terms, termWeights) =>
- terms.zip(termWeights).map { case (term, weight) => (vocabArray(term.toInt), weight) }
+ terms.zip(termWeights).map { case (term, weight) => (vocabArray(term), weight) }
}
println(s"${params.k} topics:")
topics.zipWithIndex.foreach { case (topic, i) =>
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
index 3747621..de78992 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
@@ -728,7 +728,7 @@ private[kafka010] object KafkaSourceProvider extends Logging {
parameters
.keySet
.filter(_.toLowerCase(Locale.ROOT).startsWith("kafka."))
- .map { k => k.drop(6).toString -> parameters(k) }
+ .map { k => k.drop(6) -> parameters(k) }
.toMap
}
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
index 42f88d0..f18b8af 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
@@ -1334,7 +1334,7 @@ object LogisticRegressionModel extends MLReadable[LogisticRegressionModel] {
val dataPath = new Path(path, "data").toString
val data = sparkSession.read.format("parquet").load(dataPath)
- val model = if (major.toInt < 2 || (major.toInt == 2 && minor.toInt == 0)) {
+ val model = if (major < 2 || (major == 2 && minor == 0)) {
// 2.0 and before
val Row(numClasses: Int, numFeatures: Int, intercept: Double, coefficients: Vector) =
MLUtils.convertVectorColumnsToML(data, "coefficients")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala
index fd19ec3..f947268 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala
@@ -597,7 +597,7 @@ object NaiveBayesModel extends MLReadable[NaiveBayesModel] {
val data = sparkSession.read.parquet(dataPath)
val vecConverted = MLUtils.convertVectorColumnsToML(data, "pi")
- val model = if (major.toInt < 3) {
+ val model = if (major < 3) {
val Row(pi: Vector, theta: Matrix) =
MLUtils.convertMatrixColumnsToML(vecConverted, "theta")
.select("pi", "theta")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala b/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
index 8aaa5ef..465ca6e 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
@@ -350,7 +350,7 @@ object FPGrowthModel extends MLReadable[FPGrowthModel] {
implicit val format = DefaultFormats
val metadata = DefaultParamsReader.loadMetadata(path, sc, className)
val (major, minor) = VersionUtils.majorMinorVersion(metadata.sparkVersion)
- val numTrainingRecords = if (major.toInt < 2 || (major.toInt == 2 && minor.toInt < 4)) {
+ val numTrainingRecords = if (major < 2 || (major == 2 && minor < 4)) {
// 2.3 and before don't store the count
0L
} else {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
index f7dfda8..73da2af 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
@@ -1576,7 +1576,7 @@ class GeneralizedLinearRegressionTrainingSummary private[regression] (
data.foreach { case strRow: Array[String] =>
strRow.zipWithIndex.map { case (cell: String, i: Int) =>
- StringUtils.leftPad(cell.toString, colWidths(i))
+ StringUtils.leftPad(cell, colWidths(i))
}.addString(sb, "", " ", "\n")
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala
index b6bc7aa..de6c935 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala
@@ -789,8 +789,8 @@ private[spark] object RandomForest extends Logging with Serializable {
val leftImpurity = leftImpurityCalculator.calculate() // Note: This equals 0 if count = 0
val rightImpurity = rightImpurityCalculator.calculate()
- val leftWeight = leftCount / totalCount.toDouble
- val rightWeight = rightCount / totalCount.toDouble
+ val leftWeight = leftCount / totalCount
+ val rightWeight = rightCount / totalCount
val gain = impurity - leftWeight * leftImpurity - rightWeight * rightImpurity
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
index 4858189..cc917db 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
@@ -413,8 +413,8 @@ private[ml] object DecisionTreeModelReadWrite {
val dataPath = new Path(path, "data").toString
var df = sparkSession.read.parquet(dataPath)
- val (major, minor) = VersionUtils.majorMinorVersion(metadata.sparkVersion)
- if (major.toInt < 3) {
+ val (major, _) = VersionUtils.majorMinorVersion(metadata.sparkVersion)
+ if (major < 3) {
df = df.withColumn("rawCount", lit(-1L))
}
@@ -530,8 +530,8 @@ private[ml] object EnsembleModelReadWrite {
val dataPath = new Path(path, "data").toString
var df = sparkSession.read.parquet(dataPath)
- val (major, minor) = VersionUtils.majorMinorVersion(metadata.sparkVersion)
- if (major.toInt < 3) {
+ val (major, _) = VersionUtils.majorMinorVersion(metadata.sparkVersion)
+ if (major < 3) {
val newNodeDataCol = df.schema("nodeData").dataType match {
case StructType(fields) =>
val cols = fields.map(f => col(s"nodeData.${f.name}")) :+ lit(-1L).as("rawCount")
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAModel.scala
index ec95252..aa8b6a0 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAModel.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAModel.scala
@@ -766,7 +766,7 @@ class DistributedLDAModel private[clustering] (
@Since("1.3.0")
def topicDistributions: RDD[(Long, Vector)] = {
graph.vertices.filter(LDA.isDocumentVertex).map { case (docID, topicCounts) =>
- (docID.toLong, Vectors.fromBreeze(normalize(topicCounts, 1.0)))
+ (docID, Vectors.fromBreeze(normalize(topicCounts, 1.0)))
}
}
@@ -792,7 +792,7 @@ class DistributedLDAModel private[clustering] (
} else {
topicCounts(topIndices).toArray
}
- (docID.toLong, topIndices.toArray, weights)
+ (docID, topIndices.toArray, weights)
}
}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MulticlassMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MulticlassMetrics.scala
index 1a91801..92abe3d 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MulticlassMetrics.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MulticlassMetrics.scala
@@ -149,7 +149,7 @@ class MulticlassMetrics @Since("1.1.0") (predictionAndLabels: RDD[_ <: Product])
def precision(label: Double): Double = {
val tp = tpByClass(label)
val fp = fpByClass.getOrElse(label, 0.0)
- if (tp + fp == 0) 0 else tp.toDouble / (tp + fp)
+ if (tp + fp == 0) 0 else tp / (tp + fp)
}
/**
@@ -199,7 +199,7 @@ class MulticlassMetrics @Since("1.1.0") (predictionAndLabels: RDD[_ <: Product])
*/
@Since("1.1.0")
lazy val weightedFalsePositiveRate: Double = labelCountByClass.map { case (category, count) =>
- falsePositiveRate(category) * count.toDouble / labelCount
+ falsePositiveRate(category) * count / labelCount
}.sum
/**
@@ -208,7 +208,7 @@ class MulticlassMetrics @Since("1.1.0") (predictionAndLabels: RDD[_ <: Product])
*/
@Since("1.1.0")
lazy val weightedRecall: Double = labelCountByClass.map { case (category, count) =>
- recall(category) * count.toDouble / labelCount
+ recall(category) * count / labelCount
}.sum
/**
@@ -216,7 +216,7 @@ class MulticlassMetrics @Since("1.1.0") (predictionAndLabels: RDD[_ <: Product])
*/
@Since("1.1.0")
lazy val weightedPrecision: Double = labelCountByClass.map { case (category, count) =>
- precision(category) * count.toDouble / labelCount
+ precision(category) * count / labelCount
}.sum
/**
@@ -225,7 +225,7 @@ class MulticlassMetrics @Since("1.1.0") (predictionAndLabels: RDD[_ <: Product])
*/
@Since("1.1.0")
def weightedFMeasure(beta: Double): Double = labelCountByClass.map { case (category, count) =>
- fMeasure(category, beta) * count.toDouble / labelCount
+ fMeasure(category, beta) * count / labelCount
}.sum
/**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/IndexedRowMatrix.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/IndexedRowMatrix.scala
index da5d165..f92ac07 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/IndexedRowMatrix.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/IndexedRowMatrix.scala
@@ -133,11 +133,11 @@ class IndexedRowMatrix @Since("1.0.0") (
val rowInBlock = ir.index % rowsPerBlock
ir.vector match {
- case SparseVector(size, indices, values) =>
+ case SparseVector(_, indices, values) =>
indices.zip(values).map { case (index, value) =>
val blockColumn = index / colsPerBlock
val columnInBlock = index % colsPerBlock
- ((blockRow.toInt, blockColumn.toInt), (rowInBlock.toInt, Array((value, columnInBlock))))
+ ((blockRow.toInt, blockColumn), (rowInBlock.toInt, Array((value, columnInBlock))))
}
case DenseVector(values) =>
values.grouped(colsPerBlock)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala
index 1bda9cc..43c6597 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala
@@ -96,7 +96,7 @@ private[spark] class KubernetesClusterSchedulerBackend(
* @return The application ID
*/
override def applicationId(): String = {
- conf.getOption("spark.app.id").map(_.toString).getOrElse(appId)
+ conf.getOption("spark.app.id").getOrElse(appId)
}
override def start(): Unit = {
@@ -302,7 +302,7 @@ private[spark] class KubernetesClusterSchedulerBackend(
kubernetesClient.pods()
.withName(x.podName)
.edit({p: Pod => new PodBuilder(p).editMetadata()
- .addToLabels(SPARK_EXECUTOR_ID_LABEL, newId.toString)
+ .addToLabels(SPARK_EXECUTOR_ID_LABEL, newId)
.endMetadata()
.build()})
}
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
index b7b652d..e5a6a5f1 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
@@ -525,7 +525,7 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
partitionTaskResources(resources, taskCPUs, taskMemory, taskGPUs)
val taskBuilder = MesosTaskInfo.newBuilder()
- .setTaskId(TaskID.newBuilder().setValue(taskId.toString).build())
+ .setTaskId(TaskID.newBuilder().setValue(taskId).build())
.setSlaveId(offer.getSlaveId)
.setCommand(createCommand(offer, taskCPUs + extraCoresPerExecutor, taskId))
.setName(s"${sc.appName} $taskId")
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ResourceRequestHelper.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ResourceRequestHelper.scala
index 50e8225..5a5334d 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ResourceRequestHelper.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ResourceRequestHelper.scala
@@ -51,14 +51,14 @@ private object ResourceRequestHelper extends Logging {
if (splitIndex == -1) {
val errorMessage = s"Missing suffix for ${componentName}${key}, you must specify" +
s" a suffix - $AMOUNT is currently the only supported suffix."
- throw new IllegalArgumentException(errorMessage.toString())
+ throw new IllegalArgumentException(errorMessage)
}
val resourceName = key.substring(0, splitIndex)
val resourceSuffix = key.substring(splitIndex + 1)
if (!AMOUNT.equals(resourceSuffix)) {
val errorMessage = s"Unsupported suffix: $resourceSuffix in: ${componentName}${key}, " +
s"only .$AMOUNT is supported."
- throw new IllegalArgumentException(errorMessage.toString())
+ throw new IllegalArgumentException(errorMessage)
}
(resourceName, value)
}.toMap
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClient.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClient.scala
index 2f272be..8426118 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClient.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClient.scala
@@ -124,7 +124,7 @@ private[spark] class YarnRMClient extends Logging {
/** Returns the maximum number of attempts to register the AM. */
def getMaxRegAttempts(sparkConf: SparkConf, yarnConf: YarnConfiguration): Int = {
- val sparkMaxAttempts = sparkConf.get(MAX_APP_ATTEMPTS).map(_.toInt)
+ val sparkMaxAttempts = sparkConf.get(MAX_APP_ATTEMPTS)
val yarnMaxAttempts = yarnConf.getInt(
YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)
sparkMaxAttempts match {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/QueryExecutionMetering.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/QueryExecutionMetering.scala
index 8efc359..b5a5e23 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/QueryExecutionMetering.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/QueryExecutionMetering.scala
@@ -79,7 +79,7 @@ case class QueryExecutionMetering() {
val maxLengthRuleNames = if (map.isEmpty) {
0
} else {
- map.keys.map(_.toString.length).max
+ map.keys.map(_.length).max
}
val colRuleName = "Rule".padTo(maxLengthRuleNames, " ").mkString
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
index 50056bf..0e976be 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
@@ -305,7 +305,7 @@ public class VectorizedParquetRecordReader extends SpecificParquetRecordReaderBa
if (rowsReturned >= totalRowCount) return false;
checkEndOfRowGroup();
- int num = (int) Math.min((long) capacity, totalCountLoadedSoFar - rowsReturned);
+ int num = (int) Math.min(capacity, totalCountLoadedSoFar - rowsReturned);
for (int i = 0; i < columnReaders.length; ++i) {
if (columnReaders[i] == null) continue;
columnReaders[i].readBatch(num, columnVectors[i]);
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index 8b31f17..3ec5aad 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -699,7 +699,7 @@ case class AdaptiveSparkPlanExec(
p.flatMap(_.metrics.values.map(m => SQLPlanMetric(m.name.get, m.id, m.metricType)))
}
context.session.sparkContext.listenerBus.post(SparkListenerSQLAdaptiveSQLMetricUpdates(
- executionId.toLong, newMetrics))
+ executionId, newMetrics))
} else {
val planDescriptionMode = ExplainMode.fromString(conf.uiExplainMode)
context.session.sparkContext.listenerBus.post(SparkListenerSQLAdaptiveExecutionUpdate(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala
index d081e0a..ad44048 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala
@@ -216,7 +216,7 @@ class JDBCOptions(
// The principal name of user's keytab file
val principal = parameters.getOrElse(JDBC_PRINCIPAL, null)
- val tableComment = parameters.getOrElse(JDBC_TABLE_COMMENT, "").toString
+ val tableComment = parameters.getOrElse(JDBC_TABLE_COMMENT, "")
val refreshKrb5Config = parameters.getOrElse(JDBC_REFRESH_KRB5_CONFIG, "false").toBoolean
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/StreamingQueryStatusStore.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/StreamingQueryStatusStore.scala
index 9eb14a6..6a3b4ee 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/StreamingQueryStatusStore.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/StreamingQueryStatusStore.scala
@@ -43,7 +43,7 @@ class StreamingQueryStatusStore(store: KVStore) {
}
private def makeUIData(summary: StreamingQueryData): StreamingQueryUIData = {
- val runId = summary.runId.toString
+ val runId = summary.runId
val view = store.view(classOf[StreamingQueryProgressWrapper])
.index("runId").first(runId).last(runId)
val recentProgress = KVUtils.viewToSeq(view, Int.MaxValue)(_ => true)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryStatus.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryStatus.scala
index 6ca9aac..fe18791 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryStatus.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryStatus.scala
@@ -61,7 +61,7 @@ class StreamingQueryStatus protected[sql](
}
private[sql] def jsonValue: JValue = {
- ("message" -> JString(message.toString)) ~
+ ("message" -> JString(message)) ~
("isDataAvailable" -> JBool(isDataAvailable)) ~
("isTriggerActive" -> JBool(isTriggerActive))
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala
index b59ec04..55ceab2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala
@@ -64,7 +64,7 @@ private[sql] class StreamingQueryStatusListener(
.take(numInactiveQueries - inactiveQueryStatusRetention)
val runIds = toDelete.map { e =>
store.delete(e.getClass, e.runId)
- e.runId.toString
+ e.runId
}
// Delete wrappers in one pass, as deleting them for each summary is slow
store.removeAllByIndexValues(classOf[StreamingQueryProgressWrapper], "runId", runIds)
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala
index 4cf672e..7b2da69 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala
@@ -93,7 +93,7 @@ private[thriftserver] class HiveThriftServer2Listener(
val execList = executionList.values().asScala.filter(_.groupId == groupId).toSeq
if (execList.nonEmpty) {
execList.foreach { exec =>
- exec.jobId += jobId.toString
+ exec.jobId += jobId
updateLiveStore(exec)
}
} else {
@@ -105,7 +105,7 @@ private[thriftserver] class HiveThriftServer2Listener(
storeExecInfo.foreach { exec =>
val liveExec = getOrCreateExecution(exec.execId, exec.statement, exec.sessionId,
exec.startTimestamp, exec.userName)
- liveExec.jobId += jobId.toString
+ liveExec.jobId += jobId
updateStoreWithTriggerEnabled(liveExec)
executionList.remove(liveExec.execId)
}
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala
index 54a40e3..d0378ef 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala
@@ -232,7 +232,7 @@ private[ui] class SqlStatsPagedTable(
def jobLinks(jobData: Seq[String]): Seq[Node] = {
jobData.map { jobId =>
- <a href={jobURL(request, jobId)}>[{jobId.toString}]</a>
+ <a href={jobURL(request, jobId)}>[{jobId}]</a>
}
}
diff --git a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala
index a6fee86..ef28095 100644
--- a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala
+++ b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala
@@ -69,7 +69,7 @@ object GenerateMIMAIgnore {
/* Inner classes defined within a private[spark] class or object are effectively
invisible, so we account for them as package private. */
lazy val indirectlyPrivateSpark = {
- val maybeOuter = className.toString.takeWhile(_ != '$')
+ val maybeOuter = className.takeWhile(_ != '$')
if (maybeOuter != className) {
isPackagePrivate(mirror.classSymbol(Class.forName(maybeOuter, false, classLoader))) ||
isPackagePrivateModule(mirror.staticModule(maybeOuter))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org