You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2019/01/02 04:37:45 UTC
[spark] branch master updated: [SPARK-26508][CORE][SQL] Address
warning messages in Java reported at lgtm.com
This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 79b0548 [SPARK-26508][CORE][SQL] Address warning messages in Java reported at lgtm.com
79b0548 is described below
commit 79b05481a2cff7a0aa34146c72068cc6e41e2241
Author: Kazuaki Ishizaki <is...@jp.ibm.com>
AuthorDate: Tue Jan 1 22:37:28 2019 -0600
[SPARK-26508][CORE][SQL] Address warning messages in Java reported at lgtm.com
## What changes were proposed in this pull request?
This PR addresses warning messages in Java files reported at [lgtm.com](https://lgtm.com).
[lgtm.com](https://lgtm.com) provides automated code review of Java/Python/JavaScript files for OSS projects. [Here](https://lgtm.com/projects/g/apache/spark/alerts/?mode=list&severity=warning) are warning messages regarding Apache Spark project.
This PR addresses the following warnings:
- Result of multiplication cast to wider type
- Implicit narrowing conversion in compound assignment
- Boxed variable is never null
- Useless null check
NOTE: `Potential input resource leak` looks false positive for now.
## How was this patch tested?
Existing UTs
Closes #23420 from kiszk/SPARK-26508.
Authored-by: Kazuaki Ishizaki <is...@jp.ibm.com>
Signed-off-by: Sean Owen <se...@databricks.com>
---
.../main/java/org/apache/spark/network/util/ByteUnit.java | 12 ++++++------
.../java/org/apache/spark/network/util/TransportConf.java | 6 +++---
.../java/org/apache/spark/unsafe/map/BytesToBytesMap.java | 4 ++--
examples/src/main/java/org/apache/spark/examples/JavaTC.java | 2 +-
.../java/org/apache/spark/examples/ml/JavaALSExample.java | 2 +-
.../apache/spark/examples/mllib/JavaCorrelationsExample.java | 2 +-
.../mllib/JavaRandomForestClassificationExample.java | 8 ++++----
.../main/java/org/apache/spark/launcher/LauncherServer.java | 7 +++----
.../sql/catalyst/expressions/codegen/UnsafeArrayWriter.java | 2 +-
.../sql/catalyst/expressions/codegen/UnsafeRowWriter.java | 2 +-
.../spark/sql/catalyst/expressions/xml/UDFXPathUtil.java | 2 +-
11 files changed, 24 insertions(+), 25 deletions(-)
diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/ByteUnit.java b/common/network-common/src/main/java/org/apache/spark/network/util/ByteUnit.java
index 984575a..6f7925c 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/util/ByteUnit.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/util/ByteUnit.java
@@ -18,11 +18,11 @@ package org.apache.spark.network.util;
public enum ByteUnit {
BYTE(1),
- KiB(1024L),
- MiB((long) Math.pow(1024L, 2L)),
- GiB((long) Math.pow(1024L, 3L)),
- TiB((long) Math.pow(1024L, 4L)),
- PiB((long) Math.pow(1024L, 5L));
+ KiB(1L << 10),
+ MiB(1L << 20),
+ GiB(1L << 30),
+ TiB(1L << 40),
+ PiB(1L << 50);
ByteUnit(long multiplier) {
this.multiplier = multiplier;
@@ -50,7 +50,7 @@ public enum ByteUnit {
}
}
- public double toBytes(long d) {
+ public long toBytes(long d) {
if (d < 0) {
throw new IllegalArgumentException("Negative size value. Size must be positive: " + d);
}
diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
index 43a6bc7..201628b 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
@@ -309,8 +309,8 @@ public class TransportConf {
}
int chunkFetchHandlerThreadsPercent =
conf.getInt("spark.shuffle.server.chunkFetchHandlerThreadsPercent", 100);
- return (int)Math.ceil(
- (this.serverThreads() > 0 ? this.serverThreads() : 2 * NettyRuntime.availableProcessors()) *
- chunkFetchHandlerThreadsPercent/(double)100);
+ int threads =
+ this.serverThreads() > 0 ? this.serverThreads() : 2 * NettyRuntime.availableProcessors();
+ return (int) Math.ceil(threads * (chunkFetchHandlerThreadsPercent / 100.0));
}
}
diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
index 7df8aaf..2ff98a6 100644
--- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
+++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
@@ -712,7 +712,7 @@ public final class BytesToBytesMap extends MemoryConsumer {
final long recordOffset = offset;
UnsafeAlignedOffset.putSize(base, offset, klen + vlen + uaoSize);
UnsafeAlignedOffset.putSize(base, offset + uaoSize, klen);
- offset += (2 * uaoSize);
+ offset += (2L * uaoSize);
Platform.copyMemory(kbase, koff, base, offset, klen);
offset += klen;
Platform.copyMemory(vbase, voff, base, offset, vlen);
@@ -780,7 +780,7 @@ public final class BytesToBytesMap extends MemoryConsumer {
assert (capacity >= 0);
capacity = Math.max((int) Math.min(MAX_CAPACITY, ByteArrayMethods.nextPowerOf2(capacity)), 64);
assert (capacity <= MAX_CAPACITY);
- longArray = allocateArray(capacity * 2);
+ longArray = allocateArray(capacity * 2L);
longArray.zeroOut();
this.growthThreshold = (int) (capacity * loadFactor);
diff --git a/examples/src/main/java/org/apache/spark/examples/JavaTC.java b/examples/src/main/java/org/apache/spark/examples/JavaTC.java
index c9ca9c9..7e8df69 100644
--- a/examples/src/main/java/org/apache/spark/examples/JavaTC.java
+++ b/examples/src/main/java/org/apache/spark/examples/JavaTC.java
@@ -71,7 +71,7 @@ public final class JavaTC {
JavaSparkContext jsc = new JavaSparkContext(spark.sparkContext());
- Integer slices = (args.length > 0) ? Integer.parseInt(args[0]): 2;
+ int slices = (args.length > 0) ? Integer.parseInt(args[0]): 2;
JavaPairRDD<Integer, Integer> tc = jsc.parallelizePairs(generateGraph(), slices).cache();
// Linear transitive closure: each round grows paths by one edge,
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java
index 27052be..b8d2c9f 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java
@@ -111,7 +111,7 @@ public class JavaALSExample {
.setMetricName("rmse")
.setLabelCol("rating")
.setPredictionCol("prediction");
- Double rmse = evaluator.evaluate(predictions);
+ double rmse = evaluator.evaluate(predictions);
System.out.println("Root-mean-square error = " + rmse);
// Generate top 10 movie recommendations for each user
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaCorrelationsExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaCorrelationsExample.java
index c0fa0b3..9bd858b 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaCorrelationsExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaCorrelationsExample.java
@@ -46,7 +46,7 @@ public class JavaCorrelationsExample {
// compute the correlation using Pearson's method. Enter "spearman" for Spearman's method.
// If a method is not specified, Pearson's method will be used by default.
- Double correlation = Statistics.corr(seriesX.srdd(), seriesY.srdd(), "pearson");
+ double correlation = Statistics.corr(seriesX.srdd(), seriesY.srdd(), "pearson");
System.out.println("Correlation is: " + correlation);
// note that each Vector is a row and not a column
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaRandomForestClassificationExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaRandomForestClassificationExample.java
index 6998ce2..0707db8 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaRandomForestClassificationExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaRandomForestClassificationExample.java
@@ -48,14 +48,14 @@ public class JavaRandomForestClassificationExample {
// Train a RandomForest model.
// Empty categoricalFeaturesInfo indicates all features are continuous.
- Integer numClasses = 2;
+ int numClasses = 2;
Map<Integer, Integer> categoricalFeaturesInfo = new HashMap<>();
Integer numTrees = 3; // Use more in practice.
String featureSubsetStrategy = "auto"; // Let the algorithm choose.
String impurity = "gini";
- Integer maxDepth = 5;
- Integer maxBins = 32;
- Integer seed = 12345;
+ int maxDepth = 5;
+ int maxBins = 32;
+ int seed = 12345;
RandomForestModel model = RandomForest.trainClassifier(trainingData, numClasses,
categoricalFeaturesInfo, numTrees, featureSubsetStrategy, impurity, maxDepth, maxBins,
diff --git a/launcher/src/main/java/org/apache/spark/launcher/LauncherServer.java b/launcher/src/main/java/org/apache/spark/launcher/LauncherServer.java
index 607879f..3ff7787 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/LauncherServer.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/LauncherServer.java
@@ -318,9 +318,9 @@ class LauncherServer implements Closeable {
throw new IllegalArgumentException("Received Hello for unknown client.");
}
} else {
+ String msgClassName = msg != null ? msg.getClass().getName() : "no message";
if (handle == null) {
- throw new IllegalArgumentException("Expected hello, got: " +
- msg != null ? msg.getClass().getName() : null);
+ throw new IllegalArgumentException("Expected hello, got: " + msgClassName);
}
if (msg instanceof SetAppId) {
SetAppId set = (SetAppId) msg;
@@ -328,8 +328,7 @@ class LauncherServer implements Closeable {
} else if (msg instanceof SetState) {
handle.setState(((SetState)msg).state);
} else {
- throw new IllegalArgumentException("Invalid message: " +
- msg != null ? msg.getClass().getName() : null);
+ throw new IllegalArgumentException("Invalid message: " + msgClassName);
}
}
} catch (Exception e) {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java
index a78dd97..997eecd 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java
@@ -74,7 +74,7 @@ public final class UnsafeArrayWriter extends UnsafeWriter {
}
private long getElementOffset(int ordinal) {
- return startingOffset + headerInBytes + ordinal * elementSize;
+ return startingOffset + headerInBytes + ordinal * (long) elementSize;
}
private void setNullBit(int ordinal) {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeRowWriter.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeRowWriter.java
index 3960d6d..d2298aa 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeRowWriter.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeRowWriter.java
@@ -132,7 +132,7 @@ public final class UnsafeRowWriter extends UnsafeWriter {
}
public long getFieldOffset(int ordinal) {
- return startingOffset + nullBitsSize + 8 * ordinal;
+ return startingOffset + nullBitsSize + 8L * ordinal;
}
public void write(int ordinal, boolean value) {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/xml/UDFXPathUtil.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/xml/UDFXPathUtil.java
index 023ec13..e9f1822 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/xml/UDFXPathUtil.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/xml/UDFXPathUtil.java
@@ -180,7 +180,7 @@ public class UDFXPathUtil {
return 0;
}
// Bound skip by beginning and end of the source
- long n = Math.min(length - next, ns);
+ int n = (int) Math.min(length - next, ns);
n = Math.max(-next, n);
next += n;
return n;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org