You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by iw...@apache.org on 2020/08/11 02:24:35 UTC
[hadoop] branch branch-2.10 updated: MAPREDUCE-6881. Fix warnings
from Spotbugs in hadoop-mapreduce. Contributed by Weiwei Yang.
This is an automated email from the ASF dual-hosted git repository.
iwasakims pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new 7dedf34 MAPREDUCE-6881. Fix warnings from Spotbugs in hadoop-mapreduce. Contributed by Weiwei Yang.
7dedf34 is described below
commit 7dedf344deac7739f9ccb318fe6c407c0dbcd7d6
Author: Akira Ajisaka <aa...@apache.org>
AuthorDate: Thu Apr 27 15:45:33 2017 +0900
MAPREDUCE-6881. Fix warnings from Spotbugs in hadoop-mapreduce. Contributed by Weiwei Yang.
(cherry picked from commit 3ed3062fe3979ff55a411b730a8eee2b2c96d6b3)
---
.../hadoop/mapred/LocalContainerLauncher.java | 46 +++++++++++++---------
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 14 ++++---
.../main/java/org/apache/hadoop/mapred/JVMId.java | 2 +-
.../java/org/apache/hadoop/mapred/Operation.java | 14 +++++--
.../hadoop/mapreduce/v2/hs/HistoryFileManager.java | 5 ---
.../java/org/apache/hadoop/examples/pi/Parser.java | 8 +++-
6 files changed, 53 insertions(+), 36 deletions(-)
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
index 190d988..0b942b0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
@@ -27,6 +27,8 @@ import java.lang.management.ThreadMXBean;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
+import java.util.Collections;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
@@ -81,7 +83,7 @@ public class LocalContainerLauncher extends AbstractService implements
private static final Log LOG = LogFactory.getLog(LocalContainerLauncher.class);
private FileContext curFC = null;
- private final HashSet<File> localizedFiles;
+ private Set<File> localizedFiles = new HashSet<File>();
private final AppContext context;
private final TaskUmbilicalProtocol umbilical;
private final ClassLoader jobClassLoader;
@@ -121,9 +123,12 @@ public class LocalContainerLauncher extends AbstractService implements
// users who do that get what they deserve (and will have to disable
// uberization in order to run correctly).
File[] curLocalFiles = curDir.listFiles();
- localizedFiles = new HashSet<File>(curLocalFiles.length);
- for (int j = 0; j < curLocalFiles.length; ++j) {
- localizedFiles.add(curLocalFiles[j]);
+ if (curLocalFiles != null) {
+ HashSet<File> lf = new HashSet<File>(curLocalFiles.length);
+ for (int j = 0; j < curLocalFiles.length; ++j) {
+ lf.add(curLocalFiles[j]);
+ }
+ localizedFiles = Collections.unmodifiableSet(lf);
}
// Relocalization note/future FIXME (per chrisdo, 20110315): At moment,
@@ -521,26 +526,29 @@ public class LocalContainerLauncher extends AbstractService implements
*/
private void relocalize() {
File[] curLocalFiles = curDir.listFiles();
- for (int j = 0; j < curLocalFiles.length; ++j) {
- if (!localizedFiles.contains(curLocalFiles[j])) {
- // found one that wasn't there before: delete it
- boolean deleted = false;
- try {
- if (curFC != null) {
- // this is recursive, unlike File delete():
- deleted = curFC.delete(new Path(curLocalFiles[j].getName()),true);
+ if (curLocalFiles != null) {
+ for (int j = 0; j < curLocalFiles.length; ++j) {
+ if (!localizedFiles.contains(curLocalFiles[j])) {
+ // found one that wasn't there before: delete it
+ boolean deleted = false;
+ try {
+ if (curFC != null) {
+ // this is recursive, unlike File delete():
+ deleted =
+ curFC.delete(new Path(curLocalFiles[j].getName()), true);
+ }
+ } catch (IOException e) {
+ deleted = false;
+ }
+ if (!deleted) {
+ LOG.warn("Unable to delete unexpected local file/dir "
+ + curLocalFiles[j].getName()
+ + ": insufficient permissions?");
}
- } catch (IOException e) {
- deleted = false;
- }
- if (!deleted) {
- LOG.warn("Unable to delete unexpected local file/dir "
- + curLocalFiles[j].getName() + ": insufficient permissions?");
}
}
}
}
-
} // end EventHandler
/**
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index 566bc4e..d11eee1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -557,13 +557,15 @@ public class MRAppMaster extends CompositeService {
private boolean isJobNamePatternMatch(JobConf conf, String jobTempDir) {
// Matched staging files should be preserved after job is finished.
if (conf.getKeepTaskFilesPattern() != null && jobTempDir != null) {
- String jobFileName = Paths.get(jobTempDir).getFileName().toString();
- Pattern pattern = Pattern.compile(conf.getKeepTaskFilesPattern());
- Matcher matcher = pattern.matcher(jobFileName);
- return matcher.find();
- } else {
- return false;
+ java.nio.file.Path pathName = Paths.get(jobTempDir).getFileName();
+ if (pathName != null) {
+ String jobFileName = pathName.toString();
+ Pattern pattern = Pattern.compile(conf.getKeepTaskFilesPattern());
+ Matcher matcher = pattern.matcher(jobFileName);
+ return matcher.find();
+ }
}
+ return false;
}
private boolean isKeepFailedTaskFiles(JobConf conf) {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java
index 77a7f8a..6e0e1c4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java
@@ -98,7 +98,7 @@ class JVMId {
int jobComp = this.jobId.compareTo(that.jobId);
if(jobComp == 0) {
if(this.isMap == that.isMap) {
- return Long.valueOf(this.jvmId).compareTo(that.jvmId);
+ return Long.compare(this.jvmId, that.jvmId);
} else {
return this.isMap ? -1 : 1;
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java
index 4df9be4..a5552f8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java
@@ -34,12 +34,20 @@ public enum Operation {
KILL_TASK(QueueACL.ADMINISTER_JOBS, JobACL.MODIFY_JOB),
SET_JOB_PRIORITY(QueueACL.ADMINISTER_JOBS, JobACL.MODIFY_JOB),
SUBMIT_JOB(QueueACL.SUBMIT_JOB, null);
-
- public QueueACL qACLNeeded;
- public JobACL jobACLNeeded;
+
+ private final QueueACL qACLNeeded;
+ private final JobACL jobACLNeeded;
Operation(QueueACL qACL, JobACL jobACL) {
this.qACLNeeded = qACL;
this.jobACLNeeded = jobACL;
}
+
+ public QueueACL getqACLNeeded() {
+ return qACLNeeded;
+ }
+
+ public JobACL getJobACLNeeded() {
+ return jobACLNeeded;
+ }
}
\ No newline at end of file
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 34591e4..2aae65e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -413,17 +413,12 @@ public class HistoryFileManager extends AbstractService {
}
JobId jobId = jobIndexInfo.getJobId();
- List<Path> paths = new ArrayList<Path>(2);
if (historyFile == null) {
LOG.info("No file for job-history with " + jobId + " found in cache!");
- } else {
- paths.add(historyFile);
}
if (confFile == null) {
LOG.info("No file for jobConf with " + jobId + " found in cache!");
- } else {
- paths.add(confFile);
}
if (summaryFile == null || !intermediateDoneDirFc.util().exists(
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java
index a2db9d1..60b3638 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java
@@ -67,8 +67,12 @@ public final class Parser {
private void parse(File f, Map<Parameter, List<TaskResult>> sums) throws IOException {
if (f.isDirectory()) {
println("Process directory " + f);
- for(File child : f.listFiles())
- parse(child, sums);
+ File[] files = f.listFiles();
+ if (files != null) {
+ for(File child : files) {
+ parse(child, sums);
+ }
+ }
} else if (f.getName().endsWith(".txt")) {
println("Parse file " + f);
final Map<Parameter, List<TaskResult>> m = new TreeMap<Parameter, List<TaskResult>>();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org