You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2015/01/13 18:50:46 UTC
[45/50] [abbrv] hadoop git commit: HADOOP-11465. Fix findbugs
warnings in hadoop-gridmix. (Contributed by Varun Saxena)
HADOOP-11465. Fix findbugs warnings in hadoop-gridmix. (Contributed by Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c4cba616
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c4cba616
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c4cba616
Branch: refs/heads/HDFS-6994
Commit: c4cba6165a3afbf4f1f8ff6b7f11286772d70d6f
Parents: 5188153
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Jan 12 21:22:58 2015 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Jan 12 21:22:58 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../dev-support/findbugs-exclude.xml | 7 ++++
.../gridmix/CompressionEmulationUtil.java | 8 +++--
.../gridmix/DistributedCacheEmulator.java | 10 ++++--
.../mapred/gridmix/GenerateDistCacheData.java | 6 +++-
.../apache/hadoop/mapred/gridmix/Gridmix.java | 36 ++++++++++----------
.../hadoop/mapred/gridmix/PseudoLocalFs.java | 2 +-
.../hadoop/mapred/gridmix/Statistics.java | 26 +++++++++++---
8 files changed, 68 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4cba616/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index a9300c2..59e6d80 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -475,6 +475,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-9992. Modify the NN loadGenerator to optionally run as a MapReduce job
(Akshay Radia via brandonli)
+ HADOOP-11465. Fix findbugs warnings in hadoop-gridmix. (Varun Saxena via
+ Arpit Agarwal)
+
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4cba616/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml
index 92458d4..cb043c4 100644
--- a/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml
+++ b/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml
@@ -30,4 +30,11 @@
<Bug pattern="REC_CATCH_EXCEPTION"/>
<Bug code="REC"/>
</Match>
+
+ <!-- This has been done knowingly and meant to fool JVM so that it doesn't optimize code -->
+ <Match>
+ <Class name="org.apache.hadoop.mapred.gridmix.emulators.resourceusage.CumulativeCpuUsageEmulatorPlugin$DefaultCpuUsageEmulator"/>
+ <Field name="returnValue"/>
+ <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD"/>
+ </Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4cba616/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java
index 526878c..bd7e878 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java
@@ -21,6 +21,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
@@ -97,7 +98,9 @@ class CompressionEmulationUtil {
private static final CompressionRatioLookupTable COMPRESSION_LOOKUP_TABLE =
new CompressionRatioLookupTable();
-
+
+ private static final Charset charsetUTF8 = Charset.forName("UTF-8");
+
/**
* This is a {@link Mapper} implementation for generating random text data.
* It uses {@link RandomTextDataGenerator} for generating text data and the
@@ -133,7 +136,8 @@ class CompressionEmulationUtil {
String randomKey = rtg.getRandomWord();
String randomValue = rtg.getRandomWord();
context.write(new Text(randomKey), new Text(randomValue));
- bytes -= (randomValue.getBytes().length + randomKey.getBytes().length);
+ bytes -= (randomValue.getBytes(charsetUTF8).length +
+ randomKey.getBytes(charsetUTF8).length);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4cba616/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
index ad3b113..d30a51d 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.tools.rumen.Pre21JobHistoryConstants;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -112,6 +113,8 @@ class DistributedCacheEmulator {
Configuration conf; // gridmix configuration
+ private static final Charset charsetUTF8 = Charset.forName("UTF-8");
+
// Pseudo local file system where local FS based distributed cache files are
// created by gridmix.
FileSystem pseudoLocalFs = null;
@@ -436,9 +439,10 @@ class DistributedCacheEmulator {
for (Iterator it = dcFiles.iterator(); it.hasNext();) {
Map.Entry entry = (Map.Entry)it.next();
LongWritable fileSize =
- new LongWritable(Long.valueOf(entry.getValue().toString()));
+ new LongWritable(Long.parseLong(entry.getValue().toString()));
BytesWritable filePath =
- new BytesWritable(entry.getKey().toString().getBytes());
+ new BytesWritable(
+ entry.getKey().toString().getBytes(charsetUTF8));
byteCount += fileSize.get();
bytesSync += fileSize.get();
@@ -515,7 +519,7 @@ class DistributedCacheEmulator {
// local FS based distributed cache file.
// Create this file on the pseudo local FS.
String fileId = MD5Hash.digest(files[i] + timeStamps[i]).toString();
- long fileSize = Long.valueOf(fileSizes[i]);
+ long fileSize = Long.parseLong(fileSizes[i]);
Path mappedLocalFilePath =
PseudoLocalFs.generateFilePath(fileId, fileSize)
.makeQualified(pseudoLocalFs.getUri(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4cba616/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java
index 0512746..4a75cde 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.mapred.gridmix;
import java.io.IOException;
+import java.nio.charset.Charset;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.List;
@@ -95,6 +96,8 @@ class GenerateDistCacheData extends GridmixJob {
*/
static final short GRIDMIX_DISTCACHE_FILE_PERM = 0644;
+ private static final Charset charsetUTF8 = Charset.forName("UTF-8");
+
public GenerateDistCacheData(Configuration conf) throws IOException {
super(conf, 0L, JOB_NAME);
}
@@ -152,7 +155,8 @@ class GenerateDistCacheData extends GridmixJob {
public void map(LongWritable key, BytesWritable value, Context context)
throws IOException, InterruptedException {
- String fileName = new String(value.getBytes(), 0, value.getLength());
+ String fileName = new String(value.getBytes(), 0,
+ value.getLength(), charsetUTF8);
Path path = new Path(fileName);
FSDataOutputStream dos =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4cba616/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
index 8ac590b..4386bc1 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
@@ -729,40 +729,40 @@ public class Gridmix extends Configured implements Tool {
out.println(" -users <usersResourceURI> : URI that contains the users list.");
out.println("Configuration parameters:");
out.println(" General parameters:");
- out.printf(" %-48s : Output directory\n", GRIDMIX_OUT_DIR);
- out.printf(" %-48s : Submitting threads\n", GRIDMIX_SUB_THR);
- out.printf(" %-48s : Queued job desc\n", GRIDMIX_QUE_DEP);
- out.printf(" %-48s : User resolution class\n", GRIDMIX_USR_RSV);
- out.printf(" %-48s : Job types (%s)\n", JobCreator.GRIDMIX_JOB_TYPE, getJobTypes());
+ out.printf(" %-48s : Output directory%n", GRIDMIX_OUT_DIR);
+ out.printf(" %-48s : Submitting threads%n", GRIDMIX_SUB_THR);
+ out.printf(" %-48s : Queued job desc%n", GRIDMIX_QUE_DEP);
+ out.printf(" %-48s : User resolution class%n", GRIDMIX_USR_RSV);
+ out.printf(" %-48s : Job types (%s)%n", JobCreator.GRIDMIX_JOB_TYPE, getJobTypes());
out.println(" Parameters related to job submission:");
- out.printf(" %-48s : Default queue\n",
+ out.printf(" %-48s : Default queue%n",
GridmixJob.GRIDMIX_DEFAULT_QUEUE);
- out.printf(" %-48s : Enable/disable using queues in trace\n",
+ out.printf(" %-48s : Enable/disable using queues in trace%n",
GridmixJob.GRIDMIX_USE_QUEUE_IN_TRACE);
- out.printf(" %-48s : Job submission policy (%s)\n",
+ out.printf(" %-48s : Job submission policy (%s)%n",
GridmixJobSubmissionPolicy.JOB_SUBMISSION_POLICY, getSubmissionPolicies());
out.println(" Parameters specific for LOADJOB:");
- out.printf(" %-48s : Key fraction of rec\n",
+ out.printf(" %-48s : Key fraction of rec%n",
AvgRecordFactory.GRIDMIX_KEY_FRC);
out.println(" Parameters specific for SLEEPJOB:");
- out.printf(" %-48s : Whether to ignore reduce tasks\n",
+ out.printf(" %-48s : Whether to ignore reduce tasks%n",
SleepJob.SLEEPJOB_MAPTASK_ONLY);
- out.printf(" %-48s : Number of fake locations for map tasks\n",
+ out.printf(" %-48s : Number of fake locations for map tasks%n",
JobCreator.SLEEPJOB_RANDOM_LOCATIONS);
- out.printf(" %-48s : Maximum map task runtime in mili-sec\n",
+ out.printf(" %-48s : Maximum map task runtime in mili-sec%n",
SleepJob.GRIDMIX_SLEEP_MAX_MAP_TIME);
- out.printf(" %-48s : Maximum reduce task runtime in mili-sec (merge+reduce)\n",
+ out.printf(" %-48s : Maximum reduce task runtime in mili-sec (merge+reduce)%n",
SleepJob.GRIDMIX_SLEEP_MAX_REDUCE_TIME);
out.println(" Parameters specific for STRESS submission throttling policy:");
- out.printf(" %-48s : jobs vs task-tracker ratio\n",
+ out.printf(" %-48s : jobs vs task-tracker ratio%n",
StressJobFactory.CONF_MAX_JOB_TRACKER_RATIO);
- out.printf(" %-48s : maps vs map-slot ratio\n",
+ out.printf(" %-48s : maps vs map-slot ratio%n",
StressJobFactory.CONF_OVERLOAD_MAPTASK_MAPSLOT_RATIO);
- out.printf(" %-48s : reduces vs reduce-slot ratio\n",
+ out.printf(" %-48s : reduces vs reduce-slot ratio%n",
StressJobFactory.CONF_OVERLOAD_REDUCETASK_REDUCESLOT_RATIO);
- out.printf(" %-48s : map-slot share per job\n",
+ out.printf(" %-48s : map-slot share per job%n",
StressJobFactory.CONF_MAX_MAPSLOT_SHARE_PER_JOB);
- out.printf(" %-48s : reduce-slot share per job\n",
+ out.printf(" %-48s : reduce-slot share per job%n",
StressJobFactory.CONF_MAX_REDUCESLOT_SHARE_PER_JOB);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4cba616/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java
index 497108a..d7ef563 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java
@@ -124,7 +124,7 @@ class PseudoLocalFs extends FileSystem {
} else {
String[] parts = path.toUri().getPath().split("\\.");
try {
- fileSize = Long.valueOf(parts[parts.length - 1]);
+ fileSize = Long.parseLong(parts[parts.length - 1]);
valid = (fileSize >= 0);
} catch (NumberFormatException e) {
valid = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4cba616/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java
index 322d755..915788b 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java
@@ -119,7 +119,23 @@ public class Statistics implements Component<Statistics.JobStats> {
}
return new JobStats(maps, reds, job);
}
-
+
+ private static void addToNumMapsSubmitted(int numMaps) {
+ numMapsSubmitted += numMaps;
+ }
+
+ private static void addToNumReducesSubmitted(int numReduces) {
+ numReducesSubmitted += numReduces;
+ }
+
+ private static void subtractFromNumMapsSubmitted(int numMaps) {
+ numMapsSubmitted -= numMaps;
+ }
+
+ private static void subtractFromNumReducesSubmitted(int numReduces) {
+ numReducesSubmitted -= numReduces;
+ }
+
/**
* Add a submitted job for monitoring.
*/
@@ -131,8 +147,8 @@ public class Statistics implements Component<Statistics.JobStats> {
return;
}
submittedJobsMap.put(seq, stats);
- numMapsSubmitted += stats.getNoOfMaps();
- numReducesSubmitted += stats.getNoOfReds();
+ addToNumMapsSubmitted(stats.getNoOfMaps());
+ addToNumReducesSubmitted(stats.getNoOfReds());
}
/**
@@ -156,8 +172,8 @@ public class Statistics implements Component<Statistics.JobStats> {
}
// update the total number of submitted map/reduce task count
- numMapsSubmitted -= stat.getNoOfMaps();
- numReducesSubmitted -= stat.getNoOfReds();
+ subtractFromNumMapsSubmitted(stat.getNoOfMaps());
+ subtractFromNumReducesSubmitted(stat.getNoOfReds());
completedJobsInCurrentInterval++;
//check if we have reached the maximum level of job completions.