You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dd...@apache.org on 2009/04/17 19:31:41 UTC
svn commit: r766093 - in /hadoop/core/trunk: ./
src/mapred/org/apache/hadoop/mapred/
src/mapred/org/apache/hadoop/mapred/jobcontrol/
src/mapred/org/apache/hadoop/mapred/lib/db/ src/test/
src/test/org/apache/hadoop/mapred/
Author: ddas
Date: Fri Apr 17 17:31:40 2009
New Revision: 766093
URL: http://svn.apache.org/viewvc?rev=766093&view=rev
Log:
HADOOP-5661. Fixes some findbugs warnings in o.a.h.mapred* packages and supresses a bunch of them. Contributed by Jothi Padmanabhan.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JVMId.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobEndNotifier.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/LineRecordReader.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskCompletionEvent.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskLog.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskReport.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/jobcontrol/Job.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBInputFormat.java
hadoop/core/trunk/src/test/findbugsExcludeFile.xml
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestParallelInitialization.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Fri Apr 17 17:31:40 2009
@@ -412,6 +412,9 @@
HADOOP-5647. Fix TestJobHistory to not depend on /tmp. (Ravi Gummadi
via sharad)
+ HADOOP-5661. Fixes some findbugs warnings in o.a.h.mapred* packages and supresses
+ a bunch of them. (Jothi Padmanabhan via ddas)
+
Release 0.20.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java Fri Apr 17 17:31:40 2009
@@ -171,7 +171,10 @@
TaskLog.syncLogs(firstTaskid, taskid, isCleanup);
if (!taskid.equals(firstTaskid) && !Shell.WINDOWS) {
// delete the pid-file's symlink
- new File(dstPidPath.toUri().getPath()).delete();
+ boolean b = new File(dstPidPath.toUri().getPath()).delete();
+ if (!b) {
+ LOG.warn("File delete failed. Ignoring");
+ }
}
}
if (numTasksToExecute > 0 && ++numTasksExecuted == numTasksToExecute) {
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java Fri Apr 17 17:31:40 2009
@@ -76,24 +76,6 @@
* Construct a new cluster status.
*
* @param trackers no. of tasktrackers in the cluster
- * @param maps no. of currently running map-tasks in the cluster
- * @param reduces no. of currently running reduce-tasks in the cluster
- * @param maxMaps the maximum no. of map tasks in the cluster
- * @param maxReduces the maximum no. of reduce tasks in the cluster
- * @param state the {@link JobTracker.State} of the <code>JobTracker</code>
- * @deprecated
- */
- @Deprecated
- ClusterStatus(int trackers, int maps, int reduces, int maxMaps,
- int maxReduces, JobTracker.State state) {
- this(trackers, 0, JobTracker.TASKTRACKER_EXPIRY_INTERVAL, maps, reduces,
- maxMaps, maxReduces, state);
- }
-
- /**
- * Construct a new cluster status.
- *
- * @param trackers no. of tasktrackers in the cluster
* @param blacklists no of blacklisted task trackers in the cluster
* @param ttExpiryInterval the tasktracker expiry interval
* @param maps no. of currently running map-tasks in the cluster
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JVMId.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JVMId.java?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JVMId.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JVMId.java Fri Apr 17 17:31:40 2009
@@ -56,7 +56,7 @@
public boolean equals(Object o) {
if(o == null)
return false;
- if(o.getClass().equals(JVMId.class)) {
+ if(o.getClass().equals(this.getClass())) {
JVMId that = (JVMId)o;
return this.id==that.id
&& this.isMap == that.isMap
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java Fri Apr 17 17:31:40 2009
@@ -769,11 +769,11 @@
ReflectionUtils.newInstance(context.getOutputFormatClass(), job);
output.checkOutputSpecs(context);
} else {
- job.getOutputFormat().checkOutputSpecs(fs, job);
+ job.getOutputFormat().checkOutputSpecs(getFs(), job);
}
// Create the splits for the job
- LOG.debug("Creating splits at " + fs.makeQualified(submitSplitFile));
+ LOG.debug("Creating splits at " + getFs().makeQualified(submitSplitFile));
int maps;
if (job.getUseNewMapper()) {
maps = writeNewSplits(context, submitSplitFile);
@@ -785,7 +785,7 @@
// Write job file to JobTracker's fs
FSDataOutputStream out =
- FileSystem.create(fs, submitJobFile,
+ FileSystem.create(getFs(), submitJobFile,
new FsPermission(JOB_FILE_PERMISSION));
try {
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobEndNotifier.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobEndNotifier.java?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobEndNotifier.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobEndNotifier.java Fri Apr 17 17:31:40 2009
@@ -156,9 +156,7 @@
LOG.error("Notification error [" + notification.getUri() + "]", ex);
}
try {
- synchronized (Thread.currentThread()) {
- Thread.currentThread().sleep(notification.getRetryInterval());
- }
+ Thread.sleep(notification.getRetryInterval());
}
catch (InterruptedException iex) {
LOG.error("Notification retry error [" + notification + "]", iex);
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java Fri Apr 17 17:31:40 2009
@@ -282,7 +282,8 @@
buf = new StringBuffer();
} while ((line = reader.readLine())!= null);
} finally {
- try { reader.close(); } catch (IOException ex) {}
+ try { reader.close(); } catch (IOException ex) {
+ LOG.warn("Ignoring exception in reader.close");}
}
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobTracker.java Fri Apr 17 17:31:40 2009
@@ -94,11 +94,10 @@
Configuration.addDefaultResource("mapred-site.xml");
}
- static long TASKTRACKER_EXPIRY_INTERVAL = 10 * 60 * 1000;
- static long RETIRE_JOB_INTERVAL;
- static long RETIRE_JOB_CHECK_INTERVAL;
+ private final long tasktrackerExpiryInterval;
+ private final long retireJobInterval;
+ private final long retireJobCheckInterval;
-
// The interval after which one fault of a tracker will be discarded,
// if there are no faults during this.
private static long UPDATE_FAULTY_TRACKER_INTERVAL = 24 * 60 * 60 * 1000;
@@ -234,7 +233,7 @@
while (true) {
try {
// Every 3 minutes check for any tasks that are overdue
- Thread.sleep(TASKTRACKER_EXPIRY_INTERVAL/3);
+ Thread.sleep(tasktrackerExpiryInterval/3);
long now = System.currentTimeMillis();
LOG.debug("Starting launching task sweep");
synchronized (JobTracker.this) {
@@ -246,7 +245,7 @@
TaskAttemptID taskId = pair.getKey();
long age = now - (pair.getValue()).longValue();
LOG.info(taskId + " is " + age + " ms debug.");
- if (age > TASKTRACKER_EXPIRY_INTERVAL) {
+ if (age > tasktrackerExpiryInterval) {
LOG.info("Launching task " + taskId + " timed out.");
TaskInProgress tip = null;
tip = taskidToTIPMap.get(taskId);
@@ -317,7 +316,7 @@
// The sleep interval must be no more than half the maximum expiry time
// for a task tracker.
//
- Thread.sleep(TASKTRACKER_EXPIRY_INTERVAL / 3);
+ Thread.sleep(tasktrackerExpiryInterval / 3);
//
// Loop through all expired items in the queue
@@ -336,7 +335,7 @@
TaskTrackerStatus leastRecent = null;
while ((trackerExpiryQueue.size() > 0) &&
((leastRecent = trackerExpiryQueue.first()) != null) &&
- (now - leastRecent.getLastSeen() > TASKTRACKER_EXPIRY_INTERVAL)) {
+ (now - leastRecent.getLastSeen() > tasktrackerExpiryInterval)) {
// Remove profile from head of queue
trackerExpiryQueue.remove(leastRecent);
@@ -348,7 +347,7 @@
// status stored in 'taskTrackers' might be null, which means the
// tracker has already been destroyed.
if (newProfile != null) {
- if (now - newProfile.getLastSeen() > TASKTRACKER_EXPIRY_INTERVAL) {
+ if (now - newProfile.getLastSeen() > tasktrackerExpiryInterval) {
// Remove completely after marking the tasks as 'KILLED'
lostTaskTracker(leastRecent.getTrackerName());
// tracker is lost, and if it is blacklisted, remove
@@ -392,10 +391,10 @@
public void run() {
while (true) {
try {
- Thread.sleep(RETIRE_JOB_CHECK_INTERVAL);
+ Thread.sleep(retireJobCheckInterval);
List<JobInProgress> retiredJobs = new ArrayList<JobInProgress>();
long now = System.currentTimeMillis();
- long retireBefore = now - RETIRE_JOB_INTERVAL;
+ long retireBefore = now - retireJobInterval;
synchronized (jobs) {
for(JobInProgress job: jobs.values()) {
@@ -1505,10 +1504,10 @@
//
// Grab some static constants
//
- TASKTRACKER_EXPIRY_INTERVAL =
+ tasktrackerExpiryInterval =
conf.getLong("mapred.tasktracker.expiry.interval", 10 * 60 * 1000);
- RETIRE_JOB_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.interval", 24 * 60 * 60 * 1000);
- RETIRE_JOB_CHECK_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.check", 60 * 1000);
+ retireJobInterval = conf.getLong("mapred.jobtracker.retirejob.interval", 24 * 60 * 60 * 1000);
+ retireJobCheckInterval = conf.getLong("mapred.jobtracker.retirejob.check", 60 * 1000);
MAX_COMPLETE_USER_JOBS_IN_MEMORY = conf.getInt("mapred.jobtracker.completeuserjobs.maximum", 100);
MAX_BLACKLISTS_PER_TRACKER =
conf.getInt("mapred.max.tracker.blacklists", 4);
@@ -2000,7 +1999,7 @@
* Call {@link #removeTaskEntry(String)} for each of the
* job's tasks.
* When the JobTracker is retiring the long-completed
- * job, either because it has outlived {@link #RETIRE_JOB_INTERVAL}
+ * job, either because it has outlived {@link #retireJobInterval}
* or the limit of {@link #MAX_COMPLETE_USER_JOBS_IN_MEMORY} jobs
* has been reached, we can afford to nuke all it's tasks; a little
* unsafe, but practically feasible.
@@ -3013,7 +3012,7 @@
List<List<String>> trackerNames = taskTrackerNames();
return new ClusterStatus(trackerNames.get(0),
trackerNames.get(1),
- TASKTRACKER_EXPIRY_INTERVAL,
+ tasktrackerExpiryInterval,
totalMaps,
totalReduces,
totalMapTaskCapacity,
@@ -3023,7 +3022,7 @@
return new ClusterStatus(taskTrackers.size() -
getBlacklistedTrackerCount(),
getBlacklistedTrackerCount(),
- TASKTRACKER_EXPIRY_INTERVAL,
+ tasktrackerExpiryInterval,
totalMaps,
totalReduces,
totalMapTaskCapacity,
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/LineRecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/LineRecordReader.java?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/LineRecordReader.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/LineRecordReader.java Fri Apr 17 17:31:40 2009
@@ -154,7 +154,7 @@
/**
* Get the progress within the split
*/
- public float getProgress() {
+ public synchronized float getProgress() {
if (start == end) {
return 0.0f;
} else {
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/MapTask.java Fri Apr 17 17:31:40 2009
@@ -936,65 +936,67 @@
throws IOException {
boolean buffull = false;
boolean wrap = false;
- spillLock.lock();
- try {
- do {
- if (sortSpillException != null) {
- throw (IOException)new IOException("Spill failed"
- ).initCause(sortSpillException);
- }
-
- // sufficient buffer space?
- if (bufstart <= bufend && bufend <= bufindex) {
- buffull = bufindex + len > bufvoid;
- wrap = (bufvoid - bufindex) + bufstart > len;
- } else {
- // bufindex <= bufstart <= bufend
- // bufend <= bufindex <= bufstart
- wrap = false;
- buffull = bufindex + len > bufstart;
- }
-
- if (kvstart == kvend) {
- // spill thread not running
- if (kvend != kvindex) {
- // we have records we can spill
- final boolean bufsoftlimit = (bufindex > bufend)
- ? bufindex - bufend > softBufferLimit
- : bufend - bufindex < bufvoid - softBufferLimit;
- if (bufsoftlimit || (buffull && !wrap)) {
- LOG.info("Spilling map output: buffer full= " + bufsoftlimit);
- startSpill();
- }
- } else if (buffull && !wrap) {
- // We have no buffered records, and this record is too large
- // to write into kvbuffer. We must spill it directly from
- // collect
- final int size = ((bufend <= bufindex)
- ? bufindex - bufend
- : (bufvoid - bufend) + bufindex) + len;
- bufstart = bufend = bufindex = bufmark = 0;
- kvstart = kvend = kvindex = 0;
- bufvoid = kvbuffer.length;
- throw new MapBufferTooSmallException(size + " bytes");
- }
- }
-
- if (buffull && !wrap) {
- try {
- while (kvstart != kvend) {
- reporter.progress();
- spillDone.await();
- }
- } catch (InterruptedException e) {
- throw (IOException)new IOException(
- "Buffer interrupted while waiting for the writer"
- ).initCause(e);
- }
- }
- } while (buffull && !wrap);
- } finally {
- spillLock.unlock();
+ synchronized(MapOutputBuffer.this) {
+ spillLock.lock();
+ try {
+ do {
+ if (sortSpillException != null) {
+ throw (IOException)new IOException("Spill failed"
+ ).initCause(sortSpillException);
+ }
+
+ // sufficient buffer space?
+ if (bufstart <= bufend && bufend <= bufindex) {
+ buffull = bufindex + len > bufvoid;
+ wrap = (bufvoid - bufindex) + bufstart > len;
+ } else {
+ // bufindex <= bufstart <= bufend
+ // bufend <= bufindex <= bufstart
+ wrap = false;
+ buffull = bufindex + len > bufstart;
+ }
+
+ if (kvstart == kvend) {
+ // spill thread not running
+ if (kvend != kvindex) {
+ // we have records we can spill
+ final boolean bufsoftlimit = (bufindex > bufend)
+ ? bufindex - bufend > softBufferLimit
+ : bufend - bufindex < bufvoid - softBufferLimit;
+ if (bufsoftlimit || (buffull && !wrap)) {
+ LOG.info("Spilling map output: buffer full= " + bufsoftlimit);
+ startSpill();
+ }
+ } else if (buffull && !wrap) {
+ // We have no buffered records, and this record is too large
+ // to write into kvbuffer. We must spill it directly from
+ // collect
+ final int size = ((bufend <= bufindex)
+ ? bufindex - bufend
+ : (bufvoid - bufend) + bufindex) + len;
+ bufstart = bufend = bufindex = bufmark = 0;
+ kvstart = kvend = kvindex = 0;
+ bufvoid = kvbuffer.length;
+ throw new MapBufferTooSmallException(size + " bytes");
+ }
+ }
+
+ if (buffull && !wrap) {
+ try {
+ while (kvstart != kvend) {
+ reporter.progress();
+ spillDone.await();
+ }
+ } catch (InterruptedException e) {
+ throw (IOException)new IOException(
+ "Buffer interrupted while waiting for the writer"
+ ).initCause(e);
+ }
+ }
+ } while (buffull && !wrap);
+ } finally {
+ spillLock.unlock();
+ }
}
// here, we know that we have sufficient space to write
if (buffull) {
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ReduceTask.java?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/ReduceTask.java Fri Apr 17 17:31:40 2009
@@ -1039,6 +1039,10 @@
notifyAll();
}
+ private synchronized int getNumPendingRequests() {
+ return numPendingRequests;
+ }
+
public boolean waitForDataToMerge() throws InterruptedException {
boolean done = false;
synchronized (dataAvailable) {
@@ -1060,10 +1064,10 @@
// have been fetched, so there is nothing to merge) or the
// last map outputs being transferred without
// contention, so a merge would be premature.
- (numPendingRequests <
+ (getNumPendingRequests() <
numCopiers*MAX_STALLED_SHUFFLE_THREADS_FRACTION &&
(0 == numRequiredMapOutputs ||
- numPendingRequests < numRequiredMapOutputs))) {
+ getNumPendingRequests() < numRequiredMapOutputs))) {
dataAvailable.wait();
}
done = closed;
@@ -1303,7 +1307,6 @@
filename = new Path(tmpMapOutput.getParent(), filename.getName());
if (!localFileSys.rename(tmpMapOutput, filename)) {
localFileSys.delete(tmpMapOutput, true);
- bytes = -1;
throw new IOException("Failed to rename map output " +
tmpMapOutput + " to " + filename);
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskCompletionEvent.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskCompletionEvent.java?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskCompletionEvent.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskCompletionEvent.java Fri Apr 17 17:31:40 2009
@@ -184,7 +184,7 @@
public boolean equals(Object o) {
if(o == null)
return false;
- if(o.getClass().equals(TaskCompletionEvent.class)) {
+ if(o.getClass().equals(this.getClass())) {
TaskCompletionEvent event = (TaskCompletionEvent) o;
return this.isMap == event.isMapTask()
&& this.eventId == event.getEventId()
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskLog.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskLog.java?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskLog.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskLog.java Fri Apr 17 17:31:40 2009
@@ -66,7 +66,10 @@
LOG.warn("Getting local file system failed.");
}
if (!LOG_DIR.exists()) {
- LOG_DIR.mkdirs();
+ boolean b = LOG_DIR.mkdirs();
+ if (!b) {
+ LOG.warn("mkdirs failed. Ignoring.");
+ }
}
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskReport.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskReport.java?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskReport.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskReport.java Fri Apr 17 17:31:40 2009
@@ -170,7 +170,7 @@
public boolean equals(Object o) {
if(o == null)
return false;
- if(o.getClass().equals(TaskReport.class)) {
+ if(o.getClass().equals(this.getClass())) {
TaskReport report = (TaskReport) o;
return counters.equals(report.getCounters())
&& Arrays.toString(this.diagnostics)
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java Fri Apr 17 17:31:40 2009
@@ -392,7 +392,10 @@
// Set up the redirection of the task's stdout and stderr streams
File stdout = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDOUT);
File stderr = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDERR);
- stdout.getParentFile().mkdirs();
+ boolean b = stdout.getParentFile().mkdirs();
+ if (!b) {
+ LOG.warn("mkdirs failed. Ignoring");
+ }
tracker.getTaskTrackerInstrumentation().reportTaskLaunch(taskid, stdout, stderr);
Map<String, String> env = new HashMap<String, String>();
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/jobcontrol/Job.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/jobcontrol/Job.java?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/jobcontrol/Job.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/jobcontrol/Job.java Fri Apr 17 17:31:40 2009
@@ -181,19 +181,19 @@
/**
* @return the mapred job conf of this job
*/
- public JobConf getJobConf() {
+ public synchronized JobConf getJobConf() {
return this.theJobConf;
}
-
+
/**
* Set the mapred job conf for this job.
* @param jobConf the mapred job conf for this job.
*/
- public void setJobConf(JobConf jobConf) {
+ public synchronized void setJobConf(JobConf jobConf) {
this.theJobConf = jobConf;
}
-
+
/**
* @return the state of this job
*/
@@ -212,18 +212,18 @@
/**
* @return the message of this job
*/
- public String getMessage() {
+ public synchronized String getMessage() {
return this.message;
}
-
+
/**
* Set the message for this job.
* @param message the message for this job.
*/
- public void setMessage(String message) {
+ public synchronized void setMessage(String message) {
this.message = message;
}
-
+
/**
* @return the job client of this job
@@ -260,7 +260,7 @@
/**
* @return true if this job is in a complete state
*/
- public boolean isCompleted() {
+ public synchronized boolean isCompleted() {
return this.state == Job.FAILED ||
this.state == Job.DEPENDENT_FAILED ||
this.state == Job.SUCCESS;
@@ -269,7 +269,7 @@
/**
* @return true if this job is in READY state
*/
- public boolean isReady() {
+ public synchronized boolean isReady() {
return this.state == Job.READY;
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBInputFormat.java?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBInputFormat.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBInputFormat.java Fri Apr 17 17:31:40 2009
@@ -293,10 +293,12 @@
/** {@inheritDoc} */
public InputSplit[] getSplits(JobConf job, int chunks) throws IOException {
+ ResultSet results = null;
+ Statement statement = null;
try {
- Statement statement = connection.createStatement();
+ statement = connection.createStatement();
- ResultSet results = statement.executeQuery(getCountQuery());
+ results = statement.executeQuery(getCountQuery());
results.next();
long count = results.getLong(1);
@@ -323,6 +325,12 @@
return splits;
} catch (SQLException e) {
+ try {
+ if (results != null) { results.close(); }
+ } catch (SQLException e1) {}
+ try {
+ if (statement != null) { statement.close(); }
+ } catch (SQLException e1) {}
throw new IOException(e.getMessage());
}
}
Modified: hadoop/core/trunk/src/test/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/findbugsExcludeFile.xml?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/findbugsExcludeFile.xml (original)
+++ hadoop/core/trunk/src/test/findbugsExcludeFile.xml Fri Apr 17 17:31:40 2009
@@ -28,4 +28,104 @@
<Field name="out" />
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
+ <!--
+ Ignore Cross Scripting Vulnerabilities
+ -->
+ <Match>
+ <Package name="~org.apache.hadoop.mapred.*" />
+ <Bug code="XSS" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.mapred.taskdetails_jsp" />
+ <Bug code="HRS" />
+ </Match>
+ <!--
+ Ignore warnings where child class has the same name as
+ super class. Classes based on Old API shadow names from
+ new API. Should go off after HADOOP-1.0
+ -->
+ <Match>
+ <Class name="~org.apache.hadoop.mapred.*" />
+ <Bug pattern="NM_SAME_SIMPLE_NAME_AS_SUPERCLASS" />
+ </Match>
+ <!--
+ Ignore warnings for usage of System.exit. This is
+ required and have been well thought out
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.mapred.Child$2" />
+ <Method name="run" />
+ <Bug pattern="DM_EXIT" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.mapred.JobTracker" />
+ <Method name="addHostToNodeMapping" />
+ <Bug pattern="DM_EXIT" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.mapred.Task" />
+ <Or>
+ <Method name="done" />
+ <Method name="commit" />
+ <Method name="statusUpdate" />
+ </Or>
+ <Bug pattern="DM_EXIT" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.mapred.Task$TaskReporter" />
+ <Method name="run" />
+ <Bug pattern="DM_EXIT" />
+ </Match>
+ <!--
+ We need to cast objects between old and new api objects
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.mapred.OutputCommitter" />
+ <Bug pattern="BC_UNCONFIRMED_CAST" />
+ </Match>
+ <!--
+ We intentionally do the get name from the inner class
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.mapred.TaskTracker$MapEventsFetcherThread" />
+ <Method name="run" />
+ <Bug pattern="IA_AMBIGUOUS_INVOCATION_OF_INHERITED_OR_OUTER_METHOD" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.mapred.FileOutputCommitter" />
+ <Bug pattern="NM_WRONG_PACKAGE_INTENTIONAL" />
+ </Match>
+ <!--
+ Ignoring this warning as resolving this would need a non-trivial change in code
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorBaseDescriptor" />
+ <Method name="configure" />
+ <Field name="maxNumItems" />
+ <Bug pattern="ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD" />
+ </Match>
+ <!--
+ Comes from org.apache.jasper.runtime.ResourceInjector. Cannot do much.
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.mapred.jobqueue_005fdetails_jsp" />
+ <Field name="_jspx_resourceInjector" />
+ <Bug pattern="SE_BAD_FIELD" />
+ </Match>
+ <!--
+ Storing textInputFormat and then passing it as a parameter. Safe to ignore.
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJob" />
+ <Method name="createValueAggregatorJob" />
+ <Bug pattern="DLS_DEAD_STORE_OF_CLASS_LITERAL" />
+ </Match>
+ <!--
+ Can remove this after the upgrade to findbugs1.3.8
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.mapred.lib.db.DBInputFormat" />
+ <Method name="getSplits" />
+ <Bug pattern="DLS_DEAD_LOCAL_STORE" />
+ </Match>
</FindBugsFilter>
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java Fri Apr 17 17:31:40 2009
@@ -136,7 +136,7 @@
public ClusterStatus getClusterStatus() {
int numTrackers = trackers.size();
return new ClusterStatus(numTrackers, 0,
- JobTracker.TASKTRACKER_EXPIRY_INTERVAL,
+ 10 * 60 * 1000,
maps, reduces,
numTrackers * maxMapTasksPerTracker,
numTrackers * maxReduceTasksPerTracker,
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestParallelInitialization.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestParallelInitialization.java?rev=766093&r1=766092&r2=766093&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestParallelInitialization.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestParallelInitialization.java Fri Apr 17 17:31:40 2009
@@ -95,7 +95,7 @@
public ClusterStatus getClusterStatus() {
int numTrackers = trackers.size();
return new ClusterStatus(numTrackers, 0,
- JobTracker.TASKTRACKER_EXPIRY_INTERVAL,
+ 10 * 60 * 1000,
maps, reduces,
numTrackers * maxMapTasksPerTracker,
numTrackers * maxReduceTasksPerTracker,