You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 06:24:01 UTC
svn commit: r1077804 - in
/hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred:
MapTask.java ReduceTask.java Task.java
Author: omalley
Date: Fri Mar 4 05:24:01 2011
New Revision: 1077804
URL: http://svn.apache.org/viewvc?rev=1077804&view=rev
Log:
commit 4713a5bf905e0b566a3ac7882f0e79bbb0f8fa7f
Author: Siddharth Seth <ss...@yahoo-inc.com>
Date: Thu Feb 17 12:33:14 2011 -0800
Qualify the paths before figuring out which filesystem
statistics need to be tracked
Modified:
hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/MapTask.java
hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/Task.java
Modified: hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/MapTask.java?rev=1077804&r1=1077803&r2=1077804&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/MapTask.java Fri Mar 4 05:24:01 2011
@@ -189,7 +189,7 @@ class MapTask extends Task {
Statistics matchedStats = null;
if (split instanceof FileSplit) {
- matchedStats = getFsStatistics(((FileSplit) split).getPath());
+ matchedStats = getFsStatistics(((FileSplit) split).getPath(), job);
}
fsStats = matchedStats;
@@ -480,7 +480,7 @@ class MapTask extends Task {
Statistics matchedStats = null;
if (split instanceof org.apache.hadoop.mapreduce.lib.input.FileSplit) {
matchedStats = getFsStatistics(((org.apache.hadoop.mapreduce.lib.input.FileSplit) split)
- .getPath());
+ .getPath(), job);
}
fsStats = matchedStats;
@@ -615,7 +615,7 @@ class MapTask extends Task {
Statistics matchedStats = null;
if (outputFormat instanceof org.apache.hadoop.mapreduce.lib.output.FileOutputFormat) {
matchedStats = getFsStatistics(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
- .getOutputPath(jobContext));
+ .getOutputPath(jobContext), job);
}
fsStats = matchedStats;
mapOutputRecordCounter =
@@ -808,7 +808,7 @@ class MapTask extends Task {
Statistics matchedStats = null;
if (outputFormat instanceof FileOutputFormat) {
- matchedStats = getFsStatistics(FileOutputFormat.getOutputPath(job));
+ matchedStats = getFsStatistics(FileOutputFormat.getOutputPath(job), job);
}
fsStats = matchedStats;
mapOutputRecordCounter = reporter.getCounter(MAP_OUTPUT_RECORDS);
Modified: hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/ReduceTask.java?rev=1077804&r1=1077803&r2=1077804&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/ReduceTask.java Fri Mar 4 05:24:01 2011
@@ -438,7 +438,7 @@ class ReduceTask extends Task {
.getCounter(FileOutputFormat.Counter.BYTES_WRITTEN);
Statistics matchedStats = null;
if (job.getOutputFormat() instanceof FileOutputFormat) {
- matchedStats = getFsStatistics(FileOutputFormat.getOutputPath(job));
+ matchedStats = getFsStatistics(FileOutputFormat.getOutputPath(job), job);
}
fsStats = matchedStats;
@@ -560,7 +560,7 @@ class ReduceTask extends Task {
// getTaskID());
if (outputFormat instanceof org.apache.hadoop.mapreduce.lib.output.FileOutputFormat) {
matchedStats = getFsStatistics(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
- .getOutputPath(taskContext));
+ .getOutputPath(taskContext), taskContext.getConfiguration());
}
fsStats = matchedStats;
Modified: hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/Task.java?rev=1077804&r1=1077803&r2=1077804&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/common/branches/branch-0.20-security-203/src/mapred/org/apache/hadoop/mapred/Task.java Fri Mar 4 05:24:01 2011
@@ -1021,8 +1021,10 @@ abstract public class Task implements Wr
* the path.
* @return a Statistics instance, or null if none is found for the scheme.
*/
- protected static Statistics getFsStatistics(Path path) {
+ protected static Statistics getFsStatistics(Path path, Configuration conf)
+ throws IOException {
Statistics matchedStats = null;
+ path = path.getFileSystem(conf).makeQualified(path);
for (Statistics stats : FileSystem.getAllStatistics()) {
if (stats.getScheme().equals(path.toUri().getScheme())) {
matchedStats = stats;