You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2009/04/08 07:22:06 UTC
svn commit: r763110 - in /hadoop/core/branches/branch-0.20: ./
src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/
src/core/org/apache/hadoop/fs/ src/core/org/apache/hadoop/fs/ftp/
src/core/org/apache/hadoop/fs/kfs/ src/core/org/apache/ha...
Author: omalley
Date: Wed Apr 8 05:22:05 2009
New Revision: 763110
URL: http://svn.apache.org/viewvc?rev=763110&view=rev
Log:
HADOOP-5585. Clear FileSystem statistics between tasks when jvm-reuse
is enabled. (omalley)
Modified:
hadoop/core/branches/branch-0.20/ (props changed)
hadoop/core/branches/branch-0.20/CHANGES.txt (contents, props changed)
hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ClusterWithCapacityScheduler.java
hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/FileSystem.java
hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/RawLocalFileSystem.java
hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/ftp/FTPFileSystem.java
hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/kfs/KosmosFileSystem.java
hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/s3/S3FileSystem.java
hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
hadoop/core/branches/branch-0.20/src/examples/org/apache/hadoop/examples/Sort.java
hadoop/core/branches/branch-0.20/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java
hadoop/core/branches/branch-0.20/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
hadoop/core/branches/branch-0.20/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Child.java
hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/io/TestSequenceFile.java
hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java
Propchange: hadoop/core/branches/branch-0.20/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Apr 8 05:22:05 2009
@@ -1,2 +1,2 @@
/hadoop/core/branches/branch-0.19:713112
-/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,759398,759932,760502,760783,761046,761482,761632,762216,762879
+/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107
Modified: hadoop/core/branches/branch-0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=763110&r1=763109&r2=763110&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.20/CHANGES.txt Wed Apr 8 05:22:05 2009
@@ -857,6 +857,9 @@
HADOOP-5068. Fix NPE in TestCapacityScheduler. (Vinod Kumar Vavilapalli
via szetszwo)
+ HADOOP-5585. Clear FileSystem statistics between tasks when jvm-reuse
+ is enabled. (omalley)
+
Release 0.19.2 - Unreleased
BUG FIXES
Propchange: hadoop/core/branches/branch-0.20/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Apr 8 05:22:05 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
-/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752514,752555,752590,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755986,755998,756352,757448,757624,757849,758156,759398,759932,760502,760783,761046,761482,761632,762216,762879
+/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752514,752555,752590,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755986,755998,756352,757448,757624,757849,758156,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107
Modified: hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ClusterWithCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ClusterWithCapacityScheduler.java?rev=763110&r1=763109&r2=763110&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ClusterWithCapacityScheduler.java (original)
+++ hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ClusterWithCapacityScheduler.java Wed Apr 8 05:22:05 2009
@@ -31,6 +31,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -117,11 +118,7 @@
throws IOException {
Configuration config = new Configuration(false);
- LocalFileSystem fs = new LocalFileSystem();
- fs.setConf(config);
- // The above call doesn't set the configuration for the underlying
- // RawFileSystem. Explicitly doing it.
- fs.getRawFileSystem().setConf(config);
+ LocalFileSystem fs = FileSystem.getLocal(config);
String myResourcePath = System.getProperty("test.build.data");
Path schedulerConfigFilePath =
@@ -146,9 +143,7 @@
private void cleanUpSchedulerConfigFile() throws IOException {
Configuration config = new Configuration(false);
- LocalFileSystem fs = new LocalFileSystem();
- fs.setConf(config);
- fs.getRawFileSystem().setConf(config);
+ LocalFileSystem fs = FileSystem.getLocal(config);
String myResourcePath = System.getProperty("test.build.data");
Path schedulerConfigFilePath =
Modified: hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/FileSystem.java?rev=763110&r1=763109&r2=763110&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/FileSystem.java Wed Apr 8 05:22:05 2009
@@ -17,9 +17,20 @@
*/
package org.apache.hadoop.fs;
-import java.io.*;
-import java.net.*;
-import java.util.*;
+import java.io.Closeable;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.IdentityHashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
@@ -68,10 +79,6 @@
statisticsTable =
new IdentityHashMap<Class<? extends FileSystem>, Statistics>();
- /** Recording statistics per FileSystem URI scheme */
- private static final Map<String, Statistics> statsByUriScheme =
- new HashMap<String, Statistics>();
-
/**
* The statistics for this file system.
*/
@@ -117,8 +124,9 @@
* for this FileSystem
* @param conf the configuration
*/
- public abstract void initialize(URI name, Configuration conf)
- throws IOException;
+ public void initialize(URI name, Configuration conf) throws IOException {
+ statistics = getStatistics(name.getScheme(), getClass());
+ }
/** Returns a URI whose scheme and authority identify this FileSystem.*/
public abstract URI getUri();
@@ -260,7 +268,6 @@
protected FileSystem() {
super(null);
- statistics = getStatistics(this.getClass());
}
/** Check that a Path belongs to this FileSystem. */
@@ -1364,7 +1371,6 @@
}
FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
fs.initialize(uri, conf);
- statsByUriScheme.put(uri.getScheme(), fs.statistics);
return fs;
}
@@ -1474,9 +1480,14 @@
}
public static final class Statistics {
+ private final String scheme;
private AtomicLong bytesRead = new AtomicLong();
private AtomicLong bytesWritten = new AtomicLong();
+ public Statistics(String scheme) {
+ this.scheme = scheme;
+ }
+
/**
* Increment the bytes read in the statistics
* @param newBytes the additional bytes read
@@ -1513,32 +1524,65 @@
return bytesRead + " bytes read and " + bytesWritten +
" bytes written";
}
+
+ /**
+ * Reset the counts of bytes to 0.
+ */
+ public void reset() {
+ bytesWritten.set(0);
+ bytesRead.set(0);
+ }
+
+ /**
+ * Get the uri scheme associated with this statistics object.
+ * @return the schema associated with this set of statistics
+ */
+ public String getScheme() {
+ return scheme;
+ }
}
/**
* Get the Map of Statistics object indexed by URI Scheme.
* @return a Map having a key as URI scheme and value as Statistics object
+ * @deprecated use {@link #getAllStatistics} instead
*/
public static synchronized Map<String, Statistics> getStatistics() {
- return statsByUriScheme;
+ Map<String, Statistics> result = new HashMap<String, Statistics>();
+ for(Statistics stat: statisticsTable.values()) {
+ result.put(stat.getScheme(), stat);
+ }
+ return result;
+ }
+
+ /**
+ * Return the FileSystem classes that have Statistics
+ */
+ public static synchronized List<Statistics> getAllStatistics() {
+ return new ArrayList<Statistics>(statisticsTable.values());
}
/**
* Get the statistics for a particular file system
- * @deprecated Consider using {@link #getStatistics()} instead.
* @param cls the class to lookup
* @return a statistics object
*/
public static synchronized
- Statistics getStatistics(Class<? extends FileSystem> cls) {
+ Statistics getStatistics(String scheme, Class<? extends FileSystem> cls) {
Statistics result = statisticsTable.get(cls);
if (result == null) {
- result = new Statistics();
+ result = new Statistics(scheme);
statisticsTable.put(cls, result);
}
return result;
}
+ public static synchronized void clearStatistics() {
+ for(Statistics stat: statisticsTable.values()) {
+ stat.reset();
+ }
+ }
+
public static synchronized
void printStatistics() throws IOException {
for (Map.Entry<Class<? extends FileSystem>, Statistics> pair:
Modified: hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=763110&r1=763109&r2=763110&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/RawLocalFileSystem.java (original)
+++ hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/RawLocalFileSystem.java Wed Apr 8 05:22:05 2009
@@ -53,7 +53,8 @@
public URI getUri() { return NAME; }
- public void initialize(URI uri, Configuration conf) {
+ public void initialize(URI uri, Configuration conf) throws IOException {
+ super.initialize(uri, conf);
setConf(conf);
}
Modified: hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/ftp/FTPFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/ftp/FTPFileSystem.java?rev=763110&r1=763109&r2=763110&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/ftp/FTPFileSystem.java (original)
+++ hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/ftp/FTPFileSystem.java Wed Apr 8 05:22:05 2009
@@ -56,6 +56,7 @@
@Override
public void initialize(URI uri, Configuration conf) throws IOException { // get
+ super.initialize(uri, conf);
// get host information from uri (overrides info in conf)
String host = uri.getHost();
host = (host == null) ? conf.get("fs.ftp.host", null) : host;
Modified: hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/kfs/KosmosFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/kfs/KosmosFileSystem.java?rev=763110&r1=763109&r2=763110&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/kfs/KosmosFileSystem.java (original)
+++ hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/kfs/KosmosFileSystem.java Wed Apr 8 05:22:05 2009
@@ -59,27 +59,29 @@
}
public void initialize(URI uri, Configuration conf) throws IOException {
- try {
- if (kfsImpl == null) {
- if (uri.getHost() == null) {
- kfsImpl = new KFSImpl(conf.get("fs.kfs.metaServerHost", ""),
- conf.getInt("fs.kfs.metaServerPort", -1),
- statistics);
- } else {
- kfsImpl = new KFSImpl(uri.getHost(), uri.getPort(), statistics);
- }
- }
-
- this.localFs = FileSystem.getLocal(conf);
- this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
- this.workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(this);
- setConf(conf);
-
- } catch (Exception e) {
- e.printStackTrace();
- System.out.println("Unable to initialize KFS");
- System.exit(-1);
+ super.initialize(uri, conf);
+ try {
+ if (kfsImpl == null) {
+ if (uri.getHost() == null) {
+ kfsImpl = new KFSImpl(conf.get("fs.kfs.metaServerHost", ""),
+ conf.getInt("fs.kfs.metaServerPort", -1),
+ statistics);
+ } else {
+ kfsImpl = new KFSImpl(uri.getHost(), uri.getPort(), statistics);
+ }
}
+
+ this.localFs = FileSystem.getLocal(conf);
+ this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
+ this.workingDir = new Path("/user", System.getProperty("user.name")
+ ).makeQualified(this);
+ setConf(conf);
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ System.out.println("Unable to initialize KFS");
+ System.exit(-1);
+ }
}
@Deprecated
Modified: hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/s3/S3FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/s3/S3FileSystem.java?rev=763110&r1=763109&r2=763110&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/s3/S3FileSystem.java (original)
+++ hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/s3/S3FileSystem.java Wed Apr 8 05:22:05 2009
@@ -70,6 +70,7 @@
@Override
public void initialize(URI uri, Configuration conf) throws IOException {
+ super.initialize(uri, conf);
if (store == null) {
store = createDefaultStore(conf);
}
Modified: hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java?rev=763110&r1=763109&r2=763110&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java (original)
+++ hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java Wed Apr 8 05:22:05 2009
@@ -209,6 +209,7 @@
@Override
public void initialize(URI uri, Configuration conf) throws IOException {
+ super.initialize(uri, conf);
if (store == null) {
store = createDefaultStore(conf);
}
Modified: hadoop/core/branches/branch-0.20/src/examples/org/apache/hadoop/examples/Sort.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/examples/org/apache/hadoop/examples/Sort.java?rev=763110&r1=763109&r2=763110&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/examples/org/apache/hadoop/examples/Sort.java (original)
+++ hadoop/core/branches/branch-0.20/src/examples/org/apache/hadoop/examples/Sort.java Wed Apr 8 05:22:05 2009
@@ -51,6 +51,7 @@
* <i>in-dir</i> <i>out-dir</i>
*/
public class Sort<K,V> extends Configured implements Tool {
+ private RunningJob jobResult = null;
static int printUsage() {
System.out.println("sort [-m <maps>] [-r <reduces>] " +
@@ -172,7 +173,7 @@
" with " + num_reduces + " reduces.");
Date startTime = new Date();
System.out.println("Job started: " + startTime);
- JobClient.runJob(jobConf);
+ jobResult = JobClient.runJob(jobConf);
Date end_time = new Date();
System.out.println("Job ended: " + end_time);
System.out.println("The job took " +
@@ -187,4 +188,11 @@
System.exit(res);
}
+ /**
+ * Get the last job that was run using this instance.
+ * @return the results of the last job that was run
+ */
+ public RunningJob getResult() {
+ return jobResult;
+ }
}
Modified: hadoop/core/branches/branch-0.20/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java?rev=763110&r1=763109&r2=763110&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/core/branches/branch-0.20/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java Wed Apr 8 05:22:05 2009
@@ -81,7 +81,7 @@
private int socketTimeout;
private int datanodeWriteTimeout;
final int writePacketSize;
- private FileSystem.Statistics stats;
+ private final FileSystem.Statistics stats;
private int maxBlockAcquireFailures;
@@ -146,7 +146,7 @@
* Create a new DFSClient connected to the default namenode.
*/
public DFSClient(Configuration conf) throws IOException {
- this(NameNode.getAddress(conf), conf);
+ this(NameNode.getAddress(conf), conf, null);
}
/**
@@ -189,8 +189,7 @@
public DFSClient(InetSocketAddress nameNodeAddr,
Configuration conf) throws IOException {
- this(nameNodeAddr, conf,
- FileSystem.getStatistics(DistributedFileSystem.class));
+ this(nameNodeAddr, conf, null);
}
private void checkOpen() throws IOException {
Modified: hadoop/core/branches/branch-0.20/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=763110&r1=763109&r2=763110&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java (original)
+++ hadoop/core/branches/branch-0.20/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java Wed Apr 8 05:22:05 2009
@@ -24,7 +24,6 @@
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.protocol.Block;
@@ -35,7 +34,7 @@
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.DFSClient.DFSOutputStream;
import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.util.*;
+import org.apache.hadoop.util.Progressable;
/****************************************************************
@@ -71,6 +70,7 @@
public URI getUri() { return uri; }
public void initialize(URI uri, Configuration conf) throws IOException {
+ super.initialize(uri, conf);
setConf(conf);
String host = uri.getHost();
Modified: hadoop/core/branches/branch-0.20/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java?rev=763110&r1=763109&r2=763110&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java (original)
+++ hadoop/core/branches/branch-0.20/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java Wed Apr 8 05:22:05 2009
@@ -81,6 +81,7 @@
@Override
public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
setConf(conf);
try {
this.ugi = UnixUserGroupInformation.login(conf, true);
Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Child.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Child.java?rev=763110&r1=763109&r2=763110&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Child.java (original)
+++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Child.java Wed Apr 8 05:22:05 2009
@@ -130,6 +130,9 @@
task = myTask.getTask();
taskid = task.getTaskID();
isCleanup = task.isTaskCleanupTask();
+ // reset the statistics for the task
+ FileSystem.clearStatistics();
+
//create the index file so that the log files
//are viewable immediately
TaskLog.syncLogs(firstTaskid, taskid, isCleanup);
Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java?rev=763110&r1=763109&r2=763110&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java Wed Apr 8 05:22:05 2009
@@ -37,6 +37,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem.Statistics;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.Text;
@@ -629,15 +630,14 @@
new HashMap<String, FileSystemStatisticUpdater>();
private synchronized void updateCounters() {
- for(Map.Entry<String, FileSystem.Statistics> entry :
- FileSystem.getStatistics().entrySet()) {
- String uriScheme = entry.getKey();
+ for(Statistics stat: FileSystem.getAllStatistics()) {
+ String uriScheme = stat.getScheme();
FileSystemStatisticUpdater updater = statisticUpdaters.get(uriScheme);
if(updater==null) {//new FileSystem has been found in the cache
- updater = new FileSystemStatisticUpdater(uriScheme, entry.getValue());
+ updater = new FileSystemStatisticUpdater(uriScheme, stat);
statisticUpdaters.put(uriScheme, updater);
}
- updater.updateCounters();
+ updater.updateCounters();
}
}
Modified: hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/io/TestSequenceFile.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/io/TestSequenceFile.java?rev=763110&r1=763109&r2=763110&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/io/TestSequenceFile.java (original)
+++ hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/io/TestSequenceFile.java Wed Apr 8 05:22:05 2009
@@ -390,9 +390,7 @@
public void testClose() throws IOException {
Configuration conf = new Configuration();
- LocalFileSystem fs = new LocalFileSystem();
- fs.setConf(conf);
- fs.getRawFileSystem().setConf(conf);
+ LocalFileSystem fs = FileSystem.getLocal(conf);
// create a sequence file 1
Path path1 = new Path(System.getProperty("test.build.data",".")+"/test1.seq");
Modified: hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java?rev=763110&r1=763109&r2=763110&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java (original)
+++ hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java Wed Apr 8 05:22:05 2009
@@ -94,7 +94,17 @@
String[] sortArgs = {sortInput.toString(), sortOutput.toString()};
// Run Sort
- assertEquals(ToolRunner.run(job, new Sort(), sortArgs), 0);
+ Sort sort = new Sort();
+ assertEquals(ToolRunner.run(job, sort, sortArgs), 0);
+ Counters counters = sort.getResult().getCounters();
+ long mapInput = counters.findCounter(Task.Counter.MAP_INPUT_BYTES
+ ).getValue();
+ long hdfsRead = counters.findCounter(Task.FILESYSTEM_COUNTER_GROUP,
+ "HDFS_BYTES_READ").getValue();
+ // the hdfs read should be between 100% and 110% of the map input bytes
+ assertTrue("map input = " + mapInput + ", hdfs read = " + hdfsRead,
+ (hdfsRead < (mapInput * 1.1)) &&
+ (hdfsRead > mapInput));
}
private static void runSortValidator(JobConf job,
@@ -139,8 +149,6 @@
job.setNumReduceTasks(0);
RunningJob result = JobClient.runJob(job);
long uses = result.getCounters().findCounter("jvm", "use").getValue();
- System.out.println("maps = " + job.getNumMapTasks());
- System.out.println(result.getCounters());
int maps = job.getNumMapTasks();
if (reuse) {
assertTrue("maps = " + maps + ", uses = " + uses, maps < uses);