You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2009/04/04 01:21:46 UTC
svn commit: r761838 - in /hadoop/core/trunk/src:
contrib/streaming/src/test/org/apache/hadoop/typedbytes/
core/org/apache/hadoop/fs/ core/org/apache/hadoop/fs/ftp/
core/org/apache/hadoop/fs/kfs/ core/org/apache/hadoop/fs/s3/
core/org/apache/hadoop/fs/s...
Author: omalley
Date: Fri Apr 3 23:21:46 2009
New Revision: 761838
URL: http://svn.apache.org/viewvc?rev=761838&view=rev
Log:
HADOOP-5450. Revert accidental inclusion.
Modified:
hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/typedbytes/TestIO.java
hadoop/core/trunk/src/core/org/apache/hadoop/fs/FileSystem.java
hadoop/core/trunk/src/core/org/apache/hadoop/fs/RawLocalFileSystem.java
hadoop/core/trunk/src/core/org/apache/hadoop/fs/ftp/FTPFileSystem.java
hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/KosmosFileSystem.java
hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3/S3FileSystem.java
hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java
Modified: hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/typedbytes/TestIO.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/typedbytes/TestIO.java?rev=761838&r1=761837&r2=761838&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/typedbytes/TestIO.java (original)
+++ hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/typedbytes/TestIO.java Fri Apr 3 23:21:46 2009
@@ -40,7 +40,6 @@
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.MapWritable;
-import org.apache.hadoop.io.ObjectWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.VIntWritable;
import org.apache.hadoop.io.VLongWritable;
@@ -81,7 +80,6 @@
(byte) 123, true, 12345, 123456789L, (float) 1.2, 1.234,
"random string", vector, list, map
};
- byte[] appSpecificBytes = new byte[] { 1, 2, 3 };
FileOutputStream ostream = new FileOutputStream(tmpfile);
DataOutputStream dostream = new DataOutputStream(ostream);
@@ -89,7 +87,6 @@
for (Object obj : objects) {
out.write(obj);
}
- out.writeBytes(appSpecificBytes, 100);
dostream.close();
ostream.close();
@@ -99,7 +96,6 @@
for (Object obj : objects) {
assertEquals(obj, in.read());
}
- assertEquals(new Buffer(appSpecificBytes), in.read());
distream.close();
istream.close();
@@ -118,9 +114,6 @@
dis = new DataInputStream(bais);
assertEquals(obj, (new TypedBytesInput(dis)).read());
}
- byte[] rawBytes = in.readRaw();
- assertEquals(new Buffer(appSpecificBytes),
- new Buffer(rawBytes, 5, rawBytes.length - 5));
distream.close();
istream.close();
}
@@ -171,8 +164,7 @@
new ByteWritable((byte) 123), new BooleanWritable(true),
new VIntWritable(12345), new VLongWritable(123456789L),
new FloatWritable((float) 1.2), new DoubleWritable(1.234),
- new Text("random string"),
- new ObjectWritable("test")
+ new Text("random string")
};
TypedBytesWritable tbw = new TypedBytesWritable();
tbw.setValue("typed bytes text");
@@ -209,7 +201,7 @@
TypedBytesWritableInput in = new TypedBytesWritableInput(distream);
for (Writable w : writables) {
- assertEquals(w.toString(), in.read().toString());
+ assertEquals(w, in.read());
}
assertEquals(tbw.getValue().toString(), in.read().toString());
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/FileSystem.java?rev=761838&r1=761837&r2=761838&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/FileSystem.java Fri Apr 3 23:21:46 2009
@@ -23,7 +23,6 @@
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.Iterator;
@@ -80,6 +79,10 @@
statisticsTable =
new IdentityHashMap<Class<? extends FileSystem>, Statistics>();
+ /** Recording statistics per FileSystem URI scheme */
+ private static final Map<String, Statistics> statsByUriScheme =
+ new HashMap<String, Statistics>();
+
/**
* The statistics for this file system.
*/
@@ -125,9 +128,8 @@
* for this FileSystem
* @param conf the configuration
*/
- public void initialize(URI name, Configuration conf) throws IOException {
- statistics = getStatistics(name.getScheme(), getClass());
- }
+ public abstract void initialize(URI name, Configuration conf)
+ throws IOException;
/** Returns a URI whose scheme and authority identify this FileSystem.*/
public abstract URI getUri();
@@ -301,6 +303,7 @@
protected FileSystem() {
super(null);
+ statistics = getStatistics(this.getClass());
}
/** Check that a Path belongs to this FileSystem. */
@@ -1404,6 +1407,7 @@
}
FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
fs.initialize(uri, conf);
+ statsByUriScheme.put(uri.getScheme(), fs.statistics);
return fs;
}
@@ -1533,14 +1537,9 @@
}
public static final class Statistics {
- private final String scheme;
private AtomicLong bytesRead = new AtomicLong();
private AtomicLong bytesWritten = new AtomicLong();
- public Statistics(String scheme) {
- this.scheme = scheme;
- }
-
/**
* Increment the bytes read in the statistics
* @param newBytes the additional bytes read
@@ -1577,65 +1576,32 @@
return bytesRead + " bytes read and " + bytesWritten +
" bytes written";
}
-
- /**
- * Reset the counts of bytes to 0.
- */
- public void reset() {
- bytesWritten.set(0);
- bytesRead.set(0);
- }
-
- /**
- * Get the uri scheme associated with this statistics object.
- * @return the uri scheme
- */
- public String getScheme() {
- return scheme;
- }
}
/**
* Get the Map of Statistics object indexed by URI Scheme.
* @return a Map having a key as URI scheme and value as Statistics object
- * @deprecated use @link #getFileSystemClasses instead
*/
public static synchronized Map<String, Statistics> getStatistics() {
- Map<String, Statistics> result = new HashMap<String, Statistics>();
- for(Statistics stat: statisticsTable.values()) {
- result.put(stat.getScheme(), stat);
- }
- return result;
- }
-
- /**
- * Return the FileSystem classes that have Statistics
- */
- public static synchronized List<Statistics> getAllStatistics() {
- return new ArrayList<Statistics>(statisticsTable.values());
+ return statsByUriScheme;
}
/**
* Get the statistics for a particular file system
+ * @deprecated Consider using {@link #getStatistics()} instead.
* @param cls the class to lookup
* @return a statistics object
*/
public static synchronized
- Statistics getStatistics(String scheme, Class<? extends FileSystem> cls) {
+ Statistics getStatistics(Class<? extends FileSystem> cls) {
Statistics result = statisticsTable.get(cls);
if (result == null) {
- result = new Statistics(scheme);
+ result = new Statistics();
statisticsTable.put(cls, result);
}
return result;
}
- public static synchronized void clearStatistics() {
- for(Statistics stat: statisticsTable.values()) {
- stat.reset();
- }
- }
-
public static synchronized
void printStatistics() throws IOException {
for (Map.Entry<Class<? extends FileSystem>, Statistics> pair:
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=761838&r1=761837&r2=761838&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/RawLocalFileSystem.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/RawLocalFileSystem.java Fri Apr 3 23:21:46 2009
@@ -59,8 +59,7 @@
public URI getUri() { return NAME; }
- public void initialize(URI uri, Configuration conf) throws IOException {
- super.initialize(uri, conf);
+ public void initialize(URI uri, Configuration conf) {
setConf(conf);
}
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/ftp/FTPFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/ftp/FTPFileSystem.java?rev=761838&r1=761837&r2=761838&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/ftp/FTPFileSystem.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/ftp/FTPFileSystem.java Fri Apr 3 23:21:46 2009
@@ -56,7 +56,6 @@
@Override
public void initialize(URI uri, Configuration conf) throws IOException { // get
- super.initialize(uri, conf);
// get host information from uri (overrides info in conf)
String host = uri.getHost();
host = (host == null) ? conf.get("fs.ftp.host", null) : host;
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/KosmosFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/KosmosFileSystem.java?rev=761838&r1=761837&r2=761838&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/KosmosFileSystem.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/kfs/KosmosFileSystem.java Fri Apr 3 23:21:46 2009
@@ -62,29 +62,27 @@
@Override
public void initialize(URI uri, Configuration conf) throws IOException {
- super.initialize(uri, conf);
- try {
- if (kfsImpl == null) {
- if (uri.getHost() == null) {
- kfsImpl = new KFSImpl(conf.get("fs.kfs.metaServerHost", ""),
- conf.getInt("fs.kfs.metaServerPort", -1),
- statistics);
- } else {
- kfsImpl = new KFSImpl(uri.getHost(), uri.getPort(), statistics);
- }
+ try {
+ if (kfsImpl == null) {
+ if (uri.getHost() == null) {
+ kfsImpl = new KFSImpl(conf.get("fs.kfs.metaServerHost", ""),
+ conf.getInt("fs.kfs.metaServerPort", -1),
+ statistics);
+ } else {
+ kfsImpl = new KFSImpl(uri.getHost(), uri.getPort(), statistics);
+ }
+ }
+
+ this.localFs = FileSystem.getLocal(conf);
+ this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
+ this.workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(this);
+ setConf(conf);
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ System.out.println("Unable to initialize KFS");
+ System.exit(-1);
}
-
- this.localFs = FileSystem.getLocal(conf);
- this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
- this.workingDir = new Path("/user", System.getProperty("user.name")
- ).makeQualified(this);
- setConf(conf);
-
- } catch (Exception e) {
- e.printStackTrace();
- System.out.println("Unable to initialize KFS");
- System.exit(-1);
- }
}
@Override
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3/S3FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3/S3FileSystem.java?rev=761838&r1=761837&r2=761838&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3/S3FileSystem.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3/S3FileSystem.java Fri Apr 3 23:21:46 2009
@@ -70,7 +70,6 @@
@Override
public void initialize(URI uri, Configuration conf) throws IOException {
- super.initialize(uri, conf);
if (store == null) {
store = createDefaultStore(conf);
}
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java?rev=761838&r1=761837&r2=761838&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java Fri Apr 3 23:21:46 2009
@@ -209,7 +209,6 @@
@Override
public void initialize(URI uri, Configuration conf) throws IOException {
- super.initialize(uri, conf);
if (store == null) {
store = createDefaultStore(conf);
}
Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java?rev=761838&r1=761837&r2=761838&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java Fri Apr 3 23:21:46 2009
@@ -80,7 +80,7 @@
private int socketTimeout;
private int datanodeWriteTimeout;
final int writePacketSize;
- private final FileSystem.Statistics stats;
+ private FileSystem.Statistics stats;
private int maxBlockAcquireFailures;
@@ -145,7 +145,7 @@
* Create a new DFSClient connected to the default namenode.
*/
public DFSClient(Configuration conf) throws IOException {
- this(NameNode.getAddress(conf), conf, null);
+ this(NameNode.getAddress(conf), conf);
}
/**
@@ -188,7 +188,8 @@
public DFSClient(InetSocketAddress nameNodeAddr,
Configuration conf) throws IOException {
- this(nameNodeAddr, conf, null);
+ this(nameNodeAddr, conf,
+ FileSystem.getStatistics(DistributedFileSystem.class));
}
private void checkOpen() throws IOException {
Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=761838&r1=761837&r2=761838&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java Fri Apr 3 23:21:46 2009
@@ -24,6 +24,7 @@
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.protocol.Block;
@@ -34,7 +35,7 @@
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.DFSClient.DFSOutputStream;
import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.util.*;
/****************************************************************
@@ -69,7 +70,6 @@
@Override
public void initialize(URI uri, Configuration conf) throws IOException {
- super.initialize(uri, conf);
setConf(conf);
String host = uri.getHost();
Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java?rev=761838&r1=761837&r2=761838&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java Fri Apr 3 23:21:46 2009
@@ -81,7 +81,6 @@
@Override
public void initialize(URI name, Configuration conf) throws IOException {
- super.initialize(name, conf);
setConf(conf);
try {
this.ugi = UnixUserGroupInformation.login(conf, true);
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java?rev=761838&r1=761837&r2=761838&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Child.java Fri Apr 3 23:21:46 2009
@@ -130,9 +130,7 @@
task = myTask.getTask();
taskid = task.getTaskID();
isCleanup = task.isTaskCleanupTask();
- // reset the statistics for the task
- FileSystem.clearStatistics();
-
+
//create the index file so that the log files
//are viewable immediately
TaskLog.syncLogs(firstTaskid, taskid, isCleanup);
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java?rev=761838&r1=761837&r2=761838&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/Task.java Fri Apr 3 23:21:46 2009
@@ -37,7 +37,6 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileSystem.Statistics;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.Text;
@@ -635,14 +634,15 @@
new HashMap<String, FileSystemStatisticUpdater>();
private synchronized void updateCounters() {
- for(Statistics stat: FileSystem.getAllStatistics()) {
- String uriScheme = stat.getScheme();
+ for(Map.Entry<String, FileSystem.Statistics> entry :
+ FileSystem.getStatistics().entrySet()) {
+ String uriScheme = entry.getKey();
FileSystemStatisticUpdater updater = statisticUpdaters.get(uriScheme);
if(updater==null) {//new FileSystem has been found in the cache
- updater = new FileSystemStatisticUpdater(uriScheme, stat);
+ updater = new FileSystemStatisticUpdater(uriScheme, entry.getValue());
statisticUpdaters.put(uriScheme, updater);
}
- updater.updateCounters();
+ updater.updateCounters();
}
}