You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by br...@apache.org on 2013/11/07 22:49:14 UTC
svn commit: r1539834 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop:
mount/MountdBase.java nfs/nfs3/Nfs3Base.java nfs/nfs3/Nfs3Constant.java
nfs/nfs3/request/WRITE3Request.java oncrpc/RpcProgram.java
Author: brandonli
Date: Thu Nov 7 21:49:13 2013
New Revision: 1539834
URL: http://svn.apache.org/r1539834
Log:
HDFS-5364. Add OpenFileCtx cache. Contributed by Brandon Li
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java
hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java
hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Constant.java
hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/WRITE3Request.java
hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java?rev=1539834&r1=1539833&r2=1539834&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java Thu Nov 7 21:49:13 2013
@@ -52,6 +52,7 @@ abstract public class MountdBase {
private void startUDPServer() {
SimpleUdpServer udpServer = new SimpleUdpServer(rpcProgram.getPort(),
rpcProgram, 1);
+ rpcProgram.startDaemons();
udpServer.run();
}
@@ -59,6 +60,7 @@ abstract public class MountdBase {
private void startTCPServer() {
SimpleTcpServer tcpServer = new SimpleTcpServer(rpcProgram.getPort(),
rpcProgram, 1);
+ rpcProgram.startDaemons();
tcpServer.run();
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java?rev=1539834&r1=1539833&r2=1539834&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java Thu Nov 7 21:49:13 2013
@@ -20,7 +20,6 @@ package org.apache.hadoop.nfs.nfs3;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mount.MountdBase;
import org.apache.hadoop.oncrpc.RpcProgram;
import org.apache.hadoop.oncrpc.SimpleTcpServer;
import org.apache.hadoop.portmap.PortmapMapping;
@@ -32,34 +31,27 @@ import org.apache.hadoop.portmap.Portmap
*/
public abstract class Nfs3Base {
public static final Log LOG = LogFactory.getLog(Nfs3Base.class);
- private final MountdBase mountd;
private final RpcProgram rpcProgram;
private final int nfsPort;
-
- public MountdBase getMountBase() {
- return mountd;
- }
-
+
public RpcProgram getRpcProgram() {
return rpcProgram;
}
- protected Nfs3Base(MountdBase mountd, RpcProgram program, Configuration conf) {
- this.mountd = mountd;
- this.rpcProgram = program;
+ protected Nfs3Base(RpcProgram rpcProgram, Configuration conf) {
+ this.rpcProgram = rpcProgram;
this.nfsPort = conf.getInt("nfs3.server.port", Nfs3Constant.PORT);
- LOG.info("NFS server port set to: "+nfsPort);
+ LOG.info("NFS server port set to: " + nfsPort);
}
- protected Nfs3Base(MountdBase mountd, RpcProgram program) {
- this.mountd = mountd;
- this.rpcProgram = program;
+ protected Nfs3Base(RpcProgram rpcProgram) {
+ this.rpcProgram = rpcProgram;
this.nfsPort = Nfs3Constant.PORT;
}
public void start(boolean register) {
- mountd.start(register); // Start mountd
startTCPServer(); // Start TCP server
+
if (register) {
rpcProgram.register(PortmapMapping.TRANSPORT_TCP);
}
@@ -68,6 +60,7 @@ public abstract class Nfs3Base {
private void startTCPServer() {
SimpleTcpServer tcpServer = new SimpleTcpServer(nfsPort,
rpcProgram, 0);
+ rpcProgram.startDaemons();
tcpServer.run();
}
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Constant.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Constant.java?rev=1539834&r1=1539833&r2=1539834&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Constant.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Constant.java Thu Nov 7 21:49:13 2013
@@ -205,6 +205,11 @@ public class Nfs3Constant {
public static final String FILE_DUMP_DIR_DEFAULT = "/tmp/.hdfs-nfs";
public static final String ENABLE_FILE_DUMP_KEY = "dfs.nfs3.enableDump";
public static final boolean ENABLE_FILE_DUMP_DEFAULT = true;
+ public static final String MAX_OPEN_FILES = "dfs.nfs3.max.open.files";
+ public static final int MAX_OPEN_FILES_DEFAULT = 256;
+ public static final String OUTPUT_STREAM_TIMEOUT = "dfs.nfs3.stream.timeout";
+ public static final long OUTPUT_STREAM_TIMEOUT_DEFAULT = 10 * 60 * 1000; // 10 minutes
+ public static final long OUTPUT_STREAM_TIMEOUT_MIN_DEFAULT = 10 * 1000; //10 seconds
public final static String UNKNOWN_USER = "nobody";
public final static String UNKNOWN_GROUP = "nobody";
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/WRITE3Request.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/WRITE3Request.java?rev=1539834&r1=1539833&r2=1539834&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/WRITE3Request.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/WRITE3Request.java Thu Nov 7 21:49:13 2013
@@ -83,4 +83,10 @@ public class WRITE3Request extends Reque
xdr.writeInt(count);
xdr.writeFixedOpaque(data.array(), count);
}
+
+ @Override
+ public String toString() {
+ return String.format("fileId: %d offset: %d count: %d stableHow: %s",
+ handle.getFileId(), offset, count, stableHow.name());
+ }
}
\ No newline at end of file
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java?rev=1539834&r1=1539833&r2=1539834&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java Thu Nov 7 21:49:13 2013
@@ -100,6 +100,9 @@ public abstract class RpcProgram extends
}
}
+ // Start extra daemons
+ public void startDaemons() {}
+
@Override
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e)
throws Exception {