You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by vi...@apache.org on 2014/06/07 18:29:17 UTC
svn commit: r1601151 [1/2] - in
/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project:
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/
hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/
hadoop-hdfs-nfs/src/main/ja...
Author: vinayakumarb
Date: Sat Jun 7 16:29:10 2014
New Revision: 1601151
URL: http://svn.apache.org/r1601151
Log:
Merged changes from trunk
Added:
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/
- copied from r1601150, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyBlockManagement.java
- copied unchanged from r1601150, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyBlockManagement.java
Modified:
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/FileSystemAccessService.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/Mountd.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/PrivilegedNfsGatewayStarter.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestMountd.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestOutOfOrderWrite.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestExportsTable.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestOpenFileCtxCache.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReader.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestAvailableSpaceVolumeChoosingPolicy.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
Propchange: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1598456-1601150
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/FileSystemAccessService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/FileSystemAccessService.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/FileSystemAccessService.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/FileSystemAccessService.java Sat Jun 7 16:29:10 2014
@@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.lib.server.BaseService;
import org.apache.hadoop.lib.server.ServiceException;
import org.apache.hadoop.lib.service.FileSystemAccess;
@@ -395,6 +396,10 @@ public class FileSystemAccessService ext
Configuration conf = new Configuration(true);
ConfigurationUtils.copy(serviceHadoopConf, conf);
conf.setBoolean(FILE_SYSTEM_SERVICE_CREATED, true);
+
+ // Force-clear server-side umask to make HttpFS match WebHDFS behavior
+ conf.set(FsPermission.UMASK_LABEL, "000");
+
return conf;
}
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java Sat Jun 7 16:29:10 2014
@@ -231,6 +231,105 @@ public class TestHttpFSServer extends HF
reader.close();
}
+ /**
+ * Talks to the http interface to create a file.
+ *
+ * @param filename The file to create
+ * @param perms The permission field, if any (may be null)
+ * @throws Exception
+ */
+ private void createWithHttp ( String filename, String perms )
+ throws Exception {
+ String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
+ String pathOps;
+ if ( perms == null ) {
+ pathOps = MessageFormat.format(
+ "/webhdfs/v1/{0}?user.name={1}&op=CREATE",
+ filename, user);
+ } else {
+ pathOps = MessageFormat.format(
+ "/webhdfs/v1/{0}?user.name={1}&permission={2}&op=CREATE",
+ filename, user, perms);
+ }
+ URL url = new URL(TestJettyHelper.getJettyURL(), pathOps);
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.addRequestProperty("Content-Type", "application/octet-stream");
+ conn.setRequestMethod("PUT");
+ conn.connect();
+ Assert.assertEquals(HttpURLConnection.HTTP_CREATED, conn.getResponseCode());
+ }
+
+ /**
+ * Talks to the http interface to get the json output of the GETFILESTATUS
+ * command on the given file.
+ *
+ * @param filename The file to query.
+ * @return A string containing the JSON output describing the file.
+ * @throws Exception
+ */
+ private String getFileStatus ( String filename ) throws Exception {
+ String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
+ String pathOps = MessageFormat.format(
+ "/webhdfs/v1/{0}?user.name={1}&op=GETFILESTATUS",
+ filename, user);
+ URL url = new URL(TestJettyHelper.getJettyURL(), pathOps);
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.connect();
+ Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+
+ BufferedReader reader =
+ new BufferedReader(new InputStreamReader(conn.getInputStream()));
+
+ return reader.readLine();
+ }
+
+ /**
+ * Given the JSON output from the GETFILESTATUS call, return the
+ * 'permission' value.
+ *
+ * @param statusJson JSON from GETFILESTATUS
+ * @return The value of 'permission' in statusJson
+ * @throws Exception
+ */
+ private String getPerms ( String statusJson ) throws Exception {
+ JSONParser parser = new JSONParser();
+ JSONObject jsonObject = (JSONObject) parser.parse(statusJson);
+ JSONObject details = (JSONObject) jsonObject.get("FileStatus");
+ return (String) details.get("permission");
+ }
+
+ /**
+ * Validate that files are created with 755 permissions when no
+ * 'permissions' attribute is specified, and when 'permissions'
+ * is specified, that value is honored.
+ */
+ @Test
+ @TestDir
+ @TestJetty
+ @TestHdfs
+ public void testPerms() throws Exception {
+ createHttpFSServer(false);
+
+ FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
+ fs.mkdirs(new Path("/perm"));
+
+ createWithHttp("/perm/none", null);
+ String statusJson = getFileStatus("/perm/none");
+ Assert.assertTrue("755".equals(getPerms(statusJson)));
+
+ createWithHttp("/perm/p-777", "777");
+ statusJson = getFileStatus("/perm/p-777");
+ Assert.assertTrue("777".equals(getPerms(statusJson)));
+
+ createWithHttp("/perm/p-654", "654");
+ statusJson = getFileStatus("/perm/p-654");
+ Assert.assertTrue("654".equals(getPerms(statusJson)));
+
+ createWithHttp("/perm/p-321", "321");
+ statusJson = getFileStatus("/perm/p-321");
+ Assert.assertTrue("321".equals(getPerms(statusJson)));
+ }
+
@Test
@TestDir
@TestJetty
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/Mountd.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/Mountd.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/Mountd.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/Mountd.java Sat Jun 7 16:29:10 2014
@@ -20,7 +20,7 @@ package org.apache.hadoop.hdfs.nfs.mount
import java.io.IOException;
import java.net.DatagramSocket;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.mount.MountdBase;
/**
@@ -32,13 +32,13 @@ import org.apache.hadoop.mount.MountdBas
*/
public class Mountd extends MountdBase {
- public Mountd(Configuration config, DatagramSocket registrationSocket,
+ public Mountd(NfsConfiguration config, DatagramSocket registrationSocket,
boolean allowInsecurePorts) throws IOException {
super(new RpcProgramMountd(config, registrationSocket, allowInsecurePorts));
}
public static void main(String[] args) throws IOException {
- Configuration config = new Configuration();
+ NfsConfiguration config = new NfsConfiguration();
Mountd mountd = new Mountd(config, null, true);
mountd.start(true);
}
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java Sat Jun 7 16:29:10 2014
@@ -16,9 +16,6 @@
* limitations under the License.
*/
package org.apache.hadoop.hdfs.nfs.mount;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NFS_KEYTAB_FILE_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NFS_KERBEROS_PRINCIPAL_KEY;
-
import java.io.IOException;
import java.net.DatagramSocket;
import java.net.InetAddress;
@@ -29,8 +26,9 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.mount.MountEntry;
@@ -39,7 +37,6 @@ import org.apache.hadoop.mount.MountResp
import org.apache.hadoop.nfs.AccessPrivilege;
import org.apache.hadoop.nfs.NfsExports;
import org.apache.hadoop.nfs.nfs3.FileHandle;
-import org.apache.hadoop.nfs.nfs3.Nfs3Constant;
import org.apache.hadoop.nfs.nfs3.Nfs3Status;
import org.apache.hadoop.oncrpc.RpcAcceptedReply;
import org.apache.hadoop.oncrpc.RpcCall;
@@ -66,9 +63,7 @@ public class RpcProgramMountd extends Rp
public static final int VERSION_1 = 1;
public static final int VERSION_2 = 2;
public static final int VERSION_3 = 3;
- public static final int PORT = 4242;
- // Need DFSClient for branch-1 to get ExtendedHdfsFileStatus
private final DFSClient dfsClient;
/** Synchronized list */
@@ -79,19 +74,22 @@ public class RpcProgramMountd extends Rp
private final NfsExports hostsMatcher;
- public RpcProgramMountd(Configuration config, DatagramSocket registrationSocket,
- boolean allowInsecurePorts) throws IOException {
+ public RpcProgramMountd(NfsConfiguration config,
+ DatagramSocket registrationSocket, boolean allowInsecurePorts)
+ throws IOException {
// Note that RPC cache is not enabled
- super("mountd", "localhost", config.getInt("nfs3.mountd.port", PORT),
- PROGRAM, VERSION_1, VERSION_3, registrationSocket, allowInsecurePorts);
+ super("mountd", "localhost", config.getInt(
+ NfsConfigKeys.DFS_NFS_MOUNTD_PORT_KEY,
+ NfsConfigKeys.DFS_NFS_MOUNTD_PORT_DEFAULT), PROGRAM, VERSION_1,
+ VERSION_3, registrationSocket, allowInsecurePorts);
exports = new ArrayList<String>();
- exports.add(config.get(Nfs3Constant.EXPORT_POINT,
- Nfs3Constant.EXPORT_POINT_DEFAULT));
+ exports.add(config.get(NfsConfigKeys.DFS_NFS_EXPORT_POINT_KEY,
+ NfsConfigKeys.DFS_NFS_EXPORT_POINT_DEFAULT));
this.hostsMatcher = NfsExports.getInstance(config);
this.mounts = Collections.synchronizedList(new ArrayList<MountEntry>());
UserGroupInformation.setConfiguration(config);
- SecurityUtil.login(config, DFS_NFS_KEYTAB_FILE_KEY,
- DFS_NFS_KERBEROS_PRINCIPAL_KEY);
+ SecurityUtil.login(config, NfsConfigKeys.DFS_NFS_KEYTAB_FILE_KEY,
+ NfsConfigKeys.DFS_NFS_KERBEROS_PRINCIPAL_KEY);
this.dfsClient = new DFSClient(NameNode.getAddress(config), config);
}
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java Sat Jun 7 16:29:10 2014
@@ -30,10 +30,10 @@ import java.util.concurrent.TimeUnit;
import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSInputStream;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.security.UserGroupInformation;
@@ -72,7 +72,7 @@ class DFSClientCache {
final static int DEFAULT_DFS_INPUTSTREAM_CACHE_SIZE = 1024;
final static int DEFAULT_DFS_INPUTSTREAM_CACHE_TTL = 10 * 60;
- private final Configuration config;
+ private final NfsConfiguration config;
private static class DFSInputStreamCaheKey {
final String userId;
@@ -99,11 +99,11 @@ class DFSClientCache {
}
}
- DFSClientCache(Configuration config) {
+ DFSClientCache(NfsConfiguration config) {
this(config, DEFAULT_DFS_CLIENT_CACHE_SIZE);
}
- DFSClientCache(Configuration config, int clientCache) {
+ DFSClientCache(NfsConfiguration config, int clientCache) {
this.config = config;
this.clientCache = CacheBuilder.newBuilder()
.maximumSize(clientCache)
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java Sat Jun 7 16:29:10 2014
@@ -20,8 +20,8 @@ package org.apache.hadoop.hdfs.nfs.nfs3;
import java.io.IOException;
import java.net.DatagramSocket;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.hdfs.nfs.mount.Mountd;
import org.apache.hadoop.nfs.nfs3.Nfs3Base;
import org.apache.hadoop.util.StringUtils;
@@ -36,16 +36,11 @@ import com.google.common.annotations.Vis
public class Nfs3 extends Nfs3Base {
private Mountd mountd;
- static {
- Configuration.addDefaultResource("hdfs-default.xml");
- Configuration.addDefaultResource("hdfs-site.xml");
- }
-
- public Nfs3(Configuration conf) throws IOException {
+ public Nfs3(NfsConfiguration conf) throws IOException {
this(conf, null, true);
}
- public Nfs3(Configuration conf, DatagramSocket registrationSocket,
+ public Nfs3(NfsConfiguration conf, DatagramSocket registrationSocket,
boolean allowInsecurePorts) throws IOException {
super(new RpcProgramNfs3(conf, registrationSocket, allowInsecurePorts), conf);
mountd = new Mountd(conf, registrationSocket, allowInsecurePorts);
@@ -64,11 +59,11 @@ public class Nfs3 extends Nfs3Base {
static void startService(String[] args,
DatagramSocket registrationSocket) throws IOException {
StringUtils.startupShutdownMessage(Nfs3.class, args, LOG);
- Configuration conf = new Configuration();
+ NfsConfiguration conf = new NfsConfiguration();
boolean allowInsecurePorts = conf.getBoolean(
- DFSConfigKeys.DFS_NFS_ALLOW_INSECURE_PORTS_KEY,
- DFSConfigKeys.DFS_NFS_ALLOW_INSECURE_PORTS_DEFAULT);
- final Nfs3 nfsServer = new Nfs3(new Configuration(), registrationSocket,
+ NfsConfigKeys.DFS_NFS_ALLOW_INSECURE_PORTS_KEY,
+ NfsConfigKeys.DFS_NFS_ALLOW_INSECURE_PORTS_DEFAULT);
+ final Nfs3 nfsServer = new Nfs3(conf, registrationSocket,
allowInsecurePorts);
nfsServer.startServiceInternal(true);
}
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java Sat Jun 7 16:29:10 2014
@@ -37,6 +37,7 @@ import org.apache.hadoop.fs.FSDataInputS
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
import org.apache.hadoop.hdfs.nfs.nfs3.WriteCtx.DataState;
import org.apache.hadoop.io.BytesWritable.Comparator;
import org.apache.hadoop.io.IOUtils;
@@ -823,7 +824,7 @@ class OpenFileCtx {
*/
public synchronized boolean streamCleanup(long fileId, long streamTimeout) {
Preconditions
- .checkState(streamTimeout >= Nfs3Constant.OUTPUT_STREAM_TIMEOUT_MIN_DEFAULT);
+ .checkState(streamTimeout >= NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_MIN_DEFAULT);
if (!activeState) {
return true;
}
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java Sat Jun 7 16:29:10 2014
@@ -24,9 +24,9 @@ import java.util.concurrent.ConcurrentMa
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.nfs.nfs3.FileHandle;
-import org.apache.hadoop.nfs.nfs3.Nfs3Constant;
import org.apache.hadoop.util.Daemon;
import org.apache.hadoop.util.Time;
@@ -48,9 +48,9 @@ class OpenFileCtxCache {
private final long streamTimeout;
private final StreamMonitor streamMonitor;
- OpenFileCtxCache(Configuration config, long streamTimeout) {
- maxStreams = config.getInt(Nfs3Constant.MAX_OPEN_FILES,
- Nfs3Constant.MAX_OPEN_FILES_DEFAULT);
+ OpenFileCtxCache(NfsConfiguration config, long streamTimeout) {
+ maxStreams = config.getInt(NfsConfigKeys.DFS_NFS_MAX_OPEN_FILES_KEY,
+ NfsConfigKeys.DFS_NFS_MAX_OPEN_FILES_DEFAULT);
LOG.info("Maximum open streams is " + maxStreams);
this.streamTimeout = streamTimeout;
streamMonitor = new StreamMonitor();
@@ -102,7 +102,7 @@ class OpenFileCtxCache {
} else {
long idleTime = Time.monotonicNow()
- idlest.getValue().getLastAccessTime();
- if (idleTime < Nfs3Constant.OUTPUT_STREAM_TIMEOUT_MIN_DEFAULT) {
+ if (idleTime < NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_MIN_DEFAULT) {
if (LOG.isDebugEnabled()) {
LOG.debug("idlest stream's idle time:" + idleTime);
}
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/PrivilegedNfsGatewayStarter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/PrivilegedNfsGatewayStarter.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/PrivilegedNfsGatewayStarter.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/PrivilegedNfsGatewayStarter.java Sat Jun 7 16:29:10 2014
@@ -21,9 +21,8 @@ import java.net.InetSocketAddress;
import org.apache.commons.daemon.Daemon;
import org.apache.commons.daemon.DaemonContext;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
/**
* This class is used to allow the initial registration of the NFS gateway with
@@ -42,12 +41,12 @@ public class PrivilegedNfsGatewayStarter
@Override
public void init(DaemonContext context) throws Exception {
System.err.println("Initializing privileged NFS client socket...");
- Configuration conf = new HdfsConfiguration();
- int clientPort = conf.getInt(DFSConfigKeys.DFS_NFS_REGISTRATION_PORT_KEY,
- DFSConfigKeys.DFS_NFS_REGISTRATION_PORT_DEFAULT);
+ NfsConfiguration conf = new NfsConfiguration();
+ int clientPort = conf.getInt(NfsConfigKeys.DFS_NFS_REGISTRATION_PORT_KEY,
+ NfsConfigKeys.DFS_NFS_REGISTRATION_PORT_DEFAULT);
if (clientPort < 1 || clientPort > 1023) {
throw new RuntimeException("Must start privileged NFS server with '" +
- DFSConfigKeys.DFS_NFS_REGISTRATION_PORT_KEY + "' configured to a " +
+ NfsConfigKeys.DFS_NFS_REGISTRATION_PORT_KEY + "' configured to a " +
"privileged port.");
}
registrationSocket = new DatagramSocket(
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java Sat Jun 7 16:29:10 2014
@@ -28,7 +28,6 @@ import java.util.EnumSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.DirectoryListingStartAfterNotFoundException;
@@ -41,6 +40,8 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -131,9 +132,6 @@ import org.jboss.netty.channel.ChannelHa
import com.google.common.annotations.VisibleForTesting;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NFS_KEYTAB_FILE_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NFS_KERBEROS_PRINCIPAL_KEY;
-
/**
* RPC program corresponding to nfs daemon. See {@link Nfs3}.
*/
@@ -144,7 +142,7 @@ public class RpcProgramNfs3 extends RpcP
static final Log LOG = LogFactory.getLog(RpcProgramNfs3.class);
- private final Configuration config = new Configuration();
+ private final NfsConfiguration config;
private final WriteManager writeManager;
private final IdUserGroup iug;
private final DFSClientCache clientCache;
@@ -159,15 +157,17 @@ public class RpcProgramNfs3 extends RpcP
private final RpcCallCache rpcCallCache;
- public RpcProgramNfs3(Configuration config, DatagramSocket registrationSocket,
+ public RpcProgramNfs3(NfsConfiguration config, DatagramSocket registrationSocket,
boolean allowInsecurePorts) throws IOException {
- super("NFS3", "localhost", config.getInt(Nfs3Constant.NFS3_SERVER_PORT,
- Nfs3Constant.NFS3_SERVER_PORT_DEFAULT), Nfs3Constant.PROGRAM,
+ super("NFS3", "localhost", config.getInt(
+ NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY,
+ NfsConfigKeys.DFS_NFS_SERVER_PORT_DEFAULT), Nfs3Constant.PROGRAM,
Nfs3Constant.VERSION, Nfs3Constant.VERSION, registrationSocket,
allowInsecurePorts);
+ this.config = config;
config.set(FsPermission.UMASK_LABEL, "000");
- iug = new IdUserGroup();
+ iug = new IdUserGroup(config);
exports = NfsExports.getInstance(config);
writeManager = new WriteManager(iug, config);
@@ -180,13 +180,13 @@ public class RpcProgramNfs3 extends RpcP
CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT);
- writeDumpDir = config.get(Nfs3Constant.FILE_DUMP_DIR_KEY,
- Nfs3Constant.FILE_DUMP_DIR_DEFAULT);
- boolean enableDump = config.getBoolean(Nfs3Constant.ENABLE_FILE_DUMP_KEY,
- Nfs3Constant.ENABLE_FILE_DUMP_DEFAULT);
+ writeDumpDir = config.get(NfsConfigKeys.DFS_NFS_FILE_DUMP_DIR_KEY,
+ NfsConfigKeys.DFS_NFS_FILE_DUMP_DIR_DEFAULT);
+ boolean enableDump = config.getBoolean(NfsConfigKeys.DFS_NFS_FILE_DUMP_KEY,
+ NfsConfigKeys.DFS_NFS_FILE_DUMP_DEFAULT);
UserGroupInformation.setConfiguration(config);
- SecurityUtil.login(config, DFS_NFS_KEYTAB_FILE_KEY,
- DFS_NFS_KERBEROS_PRINCIPAL_KEY);
+ SecurityUtil.login(config, NfsConfigKeys.DFS_NFS_KEYTAB_FILE_KEY,
+ NfsConfigKeys.DFS_NFS_KERBEROS_PRINCIPAL_KEY);
if (!enableDump) {
writeDumpDir = null;
@@ -567,8 +567,8 @@ public class RpcProgramNfs3 extends RpcP
+ handle.getFileId());
return new READLINK3Response(Nfs3Status.NFS3ERR_SERVERFAULT);
}
- int rtmax = config.getInt(Nfs3Constant.MAX_READ_TRANSFER_SIZE_KEY,
- Nfs3Constant.MAX_READ_TRANSFER_SIZE_DEFAULT);
+ int rtmax = config.getInt(NfsConfigKeys.DFS_NFS_MAX_READ_TRANSFER_SIZE_KEY,
+ NfsConfigKeys.DFS_NFS_MAX_READ_TRANSFER_SIZE_DEFAULT);
if (rtmax < target.getBytes().length) {
LOG.error("Link size: " + target.getBytes().length
+ " is larger than max transfer size: " + rtmax);
@@ -665,8 +665,8 @@ public class RpcProgramNfs3 extends RpcP
}
try {
- int rtmax = config.getInt(Nfs3Constant.MAX_READ_TRANSFER_SIZE_KEY,
- Nfs3Constant.MAX_READ_TRANSFER_SIZE_DEFAULT);
+ int rtmax = config.getInt(NfsConfigKeys.DFS_NFS_MAX_READ_TRANSFER_SIZE_KEY,
+ NfsConfigKeys.DFS_NFS_MAX_READ_TRANSFER_SIZE_DEFAULT);
int buffSize = Math.min(rtmax, count);
byte[] readbuffer = new byte[buffSize];
@@ -1740,12 +1740,15 @@ public class RpcProgramNfs3 extends RpcP
}
try {
- int rtmax = config.getInt(Nfs3Constant.MAX_READ_TRANSFER_SIZE_KEY,
- Nfs3Constant.MAX_READ_TRANSFER_SIZE_DEFAULT);
- int wtmax = config.getInt(Nfs3Constant.MAX_WRITE_TRANSFER_SIZE_KEY,
- Nfs3Constant.MAX_WRITE_TRANSFER_SIZE_DEFAULT);
- int dtperf = config.getInt(Nfs3Constant.MAX_READDIR_TRANSFER_SIZE_KEY,
- Nfs3Constant.MAX_READDIR_TRANSFER_SIZE_DEFAULT);
+ int rtmax = config.getInt(
+ NfsConfigKeys.DFS_NFS_MAX_READ_TRANSFER_SIZE_KEY,
+ NfsConfigKeys.DFS_NFS_MAX_READ_TRANSFER_SIZE_DEFAULT);
+ int wtmax = config.getInt(
+ NfsConfigKeys.DFS_NFS_MAX_WRITE_TRANSFER_SIZE_KEY,
+ NfsConfigKeys.DFS_NFS_MAX_WRITE_TRANSFER_SIZE_DEFAULT);
+ int dtperf = config.getInt(
+ NfsConfigKeys.DFS_NFS_MAX_READDIR_TRANSFER_SIZE_KEY,
+ NfsConfigKeys.DFS_NFS_MAX_READDIR_TRANSFER_SIZE_DEFAULT);
Nfs3FileAttributes attrs = Nfs3Utils.getFileAttr(dfsClient,
Nfs3Utils.getFileIdPath(handle), iug);
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java Sat Jun 7 16:29:10 2014
@@ -21,10 +21,11 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.COMMIT_STATUS;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.ipc.RemoteException;
@@ -50,7 +51,7 @@ import com.google.common.annotations.Vis
public class WriteManager {
public static final Log LOG = LogFactory.getLog(WriteManager.class);
- private final Configuration config;
+ private final NfsConfiguration config;
private final IdUserGroup iug;
private AsyncDataService asyncDataService;
@@ -78,19 +79,19 @@ public class WriteManager {
return fileContextCache.put(h, ctx);
}
- WriteManager(IdUserGroup iug, final Configuration config) {
+ WriteManager(IdUserGroup iug, final NfsConfiguration config) {
this.iug = iug;
this.config = config;
- streamTimeout = config.getLong(Nfs3Constant.OUTPUT_STREAM_TIMEOUT,
- Nfs3Constant.OUTPUT_STREAM_TIMEOUT_DEFAULT);
+ streamTimeout = config.getLong(NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_KEY,
+ NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_DEFAULT);
LOG.info("Stream timeout is " + streamTimeout + "ms.");
- if (streamTimeout < Nfs3Constant.OUTPUT_STREAM_TIMEOUT_MIN_DEFAULT) {
+ if (streamTimeout < NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_MIN_DEFAULT) {
LOG.info("Reset stream timeout to minimum value "
- + Nfs3Constant.OUTPUT_STREAM_TIMEOUT_MIN_DEFAULT + "ms.");
- streamTimeout = Nfs3Constant.OUTPUT_STREAM_TIMEOUT_MIN_DEFAULT;
+ + NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_MIN_DEFAULT + "ms.");
+ streamTimeout = NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_MIN_DEFAULT;
}
- maxStreams = config.getInt(Nfs3Constant.MAX_OPEN_FILES,
- Nfs3Constant.MAX_OPEN_FILES_DEFAULT);
+ maxStreams = config.getInt(NfsConfigKeys.DFS_NFS_MAX_OPEN_FILES_KEY,
+ NfsConfigKeys.DFS_NFS_MAX_OPEN_FILES_DEFAULT);
LOG.info("Maximum open streams is "+ maxStreams);
this.fileContextCache = new OpenFileCtxCache(config, streamTimeout);
}
@@ -171,8 +172,8 @@ public class WriteManager {
}
// Add open stream
- String writeDumpDir = config.get(Nfs3Constant.FILE_DUMP_DIR_KEY,
- Nfs3Constant.FILE_DUMP_DIR_DEFAULT);
+ String writeDumpDir = config.get(NfsConfigKeys.DFS_NFS_FILE_DUMP_DIR_KEY,
+ NfsConfigKeys.DFS_NFS_FILE_DUMP_DIR_DEFAULT);
openFileCtx = new OpenFileCtx(fos, latestAttr, writeDumpDir + "/"
+ fileHandle.getFileId(), dfsClient, iug);
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestMountd.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestMountd.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestMountd.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestMountd.java Sat Jun 7 16:29:10 2014
@@ -23,8 +23,8 @@ import java.net.InetAddress;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.hdfs.nfs.mount.RpcProgramMountd;
import org.apache.hadoop.hdfs.nfs.nfs3.Nfs3;
import org.apache.hadoop.hdfs.nfs.nfs3.RpcProgramNfs3;
@@ -38,7 +38,7 @@ public class TestMountd {
@Test
public void testStart() throws IOException {
// Start minicluster
- Configuration config = new Configuration();
+ NfsConfiguration config = new NfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(config).numDataNodes(1)
.build();
cluster.waitActive();
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestOutOfOrderWrite.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestOutOfOrderWrite.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestOutOfOrderWrite.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestOutOfOrderWrite.java Sat Jun 7 16:29:10 2014
@@ -23,7 +23,8 @@ import java.util.Arrays;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.hdfs.nfs.nfs3.Nfs3Utils;
import org.apache.hadoop.nfs.nfs3.FileHandle;
import org.apache.hadoop.nfs.nfs3.Nfs3Constant;
@@ -155,10 +156,10 @@ public class TestOutOfOrderWrite {
Arrays.fill(data3, (byte) 9);
// NFS3 Create request
- Configuration conf = new Configuration();
+ NfsConfiguration conf = new NfsConfiguration();
WriteClient client = new WriteClient("localhost", conf.getInt(
- Nfs3Constant.NFS3_SERVER_PORT, Nfs3Constant.NFS3_SERVER_PORT_DEFAULT),
- create(), false);
+ NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY,
+ NfsConfigKeys.DFS_NFS_SERVER_PORT_DEFAULT), create(), false);
client.run();
while (handle == null) {
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java Sat Jun 7 16:29:10 2014
@@ -24,11 +24,11 @@ import java.io.IOException;
import java.net.InetAddress;
import java.util.List;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.hdfs.nfs.nfs3.Nfs3;
import org.apache.hadoop.hdfs.nfs.nfs3.RpcProgramNfs3;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -53,7 +53,7 @@ import org.mockito.Mockito;
*/
public class TestReaddir {
- static Configuration config = new Configuration();
+ static NfsConfiguration config = new NfsConfiguration();
static MiniDFSCluster cluster = null;
static DistributedFileSystem hdfs;
static NameNode nn;
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java Sat Jun 7 16:29:10 2014
@@ -18,24 +18,24 @@
package org.apache.hadoop.hdfs.nfs.nfs3;
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
+import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertThat;
-import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.Test;
public class TestDFSClientCache {
@Test
public void testEviction() throws IOException {
- Configuration conf = new Configuration();
+ NfsConfiguration conf = new NfsConfiguration();
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "hdfs://localhost");
// Only one entry will be in the cache
@@ -59,7 +59,7 @@ public class TestDFSClientCache {
String currentUser = "test-user";
- Configuration conf = new Configuration();
+ NfsConfiguration conf = new NfsConfiguration();
UserGroupInformation currentUserUgi
= UserGroupInformation.createRemoteUser(currentUser);
currentUserUgi.setAuthenticationMethod(KERBEROS);
@@ -83,7 +83,7 @@ public class TestDFSClientCache {
UserGroupInformation currentUserUgi = UserGroupInformation
.createUserForTesting(currentUser, new String[0]);
- Configuration conf = new Configuration();
+ NfsConfiguration conf = new NfsConfiguration();
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "hdfs://localhost");
DFSClientCache cache = new DFSClientCache(conf);
UserGroupInformation ugiResult
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestExportsTable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestExportsTable.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestExportsTable.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestExportsTable.java Sat Jun 7 16:29:10 2014
@@ -21,22 +21,22 @@ import static org.junit.Assert.assertTru
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.hdfs.nfs.mount.Mountd;
import org.apache.hadoop.hdfs.nfs.mount.RpcProgramMountd;
-import org.apache.hadoop.nfs.nfs3.Nfs3Constant;
import org.junit.Test;
public class TestExportsTable {
@Test
public void testExportPoint() throws IOException {
- Configuration config = new Configuration();
+ NfsConfiguration config = new NfsConfiguration();
MiniDFSCluster cluster = null;
String exportPoint = "/myexport1";
- config.setStrings(Nfs3Constant.EXPORT_POINT, exportPoint);
+ config.setStrings(NfsConfigKeys.DFS_NFS_EXPORT_POINT_KEY, exportPoint);
// Use emphral port in case tests are running in parallel
config.setInt("nfs3.mountd.port", 0);
config.setInt("nfs3.server.port", 0);
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestOpenFileCtxCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestOpenFileCtxCache.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestOpenFileCtxCache.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestOpenFileCtxCache.java Sat Jun 7 16:29:10 2014
@@ -22,13 +22,13 @@ import static org.junit.Assert.assertTru
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.CommitCtx;
import org.apache.hadoop.nfs.nfs3.FileHandle;
import org.apache.hadoop.nfs.nfs3.IdUserGroup;
-import org.apache.hadoop.nfs.nfs3.Nfs3Constant;
import org.apache.hadoop.nfs.nfs3.Nfs3FileAttributes;
import org.junit.Test;
import org.mockito.Mockito;
@@ -38,10 +38,10 @@ public class TestOpenFileCtxCache {
@Test
public void testEviction() throws IOException, InterruptedException {
- Configuration conf = new Configuration();
+ NfsConfiguration conf = new NfsConfiguration();
// Only two entries will be in the cache
- conf.setInt(Nfs3Constant.MAX_OPEN_FILES, 2);
+ conf.setInt(NfsConfigKeys.DFS_NFS_MAX_OPEN_FILES_KEY, 2);
DFSClient dfsClient = Mockito.mock(DFSClient.class);
Nfs3FileAttributes attr = new Nfs3FileAttributes();
@@ -49,15 +49,15 @@ public class TestOpenFileCtxCache {
Mockito.when(fos.getPos()).thenReturn((long) 0);
OpenFileCtx context1 = new OpenFileCtx(fos, attr, "/dumpFilePath",
- dfsClient, new IdUserGroup());
+ dfsClient, new IdUserGroup(new NfsConfiguration()));
OpenFileCtx context2 = new OpenFileCtx(fos, attr, "/dumpFilePath",
- dfsClient, new IdUserGroup());
+ dfsClient, new IdUserGroup(new NfsConfiguration()));
OpenFileCtx context3 = new OpenFileCtx(fos, attr, "/dumpFilePath",
- dfsClient, new IdUserGroup());
+ dfsClient, new IdUserGroup(new NfsConfiguration()));
OpenFileCtx context4 = new OpenFileCtx(fos, attr, "/dumpFilePath",
- dfsClient, new IdUserGroup());
+ dfsClient, new IdUserGroup(new NfsConfiguration()));
OpenFileCtx context5 = new OpenFileCtx(fos, attr, "/dumpFilePath",
- dfsClient, new IdUserGroup());
+ dfsClient, new IdUserGroup(new NfsConfiguration()));
OpenFileCtxCache cache = new OpenFileCtxCache(conf, 10 * 60 * 100);
@@ -71,7 +71,7 @@ public class TestOpenFileCtxCache {
assertTrue(cache.size() == 2);
// Wait for the oldest stream to be evict-able, insert again
- Thread.sleep(Nfs3Constant.OUTPUT_STREAM_TIMEOUT_MIN_DEFAULT);
+ Thread.sleep(NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_MIN_DEFAULT);
assertTrue(cache.size() == 2);
ret = cache.put(new FileHandle(3), context3);
@@ -90,17 +90,17 @@ public class TestOpenFileCtxCache {
new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null));
context4.getPendingCommitsForTest().put(new Long(100),
new CommitCtx(0, null, 0, attr));
- Thread.sleep(Nfs3Constant.OUTPUT_STREAM_TIMEOUT_MIN_DEFAULT);
+ Thread.sleep(NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_MIN_DEFAULT);
ret = cache.put(new FileHandle(5), context5);
assertFalse(ret);
}
@Test
public void testScan() throws IOException, InterruptedException {
- Configuration conf = new Configuration();
+ NfsConfiguration conf = new NfsConfiguration();
// Only two entries will be in the cache
- conf.setInt(Nfs3Constant.MAX_OPEN_FILES, 2);
+ conf.setInt(NfsConfigKeys.DFS_NFS_MAX_OPEN_FILES_KEY, 2);
DFSClient dfsClient = Mockito.mock(DFSClient.class);
Nfs3FileAttributes attr = new Nfs3FileAttributes();
@@ -108,13 +108,13 @@ public class TestOpenFileCtxCache {
Mockito.when(fos.getPos()).thenReturn((long) 0);
OpenFileCtx context1 = new OpenFileCtx(fos, attr, "/dumpFilePath",
- dfsClient, new IdUserGroup());
+ dfsClient, new IdUserGroup(new NfsConfiguration()));
OpenFileCtx context2 = new OpenFileCtx(fos, attr, "/dumpFilePath",
- dfsClient, new IdUserGroup());
+ dfsClient, new IdUserGroup(new NfsConfiguration()));
OpenFileCtx context3 = new OpenFileCtx(fos, attr, "/dumpFilePath",
- dfsClient, new IdUserGroup());
+ dfsClient, new IdUserGroup(new NfsConfiguration()));
OpenFileCtx context4 = new OpenFileCtx(fos, attr, "/dumpFilePath",
- dfsClient, new IdUserGroup());
+ dfsClient, new IdUserGroup(new NfsConfiguration()));
OpenFileCtxCache cache = new OpenFileCtxCache(conf, 10 * 60 * 100);
@@ -123,8 +123,8 @@ public class TestOpenFileCtxCache {
assertTrue(ret);
ret = cache.put(new FileHandle(2), context2);
assertTrue(ret);
- Thread.sleep(Nfs3Constant.OUTPUT_STREAM_TIMEOUT_MIN_DEFAULT + 1);
- cache.scan(Nfs3Constant.OUTPUT_STREAM_TIMEOUT_MIN_DEFAULT);
+ Thread.sleep(NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_MIN_DEFAULT + 1);
+ cache.scan(NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_MIN_DEFAULT);
assertTrue(cache.size() == 0);
// Test cleaning inactive entry
@@ -133,7 +133,7 @@ public class TestOpenFileCtxCache {
ret = cache.put(new FileHandle(4), context4);
assertTrue(ret);
context3.setActiveStatusForTest(false);
- cache.scan(Nfs3Constant.OUTPUT_STREAM_TIMEOUT_DEFAULT);
+ cache.scan(NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_DEFAULT);
assertTrue(cache.size() == 1);
assertTrue(cache.get(new FileHandle(3)) == null);
assertTrue(cache.get(new FileHandle(4)) != null);
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java Sat Jun 7 16:29:10 2014
@@ -17,6 +17,11 @@
*/
package org.apache.hadoop.hdfs.nfs.nfs3;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.nfs.nfs3.Nfs3Constant;
import org.junit.Assert;
import org.junit.Test;
@@ -63,4 +68,41 @@ public class TestRpcProgramNfs3 {
}
}
}
+
+ @Test
+ public void testDeprecatedKeys() {
+ NfsConfiguration conf = new NfsConfiguration();
+ conf.setInt("nfs3.server.port", 998);
+ assertTrue(conf.getInt(NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY, 0) == 998);
+
+ conf.setInt("nfs3.mountd.port", 999);
+ assertTrue(conf.getInt(NfsConfigKeys.DFS_NFS_MOUNTD_PORT_KEY, 0) == 999);
+
+ conf.set("dfs.nfs.exports.allowed.hosts", "host1");
+ assertTrue(conf.get(CommonConfigurationKeys.NFS_EXPORTS_ALLOWED_HOSTS_KEY)
+ .equals("host1"));
+
+ conf.setInt("dfs.nfs.exports.cache.expirytime.millis", 1000);
+ assertTrue(conf.getInt(
+ Nfs3Constant.NFS_EXPORTS_CACHE_EXPIRYTIME_MILLIS_KEY, 0) == 1000);
+
+ conf.setInt("hadoop.nfs.userupdate.milly", 10);
+ assertTrue(conf.getInt(Nfs3Constant.NFS_USERGROUP_UPDATE_MILLIS_KEY, 0) == 10);
+
+ conf.set("dfs.nfs3.dump.dir", "/nfs/tmp");
+ assertTrue(conf.get(NfsConfigKeys.DFS_NFS_FILE_DUMP_DIR_KEY).equals(
+ "/nfs/tmp"));
+
+ conf.setBoolean("dfs.nfs3.enableDump", false);
+ assertTrue(conf.getBoolean(NfsConfigKeys.DFS_NFS_FILE_DUMP_KEY, true) == false);
+
+ conf.setInt("dfs.nfs3.max.open.files", 500);
+ assertTrue(conf.getInt(NfsConfigKeys.DFS_NFS_MAX_OPEN_FILES_KEY, 0) == 500);
+
+ conf.setInt("dfs.nfs3.stream.timeout", 6000);
+ assertTrue(conf.getInt(NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_KEY, 0) == 6000);
+
+ conf.set("dfs.nfs3.export.point", "/dir1");
+ assertTrue(conf.get(NfsConfigKeys.DFS_NFS_EXPORT_POINT_KEY).equals("/dir1"));
+ }
}
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java Sat Jun 7 16:29:10 2014
@@ -27,11 +27,10 @@ import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.concurrent.ConcurrentNavigableMap;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSClient;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.COMMIT_STATUS;
import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.CommitCtx;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -139,7 +138,7 @@ public class TestWrites {
Mockito.when(fos.getPos()).thenReturn((long) 0);
OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
- new IdUserGroup());
+ new IdUserGroup(new NfsConfiguration()));
COMMIT_STATUS ret;
@@ -201,13 +200,14 @@ public class TestWrites {
Nfs3FileAttributes attr = new Nfs3FileAttributes();
HdfsDataOutputStream fos = Mockito.mock(HdfsDataOutputStream.class);
Mockito.when(fos.getPos()).thenReturn((long) 0);
+ NfsConfiguration config = new NfsConfiguration();
OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
- new IdUserGroup());
+ new IdUserGroup(config));
FileHandle h = new FileHandle(1); // fake handle for "/dumpFilePath"
COMMIT_STATUS ret;
- WriteManager wm = new WriteManager(new IdUserGroup(), new Configuration());
+ WriteManager wm = new WriteManager(new IdUserGroup(config), config);
assertTrue(wm.addOpenFileStream(h, ctx));
// Test inactive open file context
@@ -280,7 +280,7 @@ public class TestWrites {
@Test
public void testWriteStableHow() throws IOException, InterruptedException {
- HdfsConfiguration config = new HdfsConfiguration();
+ NfsConfiguration config = new NfsConfiguration();
DFSClient client = null;
MiniDFSCluster cluster = null;
RpcProgramNfs3 nfsd;
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml Sat Jun 7 16:29:10 2014
@@ -18,12 +18,12 @@
<configuration>
<property>
- <name>nfs3.server.port</name>
+ <name>nfs.server.port</name>
<value>2079</value>
</property>
<property>
- <name>nfs3.mountd.port</name>
+ <name>nfs.mountd.port</name>
<value>4272</value>
</property>
</configuration>
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Sat Jun 7 16:29:10 2014
@@ -254,6 +254,9 @@ Trunk (Unreleased)
HDFS-5794. Fix the inconsistency of layout version number of
ADD_DATANODE_AND_STORAGE_UUIDS between trunk and branch-2. (jing9)
+ HDFS-6464. Support multiple xattr.name parameters for WebHDFS getXAttrs.
+ (Yi Liu via umamahesh)
+
BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS
HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh)
@@ -317,8 +320,8 @@ Trunk (Unreleased)
HDFS-6374. setXAttr should require the user to be the owner of the file
or directory (Charles Lamb via wang)
- HDFS-6110 adding more slow action log in critical write path
- (Liang Xie via stack)
+ HDFS-6375. Listing extended attributes with the search permission.
+ (Charles Lamb via wang)
Release 2.5.0 - UNRELEASED
@@ -452,11 +455,33 @@ Release 2.5.0 - UNRELEASED
HDFS-6356. Fix typo in DatanodeLayoutVersion. (Tulasi G via wang)
HDFS-6447. balancer should timestamp the completion message.
- (Juan Yu via wang).
+ (Juan Yu via wang)
HDFS-6463. Clarify behavior of AclStorage#createFsPermissionForExtendedAcl
in comments. (cnauroth)
+ HDFS-6472. fix typo in webapps/hdfs/explorer.js. (Juan Yu via wang)
+
+ HDFS-6056. Clean up NFS config settings (brandonli)
+
+ HDFS-6110 adding more slow action log in critical write path
+ (Liang Xie via stack)
+
+ HDFS-6109 let sync_file_range() system call run in background
+ (Liang Xie via stack)
+
+ HDFS-6268. Better sorting in NetworkTopology#pseudoSortByDistance when
+ no local node is found. (wang)
+
+ HDFS-6369. Document that BlockReader#available() can return more bytes than
+ are remaining in the block (Ted Yu via Colin Patrick McCabe)
+
+ HDFS-6487. TestStandbyCheckpoint#testSBNCheckpoints is racy.
+ (Mit Desai via wang)
+
+ HDFS-6297. Add CLI testcases to reflect new features of dfs and dfsadmin
+ (Dasha Boudnik via cos)
+
OPTIMIZATIONS
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
@@ -620,6 +645,15 @@ Release 2.5.0 - UNRELEASED
HDFS-6462. NFS: fsstat request fails with the secure hdfs (brandonli)
+ HDFS-6404. HttpFS should use a 000 umask for mkdir and create
+ operations. (yoderme via tucu)
+
+ HDFS-6424. blockReport doesn't need to invalidate blocks on SBN. (Ming Ma
+ via jing9)
+
+ HDFS-6497. Make TestAvailableSpaceVolumeChoosingPolicy deterministic
+ (cmccabe)
+
Release 2.4.1 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1598456-1601150
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java Sat Jun 7 16:29:10 2014
@@ -437,6 +437,11 @@ public class Hdfs extends AbstractFileSy
throws IOException {
return dfs.getXAttrs(getUriPath(path), names);
}
+
+ @Override
+ public List<String> listXAttrs(Path path) throws IOException {
+ return dfs.listXAttrs(getUriPath(path));
+ }
@Override
public void removeXAttr(Path path, String name) throws IOException {
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReader.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReader.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReader.java Sat Jun 7 16:29:10 2014
@@ -51,6 +51,7 @@ public interface BlockReader extends Byt
* Returns an estimate of the number of bytes that can be read
* (or skipped over) from this input stream without performing
* network I/O.
+ * This may return more than what is actually present in the block.
*/
int available() throws IOException;
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java Sat Jun 7 16:29:10 2014
@@ -89,6 +89,7 @@ import java.util.concurrent.atomic.Atomi
import javax.net.SocketFactory;
+import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -2827,6 +2828,20 @@ public class DFSClient implements java.i
}
}
+ public List<String> listXAttrs(String src)
+ throws IOException {
+ checkOpen();
+ try {
+ final Map<String, byte[]> xattrs =
+ XAttrHelper.buildXAttrMap(namenode.listXAttrs(src));
+ return Lists.newArrayList(xattrs.keySet());
+ } catch(RemoteException re) {
+ throw re.unwrapRemoteException(AccessControlException.class,
+ FileNotFoundException.class,
+ UnresolvedPathException.class);
+ }
+ }
+
public void removeXAttr(String src, String name) throws IOException {
checkOpen();
try {
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Sat Jun 7 16:29:10 2014
@@ -111,6 +111,8 @@ public class DFSConfigKeys extends Commo
public static final boolean DFS_DATANODE_DROP_CACHE_BEHIND_WRITES_DEFAULT = false;
public static final String DFS_DATANODE_SYNC_BEHIND_WRITES_KEY = "dfs.datanode.sync.behind.writes";
public static final boolean DFS_DATANODE_SYNC_BEHIND_WRITES_DEFAULT = false;
+ public static final String DFS_DATANODE_SYNC_BEHIND_WRITES_IN_BACKGROUND_KEY = "dfs.datanode.sync.behind.writes.in.background";
+ public static final boolean DFS_DATANODE_SYNC_BEHIND_WRITES_IN_BACKGROUND_DEFAULT = false;
public static final String DFS_DATANODE_DROP_CACHE_BEHIND_READS_KEY = "dfs.datanode.drop.cache.behind.reads";
public static final boolean DFS_DATANODE_DROP_CACHE_BEHIND_READS_DEFAULT = false;
public static final String DFS_DATANODE_USE_DN_HOSTNAME = "dfs.datanode.use.datanode.hostname";
@@ -639,12 +641,6 @@ public class DFSConfigKeys extends Commo
public static final String DFS_DFSCLIENT_HEDGED_READ_THREADPOOL_SIZE =
"dfs.client.hedged.read.threadpool.size";
public static final int DEFAULT_DFSCLIENT_HEDGED_READ_THREADPOOL_SIZE = 0;
- public static final String DFS_NFS_KEYTAB_FILE_KEY = "dfs.nfs.keytab.file";
- public static final String DFS_NFS_KERBEROS_PRINCIPAL_KEY = "dfs.nfs.kerberos.principal";
- public static final String DFS_NFS_REGISTRATION_PORT_KEY = "dfs.nfs.registration.port";
- public static final int DFS_NFS_REGISTRATION_PORT_DEFAULT = 40; // Currently unassigned.
- public static final String DFS_NFS_ALLOW_INSECURE_PORTS_KEY = "dfs.nfs.allow.insecure.ports";
- public static final boolean DFS_NFS_ALLOW_INSECURE_PORTS_DEFAULT = true;
// Slow io warning log threshold settings for dfsclient and datanode.
public static final String DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY =
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java Sat Jun 7 16:29:10 2014
@@ -1865,6 +1865,23 @@ public class DistributedFileSystem exten
}
@Override
+ public List<String> listXAttrs(Path path)
+ throws IOException {
+ final Path absF = fixRelativePart(path);
+ return new FileSystemLinkResolver<List<String>>() {
+ @Override
+ public List<String> doCall(final Path p) throws IOException {
+ return dfs.listXAttrs(getPathName(p));
+ }
+ @Override
+ public List<String> next(final FileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ return fs.listXAttrs(p);
+ }
+ }.resolve(this, absF);
+ }
+
+ @Override
public void removeXAttr(Path path, final String name) throws IOException {
Path absF = fixRelativePart(path);
new FileSystemLinkResolver<Void>() {
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java Sat Jun 7 16:29:10 2014
@@ -152,7 +152,7 @@ public class XAttrHelper {
public static List<XAttr> buildXAttrs(List<String> names) {
if (names == null || names.isEmpty()) {
throw new HadoopIllegalArgumentException("XAttr names can not be " +
- "null or empty.");
+ "null or empty.");
}
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(names.size());
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java Sat Jun 7 16:29:10 2014
@@ -1300,6 +1300,27 @@ public interface ClientProtocol {
@Idempotent
public List<XAttr> getXAttrs(String src, List<XAttr> xAttrs)
throws IOException;
+
+ /**
+ * List the xattrs names for a file or directory.
+ * Only the xattr names for which the logged in user has the permissions to
+ * access will be returned.
+ * <p/>
+ * A regular user only can get xattr names from the "user" namespace.
+ * A super user can get xattr names of the "user" and "trusted" namespace.
+ * XAttr names of the "security" and "system" namespaces are only used/exposed
+ * internally by the file system impl.
+ * <p/>
+ * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
+ * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * @param src file or directory
+ * @param xAttrs xAttrs to get
+ * @return List<XAttr> <code>XAttr</code> list
+ * @throws IOException
+ */
+ @Idempotent
+ public List<XAttr> listXAttrs(String src)
+ throws IOException;
/**
* Remove xattr of a file or directory.Value in xAttr parameter is ignored.
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java Sat Jun 7 16:29:10 2014
@@ -174,8 +174,11 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.RemoveXAttrRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.RemoveXAttrResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.SetXAttrRequestProto;
@@ -1297,6 +1300,16 @@ public class ClientNamenodeProtocolServe
throw new ServiceException(e);
}
}
+
+ @Override
+ public ListXAttrsResponseProto listXAttrs(RpcController controller,
+ ListXAttrsRequestProto req) throws ServiceException {
+ try {
+ return PBHelper.convertListXAttrsResponse(server.listXAttrs(req.getSrc()));
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
@Override
public RemoveXAttrResponseProto removeXAttr(RpcController controller,
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java Sat Jun 7 16:29:10 2014
@@ -144,6 +144,7 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdateBlockForPipelineRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.RemoveXAttrRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.SetXAttrRequestProto;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
@@ -1306,6 +1307,19 @@ public class ClientNamenodeProtocolTrans
}
@Override
+ public List<XAttr> listXAttrs(String src)
+ throws IOException {
+ ListXAttrsRequestProto.Builder builder = ListXAttrsRequestProto.newBuilder();
+ builder.setSrc(src);
+ ListXAttrsRequestProto req = builder.build();
+ try {
+ return PBHelper.convert(rpcProxy.listXAttrs(null, req));
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
public void removeXAttr(String src, XAttr xAttr) throws IOException {
RemoveXAttrRequestProto req = RemoveXAttrRequestProto
.newBuilder().setSrc(src)
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java Sat Jun 7 16:29:10 2014
@@ -44,6 +44,7 @@ import org.apache.hadoop.ha.HAServicePro
import org.apache.hadoop.ha.proto.HAServiceProtocolProtos;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
@@ -153,6 +154,7 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageUuidsProto;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrProto.XAttrNamespaceProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrSetFlagProto;
@@ -2179,6 +2181,21 @@ public class PBHelper {
return builder.build();
}
+ public static List<XAttr> convert(ListXAttrsResponseProto a) {
+ final List<XAttrProto> xAttrs = a.getXAttrsList();
+ return convertXAttrs(xAttrs);
+ }
+
+ public static ListXAttrsResponseProto convertListXAttrsResponse(
+ List<XAttr> names) {
+ ListXAttrsResponseProto.Builder builder =
+ ListXAttrsResponseProto.newBuilder();
+ if (names != null) {
+ builder.addAllXAttrs(convertXAttrProto(names));
+ }
+ return builder.build();
+ }
+
public static ShortCircuitShmSlotProto convert(SlotId slotId) {
return ShortCircuitShmSlotProto.newBuilder().
setShmId(convert(slotId.getShmId())).
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Sat Jun 7 16:29:10 2014
@@ -1050,6 +1050,9 @@ public class BlockManager {
* datanode and log the operation
*/
void addToInvalidates(final Block block, final DatanodeInfo datanode) {
+ if (!namesystem.isPopulatingReplQueues()) {
+ return;
+ }
invalidateBlocks.add(block, datanode, true);
}
@@ -1058,6 +1061,9 @@ public class BlockManager {
* datanodes.
*/
private void addToInvalidates(Block b) {
+ if (!namesystem.isPopulatingReplQueues()) {
+ return;
+ }
StringBuilder datanodes = new StringBuilder();
for(DatanodeStorageInfo storage : blocksMap.getStorages(b, State.NORMAL)) {
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
Modified: hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java?rev=1601151&r1=1601150&r2=1601151&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java (original)
+++ hadoop/common/branches/HDFS-5442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java Sat Jun 7 16:29:10 2014
@@ -351,7 +351,8 @@ public class DatanodeManager {
DFSUtil.DECOM_COMPARATOR;
for (LocatedBlock b : locatedblocks) {
- networktopology.pseudoSortByDistance(client, b.getLocations());
+ networktopology.sortByDistance(client, b.getLocations(), b
+ .getBlock().getBlockId());
// Move decommissioned/stale datanodes to the bottom
Arrays.sort(b.getLocations(), comparator);
}