You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ki...@apache.org on 2013/04/02 21:13:47 UTC
svn commit: r1463699 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/
src/test/java/org/apache/hadoop/hdfs/web/
Author: kihwal
Date: Tue Apr 2 19:13:46 2013
New Revision: 1463699
URL: http://svn.apache.org/r1463699
Log:
svn merge -c 1463698 Merging from trunk to branch-2 to fix HDFS-4649.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1463699&r1=1463698&r2=1463699&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Apr 2 19:13:46 2013
@@ -2216,6 +2216,8 @@ Release 0.23.7 - UNRELEASED
HDFS-4581. checkDiskError should not be called on network errors (Rohit
Kochar via kihwal)
+ HDFS-4649. Webhdfs cannot list large directories (daryn via kihwal)
+
Release 0.23.6 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java?rev=1463699&r1=1463698&r2=1463699&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java Tue Apr 2 19:13:46 2013
@@ -718,9 +718,15 @@ public class NamenodeWebHdfsMethods {
private static StreamingOutput getListingStream(final NamenodeProtocols np,
final String p) throws IOException {
- final DirectoryListing first = getDirectoryListing(np, p,
+ // allows exceptions like FNF or ACE to prevent http response of 200 for
+ // a failure since we can't (currently) return error responses in the
+ // middle of a streaming operation
+ final DirectoryListing firstDirList = getDirectoryListing(np, p,
HdfsFileStatus.EMPTY_NAME);
+ // must save ugi because the streaming object will be executed outside
+ // the remote user's ugi
+ final UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
return new StreamingOutput() {
@Override
public void write(final OutputStream outstream) throws IOException {
@@ -729,21 +735,32 @@ public class NamenodeWebHdfsMethods {
out.println("{\"" + FileStatus.class.getSimpleName() + "es\":{\""
+ FileStatus.class.getSimpleName() + "\":[");
- final HdfsFileStatus[] partial = first.getPartialListing();
- if (partial.length > 0) {
- out.print(JsonUtil.toJsonString(partial[0], false));
- }
- for(int i = 1; i < partial.length; i++) {
- out.println(',');
- out.print(JsonUtil.toJsonString(partial[i], false));
- }
-
- for(DirectoryListing curr = first; curr.hasMore(); ) {
- curr = getDirectoryListing(np, p, curr.getLastName());
- for(HdfsFileStatus s : curr.getPartialListing()) {
- out.println(',');
- out.print(JsonUtil.toJsonString(s, false));
- }
+ try {
+ // restore remote user's ugi
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws IOException {
+ long n = 0;
+ for (DirectoryListing dirList = firstDirList; ;
+ dirList = getDirectoryListing(np, p, dirList.getLastName())
+ ) {
+ // send each segment of the directory listing
+ for (HdfsFileStatus s : dirList.getPartialListing()) {
+ if (n++ > 0) {
+ out.println(',');
+ }
+ out.print(JsonUtil.toJsonString(s, false));
+ }
+ // stop if last segment
+ if (!dirList.hasMore()) {
+ break;
+ }
+ }
+ return null;
+ }
+ });
+ } catch (InterruptedException e) {
+ throw new IOException(e);
}
out.println();
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java?rev=1463699&r1=1463698&r2=1463699&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java Tue Apr 2 19:13:46 2013
@@ -19,6 +19,8 @@
package org.apache.hadoop.hdfs.web;
import java.io.IOException;
+import java.net.URISyntaxException;
+import java.security.PrivilegedExceptionAction;
import java.util.Random;
import org.apache.commons.logging.Log;
@@ -29,9 +31,13 @@ import org.apache.hadoop.fs.FSDataInputS
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.TestDFSClientRetries;
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.log4j.Level;
import org.junit.Assert;
import org.junit.Test;
@@ -208,4 +214,48 @@ public class TestWebHDFS {
final Configuration conf = WebHdfsTestUtil.createConf();
TestDFSClientRetries.namenodeRestartTest(conf, true);
}
+
+ @Test(timeout=300000)
+ public void testLargeDirectory() throws Exception {
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ final int listLimit = 2;
+ // force small chunking of directory listing
+ conf.setInt(DFSConfigKeys.DFS_LIST_LIMIT, listLimit);
+ // force paths to be only owner-accessible to ensure ugi isn't changing
+ // during listStatus
+ FsPermission.setUMask(conf, new FsPermission((short)0077));
+
+ final MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
+ try {
+ cluster.waitActive();
+ WebHdfsTestUtil.getWebHdfsFileSystem(conf).setPermission(
+ new Path("/"),
+ new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
+
+ // trick the NN into not believing it's not the superuser so we can
+ // tell if the correct user is used by listStatus
+ UserGroupInformation.setLoginUser(
+ UserGroupInformation.createUserForTesting(
+ "not-superuser", new String[]{"not-supergroup"}));
+
+ UserGroupInformation.createUserForTesting("me", new String[]{"my-group"})
+ .doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws IOException, URISyntaxException {
+ FileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf);
+ Path d = new Path("/my-dir");
+ Assert.assertTrue(fs.mkdirs(d));
+ for (int i=0; i < listLimit*3; i++) {
+ Path p = new Path(d, "file-"+i);
+ Assert.assertTrue(fs.createNewFile(p));
+ }
+ Assert.assertEquals(listLimit*3, fs.listStatus(d).length);
+ return null;
+ }
+ });
+ } finally {
+ cluster.shutdown();
+ }
+ }
}