You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by va...@apache.org on 2016/11/06 09:58:28 UTC
[42/50] [abbrv] hadoop git commit: HDFS-10756. Expose getTrashRoot to
HTTPFS and WebHDFS. Contributed by Yuanbo Liu.
HDFS-10756. Expose getTrashRoot to HTTPFS and WebHDFS. Contributed by Yuanbo Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d8bab3dc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d8bab3dc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d8bab3dc
Branch: refs/heads/YARN-5355
Commit: d8bab3dcb693b2773ede9a6e4f71ae85ee056f79
Parents: 95665a6
Author: Xiao Chen <xi...@apache.org>
Authored: Fri Nov 4 18:06:55 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Fri Nov 4 18:06:55 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileSystem.java | 3 +-
.../hadoop/hdfs/DFSOpsCountStatistics.java | 1 +
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 21 +++
.../hadoop/hdfs/web/resources/GetOpParam.java | 1 +
.../hadoop/fs/http/client/HttpFSFileSystem.java | 31 +++-
.../hadoop/fs/http/server/FSOperations.java | 23 +++
.../http/server/HttpFSParametersProvider.java | 1 +
.../hadoop/fs/http/server/HttpFSServer.java | 7 +
.../src/site/markdown/index.md | 2 +
.../fs/http/client/BaseTestHttpFSWith.java | 185 +++++++++++--------
.../hadoop/fs/http/server/TestHttpFSServer.java | 70 ++++++-
.../web/resources/NamenodeWebHdfsMethods.java | 12 ++
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 34 +++-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 41 +++-
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 22 +++
15 files changed, 366 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 375af90..529745c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -113,6 +113,7 @@ public abstract class FileSystem extends Configured implements Closeable {
public static final int SHUTDOWN_HOOK_PRIORITY = 10;
public static final String TRASH_PREFIX = ".Trash";
+ public static final String USER_HOME_PREFIX = "/user";
/** FileSystem cache */
static final Cache CACHE = new Cache();
@@ -1965,7 +1966,7 @@ public abstract class FileSystem extends Configured implements Closeable {
*/
public Path getHomeDirectory() {
return this.makeQualified(
- new Path("/user/"+System.getProperty("user.name")));
+ new Path(USER_HOME_PREFIX + "/" + System.getProperty("user.name")));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
index d631dd4..4b2a761 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
@@ -64,6 +64,7 @@ public class DFSOpsCountStatistics extends StorageStatistics {
GET_STATUS(CommonStatisticNames.OP_GET_STATUS),
GET_STORAGE_POLICIES("op_get_storage_policies"),
GET_STORAGE_POLICY("op_get_storage_policy"),
+ GET_TRASH_ROOT("op_get_trash_root"),
GET_XATTR("op_get_xattr"),
LIST_LOCATED_STATUS(CommonStatisticNames.OP_LIST_LOCATED_STATUS),
LIST_STATUS(CommonStatisticNames.OP_LIST_STATUS),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 782f113..92d8a13 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -1622,6 +1622,27 @@ public class WebHdfsFileSystem extends FileSystem
}
@Override
+ public Path getTrashRoot(Path path) {
+ statistics.incrementReadOps(1);
+ storageStatistics.incrementOpCounter(OpType.GET_TRASH_ROOT);
+
+ final HttpOpParam.Op op = GetOpParam.Op.GETTRASHROOT;
+ try {
+ String strTrashPath = new FsPathResponseRunner<String>(op, path) {
+ @Override
+ String decodeResponse(Map<?, ?> json) throws IOException {
+ return JsonUtilClient.getPath(json);
+ }
+ }.run();
+ return new Path(strTrashPath).makeQualified(getUri(), null);
+ } catch(IOException e) {
+ LOG.warn("Cannot find trash root of " + path, e);
+ // keep the same behavior with dfs
+ return super.getTrashRoot(path).makeQualified(getUri(), null);
+ }
+ }
+
+ @Override
public void access(final Path path, final FsAction mode) throws IOException {
final HttpOpParam.Op op = GetOpParam.Op.CHECKACCESS;
new FsPathRunner(op, path, new FsActionParam(mode)).run();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
index 8fdf581..635e6d7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
@@ -37,6 +37,7 @@ public class GetOpParam extends HttpOpParam<GetOpParam.Op> {
GET_BLOCK_LOCATIONS(false, HttpURLConnection.HTTP_OK),
GETACLSTATUS(false, HttpURLConnection.HTTP_OK),
GETXATTRS(false, HttpURLConnection.HTTP_OK),
+ GETTRASHROOT(false, HttpURLConnection.HTTP_OK),
LISTXATTRS(false, HttpURLConnection.HTTP_OK),
NULL(false, HttpURLConnection.HTTP_NOT_IMPLEMENTED),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
index 77e3323..6cc3909 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
@@ -128,6 +128,8 @@ public class HttpFSFileSystem extends FileSystem
public static final String HOME_DIR_JSON = "Path";
+ public static final String TRASH_DIR_JSON = "Path";
+
public static final String SET_REPLICATION_JSON = "boolean";
public static final String UPLOAD_CONTENT_TYPE= "application/octet-stream";
@@ -203,7 +205,7 @@ public class HttpFSFileSystem extends FileSystem
OPEN(HTTP_GET), GETFILESTATUS(HTTP_GET), LISTSTATUS(HTTP_GET),
GETHOMEDIRECTORY(HTTP_GET), GETCONTENTSUMMARY(HTTP_GET),
GETFILECHECKSUM(HTTP_GET), GETFILEBLOCKLOCATIONS(HTTP_GET),
- INSTRUMENTATION(HTTP_GET), GETACLSTATUS(HTTP_GET),
+ INSTRUMENTATION(HTTP_GET), GETACLSTATUS(HTTP_GET), GETTRASHROOT(HTTP_GET),
APPEND(HTTP_POST), CONCAT(HTTP_POST), TRUNCATE(HTTP_POST),
CREATE(HTTP_PUT), MKDIRS(HTTP_PUT), RENAME(HTTP_PUT), SETOWNER(HTTP_PUT),
SETPERMISSION(HTTP_PUT), SETREPLICATION(HTTP_PUT), SETTIMES(HTTP_PUT),
@@ -819,6 +821,33 @@ public class HttpFSFileSystem extends FileSystem
}
/**
+ * Get the root directory of Trash for a path in HDFS.
+ * 1. File in encryption zone returns /ez1/.Trash/username.
+ * 2. File not in encryption zone, or encountered exception when checking
+ * the encryption zone of the path, returns /users/username/.Trash.
+ * Caller appends either Current or checkpoint timestamp
+ * for trash destination.
+ * The default implementation returns "/user/username/.Trash".
+ * @param fullPath the trash root of the path to be determined.
+ * @return trash root
+ */
+ @Override
+ public Path getTrashRoot(Path fullPath) {
+ Map<String, String> params = new HashMap<>();
+ params.put(OP_PARAM, Operation.GETTRASHROOT.toString());
+ try {
+ HttpURLConnection conn = getConnection(
+ Operation.GETTRASHROOT.getMethod(), params, fullPath, true);
+ HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
+ JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
+ return new Path((String) json.get(TRASH_DIR_JSON));
+ } catch (IOException ex) {
+ LOG.warn("Cannot find trash root of " + fullPath, ex);
+ return super.getTrashRoot(fullPath);
+ }
+ }
+
+ /**
* Set owner of a path (i.e. a file or a directory).
* The parameters username and groupname cannot both be null.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
index 001bc92..ebdb73e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
@@ -1057,6 +1057,29 @@ public class FSOperations {
}
/**
+ * Executor that performs getting trash root FileSystemAccess
+ * files system operation.
+ */
+ @InterfaceAudience.Private
+ public static class FSTrashRoot
+ implements FileSystemAccess.FileSystemExecutor<JSONObject> {
+ private Path path;
+ public FSTrashRoot(String path) {
+ this.path = new Path(path);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public JSONObject execute(FileSystem fs) throws IOException {
+ Path trashRoot = fs.getTrashRoot(this.path);
+ JSONObject json = new JSONObject();
+ json.put(HttpFSFileSystem.TRASH_DIR_JSON, trashRoot.toUri().getPath());
+ return json;
+ }
+
+ }
+
+ /**
* Executor that gets the ACL information for a given file.
*/
@InterfaceAudience.Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
index 25585c5..9e89405 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
@@ -60,6 +60,7 @@ public class HttpFSParametersProvider extends ParametersProvider {
PARAMS_DEF.put(Operation.GETFILECHECKSUM, new Class[]{});
PARAMS_DEF.put(Operation.GETFILEBLOCKLOCATIONS, new Class[]{});
PARAMS_DEF.put(Operation.GETACLSTATUS, new Class[]{});
+ PARAMS_DEF.put(Operation.GETTRASHROOT, new Class[]{});
PARAMS_DEF.put(Operation.INSTRUMENTATION, new Class[]{});
PARAMS_DEF.put(Operation.APPEND, new Class[]{DataParam.class});
PARAMS_DEF.put(Operation.CONCAT, new Class[]{SourcesParam.class});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
index 677bca7..3d81ac2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
@@ -339,6 +339,13 @@ public class HttpFSServer {
response = Response.ok(json).type(MediaType.APPLICATION_JSON).build();
break;
}
+ case GETTRASHROOT: {
+ FSOperations.FSTrashRoot command = new FSOperations.FSTrashRoot(path);
+ JSONObject json = fsExecute(user, command);
+ AUDIT_LOG.info("[{}]", path);
+ response = Response.ok(json).type(MediaType.APPLICATION_JSON).build();
+ break;
+ }
default: {
throw new IOException(
MessageFormat.format("Invalid HTTP GET operation [{0}]", op.value()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
index 3e8daa0..750b7f4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
@@ -40,6 +40,8 @@ HttpFS HTTP web-service API calls are HTTP REST calls that map to a HDFS file sy
* `$ curl http://httpfs-host:14000/webhdfs/v1/user/foo?op=list` returns the contents of the HDFS `/user/foo` directory in JSON format.
+* `$ curl http://httpfs-host:14000/webhdfs/v1/user/foo?op=GETTRASHROOT` returns the path `/user/foo/.Trash`, if `/` is an encrypted zone, returns the path `/.Trash/foo`. See [more details](../hadoop-project-dist/hadoop-hdfs/TransparentEncryption.html#Rename_and_Trash_considerations) about trash path in an encrypted zone.
+
* `$ curl -X POST http://httpfs-host:14000/webhdfs/v1/user/foo/bar?op=mkdirs` creates the HDFS `/user/foo.bar` directory.
User and Developer Documentation
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
index 4e25e56..e130c68 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
@@ -430,6 +430,35 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
new Path("/tmp").toUri().getPath());
}
+ private void testTrashRoot() throws Exception {
+ if (!isLocalFS()) {
+ FileSystem fs = FileSystem.get(getProxiedFSConf());
+
+ final Path rootDir = new Path("/");
+ final Path fooPath = new Path(getProxiedFSTestDir(), "foo.txt");
+ OutputStream os = fs.create(fooPath);
+ os.write(1);
+ os.close();
+
+ Path trashPath = fs.getTrashRoot(rootDir);
+ Path fooTrashPath = fs.getTrashRoot(fooPath);
+ fs.close();
+
+ fs = getHttpFSFileSystem();
+ Path httpFSTrashPath = fs.getTrashRoot(rootDir);
+ Path httpFSFooTrashPath = fs.getTrashRoot(fooPath);
+ fs.close();
+
+ assertEquals(trashPath.toUri().getPath(),
+ httpFSTrashPath.toUri().getPath());
+ assertEquals(fooTrashPath.toUri().getPath(),
+ httpFSFooTrashPath.toUri().getPath());
+ // trash path is related to USER, not path
+ assertEquals(trashPath.toUri().getPath(),
+ fooTrashPath.toUri().getPath());
+ }
+ }
+
private void testMkdirs() throws Exception {
Path path = new Path(getProxiedFSTestDir(), "foo");
FileSystem fs = getHttpFSFileSystem();
@@ -916,86 +945,90 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
GET, OPEN, CREATE, APPEND, TRUNCATE, CONCAT, RENAME, DELETE, LIST_STATUS,
WORKING_DIRECTORY, MKDIRS, SET_TIMES, SET_PERMISSION, SET_OWNER,
SET_REPLICATION, CHECKSUM, CONTENT_SUMMARY, FILEACLS, DIRACLS, SET_XATTR,
- GET_XATTRS, REMOVE_XATTR, LIST_XATTRS, ENCRYPTION, LIST_STATUS_BATCH
+ GET_XATTRS, REMOVE_XATTR, LIST_XATTRS, ENCRYPTION, LIST_STATUS_BATCH,
+ GETTRASHROOT
}
private void operation(Operation op) throws Exception {
switch (op) {
- case GET:
- testGet();
- break;
- case OPEN:
- testOpen();
- break;
- case CREATE:
- testCreate();
- break;
- case APPEND:
- testAppend();
- break;
- case TRUNCATE:
- testTruncate();
- break;
- case CONCAT:
- testConcat();
- break;
- case RENAME:
- testRename();
- break;
- case DELETE:
- testDelete();
- break;
- case LIST_STATUS:
- testListStatus();
- break;
- case WORKING_DIRECTORY:
- testWorkingdirectory();
- break;
- case MKDIRS:
- testMkdirs();
- break;
- case SET_TIMES:
- testSetTimes();
- break;
- case SET_PERMISSION:
- testSetPermission();
- break;
- case SET_OWNER:
- testSetOwner();
- break;
- case SET_REPLICATION:
- testSetReplication();
- break;
- case CHECKSUM:
- testChecksum();
- break;
- case CONTENT_SUMMARY:
- testContentSummary();
- break;
- case FILEACLS:
- testFileAcls();
- break;
- case DIRACLS:
- testDirAcls();
- break;
- case SET_XATTR:
- testSetXAttr();
- break;
- case REMOVE_XATTR:
- testRemoveXAttr();
- break;
- case GET_XATTRS:
- testGetXAttrs();
- break;
- case LIST_XATTRS:
- testListXAttrs();
- break;
- case ENCRYPTION:
- testEncryption();
- break;
- case LIST_STATUS_BATCH:
- testListStatusBatch();
- break;
+ case GET:
+ testGet();
+ break;
+ case OPEN:
+ testOpen();
+ break;
+ case CREATE:
+ testCreate();
+ break;
+ case APPEND:
+ testAppend();
+ break;
+ case TRUNCATE:
+ testTruncate();
+ break;
+ case CONCAT:
+ testConcat();
+ break;
+ case RENAME:
+ testRename();
+ break;
+ case DELETE:
+ testDelete();
+ break;
+ case LIST_STATUS:
+ testListStatus();
+ break;
+ case WORKING_DIRECTORY:
+ testWorkingdirectory();
+ break;
+ case MKDIRS:
+ testMkdirs();
+ break;
+ case SET_TIMES:
+ testSetTimes();
+ break;
+ case SET_PERMISSION:
+ testSetPermission();
+ break;
+ case SET_OWNER:
+ testSetOwner();
+ break;
+ case SET_REPLICATION:
+ testSetReplication();
+ break;
+ case CHECKSUM:
+ testChecksum();
+ break;
+ case CONTENT_SUMMARY:
+ testContentSummary();
+ break;
+ case FILEACLS:
+ testFileAcls();
+ break;
+ case DIRACLS:
+ testDirAcls();
+ break;
+ case SET_XATTR:
+ testSetXAttr();
+ break;
+ case REMOVE_XATTR:
+ testRemoveXAttr();
+ break;
+ case GET_XATTRS:
+ testGetXAttrs();
+ break;
+ case LIST_XATTRS:
+ testListXAttrs();
+ break;
+ case ENCRYPTION:
+ testEncryption();
+ break;
+ case LIST_STATUS_BATCH:
+ testListStatusBatch();
+ break;
+ case GETTRASHROOT:
+ testTrashRoot();
+ break;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
index a9c36b0..fcbddc4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
@@ -43,7 +43,6 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.XAttrCodec;
@@ -137,9 +136,9 @@ public class TestHttpFSServer extends HFSTestCase {
//HDFS configuration
File hadoopConfDir = new File(new File(homeDir, "conf"), "hadoop-conf");
hadoopConfDir.mkdirs();
- String fsDefaultName = TestHdfsHelper.getHdfsConf().get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY);
- Configuration conf = new Configuration(false);
- conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fsDefaultName);
+ Configuration hdfsConf = TestHdfsHelper.getHdfsConf();
+ // Http Server's conf should be based on HDFS's conf
+ Configuration conf = new Configuration(hdfsConf);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true);
File hdfsSite = new File(hadoopConfDir, "hdfs-site.xml");
@@ -347,6 +346,20 @@ public class TestHttpFSServer extends HFSTestCase {
}
/**
+ * Given the JSON output from the GETTRASHPATH call, return the
+ * 'path' value.
+ *
+ * @param statusJson JSON from GETTRASHPATH
+ * @return The value of 'path' in statusJson
+ * @throws Exception
+ */
+ private String getPath(String statusJson) throws Exception {
+ JSONParser parser = new JSONParser();
+ JSONObject details = (JSONObject) parser.parse(statusJson);
+ return (String) details.get("Path");
+ }
+
+ /**
* Given the JSON output from the GETACLSTATUS call, return the
* 'entries' value as a List<String>.
* @param statusJson JSON from GETACLSTATUS
@@ -675,6 +688,40 @@ public class TestHttpFSServer extends HFSTestCase {
@TestDir
@TestJetty
@TestHdfs
+ public void testGetTrashRoot() throws Exception {
+ String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
+ createHttpFSServer(false);
+ String trashJson = getStatus("/", "GETTRASHROOT");
+ String trashPath = getPath(trashJson);
+
+ Path expectedPath = new Path(FileSystem.USER_HOME_PREFIX,
+ new Path(user, FileSystem.TRASH_PREFIX));
+ Assert.assertEquals(expectedPath.toUri().getPath(), trashPath);
+
+ byte[] array = new byte[]{0, 1, 2, 3};
+ FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
+ fs.mkdirs(new Path("/tmp"));
+ OutputStream os = fs.create(new Path("/tmp/foo"));
+ os.write(array);
+ os.close();
+
+ trashJson = getStatus("/tmp/foo", "GETTRASHROOT");
+ trashPath = getPath(trashJson);
+ Assert.assertEquals(expectedPath.toUri().getPath(), trashPath);
+
+ //TestHdfsHelp has already set up EZ environment
+ final Path ezFile = TestHdfsHelper.ENCRYPTED_FILE;
+ final Path ezPath = TestHdfsHelper.ENCRYPTION_ZONE;
+ trashJson = getStatus(ezFile.toUri().getPath(), "GETTRASHROOT");
+ trashPath = getPath(trashJson);
+ expectedPath = new Path(ezPath, new Path(FileSystem.TRASH_PREFIX, user));
+ Assert.assertEquals(expectedPath.toUri().getPath(), trashPath);
+ }
+
+ @Test
+ @TestDir
+ @TestJetty
+ @TestHdfs
public void testDelegationTokenOperations() throws Exception {
createHttpFSServer(true);
@@ -754,6 +801,21 @@ public class TestHttpFSServer extends HFSTestCase {
conn = (HttpURLConnection) url.openConnection();
Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN,
conn.getResponseCode());
+
+ // getTrash test with delegation
+ url = new URL(TestJettyHelper.getJettyURL(),
+ "/webhdfs/v1/?op=GETTRASHROOT&delegation=" + tokenStr);
+ conn = (HttpURLConnection) url.openConnection();
+ Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN,
+ conn.getResponseCode());
+
+ url = new URL(TestJettyHelper.getJettyURL(),
+ "/webhdfs/v1/?op=GETTRASHROOT");
+ conn = (HttpURLConnection) url.openConnection();
+ conn.setRequestProperty("Cookie",
+ AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned);
+ Assert.assertEquals(HttpURLConnection.HTTP_OK,
+ conn.getResponseCode());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index cea3339..15195e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -1057,6 +1057,11 @@ public class NamenodeWebHdfsMethods {
np.checkAccess(fullpath, FsAction.getFsAction(fsAction.getValue()));
return Response.ok().build();
}
+ case GETTRASHROOT: {
+ final String trashPath = getTrashRoot(fullpath, conf);
+ final String jsonStr = JsonUtil.toJsonString("Path", trashPath);
+ return Response.ok(jsonStr).type(MediaType.APPLICATION_JSON).build();
+ }
case LISTSTATUS_BATCH:
{
byte[] start = HdfsFileStatus.EMPTY_NAME;
@@ -1072,6 +1077,13 @@ public class NamenodeWebHdfsMethods {
}
}
+ private static String getTrashRoot(String fullPath,
+ Configuration conf) throws IOException {
+ FileSystem fs = FileSystem.get(conf != null ? conf : new Configuration());
+ return fs.getTrashRoot(
+ new org.apache.hadoop.fs.Path(fullPath)).toUri().getPath();
+ }
+
private static DirectoryListing getDirectoryListing(final NamenodeProtocols np,
final String p, byte[] startAfter) throws IOException {
final DirectoryListing listing = np.getListing(p, startAfter, false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index f904bda..1d7d704 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -42,6 +42,7 @@ WebHDFS REST API
* [Get Content Summary of a Directory](#Get_Content_Summary_of_a_Directory)
* [Get File Checksum](#Get_File_Checksum)
* [Get Home Directory](#Get_Home_Directory)
+ * [Get Trash Root](#Get_Trash_Root)
* [Set Permission](#Set_Permission)
* [Set Owner](#Set_Owner)
* [Set Replication Factor](#Set_Replication_Factor)
@@ -149,6 +150,7 @@ The HTTP REST API supports the complete [FileSystem](../../api/org/apache/hadoop
* [`GETFILECHECKSUM`](#Get_File_Checksum) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileChecksum)
* [`GETHOMEDIRECTORY`](#Get_Home_Directory) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getHomeDirectory)
* [`GETDELEGATIONTOKEN`](#Get_Delegation_Token) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getDelegationToken)
+ * [`GETTRASHROOT`](#Get_Trash_Root) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getTrashRoot)
* [`GETXATTRS`](#Get_an_XAttr) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getXAttr)
* [`GETXATTRS`](#Get_multiple_XAttrs) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getXAttrs)
* [`GETXATTRS`](#Get_all_XAttrs) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getXAttrs)
@@ -578,7 +580,7 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileSt
"group" : "supergroup",
"length" : 0,
"modificationTime": 1320895981256,
- "owner" : "szetszwo",
+ "owner" : "username",
"pathSuffix" : "bar",
"permission" : "711",
"replication" : 0,
@@ -815,10 +817,36 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileCh
Content-Type: application/json
Transfer-Encoding: chunked
- {"Path": "/user/szetszwo"}
+ {"Path": "/user/username"}
See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getHomeDirectory
+### Get Trash Root
+
+* Submit a HTTP GET request.
+
+ curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETTRASHROOT"
+
+ The client receives a response with a [`Path` JSON object](#Path_JSON_Schema):
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+
+ {"Path": "/user/username/.Trash"}
+
+ if the path is an encrypted zone path and user has permission of the path, the client receives a response like this:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+
+ {"Path": "/PATH/.Trash/username"}
+
+See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getTrashRoot
+
+For more details about trash root in an encrypted zone, please refer to [Transparent Encryption Guide](./TransparentEncryption.html#Rename_and_Trash_considerations).
+
### Set Permission
* Submit a HTTP PUT request.
@@ -1137,7 +1165,7 @@ Snapshot Operations
Content-Type: application/json
Transfer-Encoding: chunked
- {"Path": "/user/szetszwo/.snapshot/s1"}
+ {"Path": "/user/username/.snapshot/s1"}
See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).createSnapshot
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index 8cce7ef..3a0586e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -78,6 +78,7 @@ import org.apache.hadoop.hdfs.tools.CryptoAdmin;
import org.apache.hadoop.hdfs.tools.DFSck;
import org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter;
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.hdfs.web.WebHdfsTestUtil;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.security.AccessControlException;
@@ -1558,6 +1559,25 @@ public class TestEncryptionZones {
DFSTestUtil.createFile(fs, nestedEZFile, len, (short) 1, 0xFEED);
verifyShellDeleteWithTrash(shell, topEZFile);
verifyShellDeleteWithTrash(shell, nestedEZFile);
+
+ //Test nested EZ with webHDFS
+ final WebHdfsFileSystem webFS = WebHdfsTestUtil.getWebHdfsFileSystem(
+ conf, WebHdfsConstants.WEBHDFS_SCHEME);
+ final String currentUser =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ final Path expectedTopTrash = new Path(topEZ,
+ new Path(FileSystem.TRASH_PREFIX, currentUser));
+ final Path expectedNestedTrash = new Path(nestedEZ,
+ new Path(FileSystem.TRASH_PREFIX, currentUser));
+
+ final Path topTrash = webFS.getTrashRoot(topEZFile);
+ final Path nestedTrash = webFS.getTrashRoot(nestedEZFile);
+
+ assertEquals(expectedTopTrash.toUri().getPath(),
+ topTrash.toUri().getPath());
+ assertEquals(expectedNestedTrash.toUri().getPath(),
+ nestedTrash.toUri().getPath());
+
verifyShellDeleteWithTrash(shell, nestedEZ);
verifyShellDeleteWithTrash(shell, topEZ);
}
@@ -1566,6 +1586,8 @@ public class TestEncryptionZones {
public void testRootDirEZTrash() throws Exception {
final HdfsAdmin dfsAdmin =
new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
+ final String currentUser =
+ UserGroupInformation.getCurrentUser().getShortUserName();
final Path rootDir = new Path("/");
dfsAdmin.createEncryptionZone(rootDir, TEST_KEY, NO_TRASH);
final Path encFile = new Path("/encFile");
@@ -1578,10 +1600,23 @@ public class TestEncryptionZones {
// Trash path should be consistent
// if root path is an encryption zone
- Path encFileTrash = shell.getCurrentTrashDir(encFile);
- Path rootDirTrash = shell.getCurrentTrashDir(rootDir);
+ Path encFileCurrentTrash = shell.getCurrentTrashDir(encFile);
+ Path rootDirCurrentTrash = shell.getCurrentTrashDir(rootDir);
assertEquals("Root trash should be equal with ezFile trash",
- encFileTrash, rootDirTrash);
+ encFileCurrentTrash, rootDirCurrentTrash);
+
+ // Use webHDFS client to test trash root path
+ final WebHdfsFileSystem webFS = WebHdfsTestUtil.getWebHdfsFileSystem(
+ conf, WebHdfsConstants.WEBHDFS_SCHEME);
+ final Path expectedTrash = new Path(rootDir,
+ new Path(FileSystem.TRASH_PREFIX, currentUser));
+
+ Path webHDFSTrash = webFS.getTrashRoot(encFile);
+ assertEquals(expectedTrash.toUri().getPath(),
+ webHDFSTrash.toUri().getPath());
+ assertEquals(encFileCurrentTrash.getParent().toUri().getPath(),
+ webHDFSTrash.toUri().getPath());
+
}
@Test(timeout = 120000)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab3dc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index 604a063..23d543d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -1072,4 +1072,26 @@ public class TestWebHDFS {
}
}
}
+
+ @Test
+ public void testGetTrashRoot() throws Exception {
+ MiniDFSCluster cluster = null;
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ final String currentUser =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ final WebHdfsFileSystem webFS = WebHdfsTestUtil.getWebHdfsFileSystem(
+ conf, WebHdfsConstants.WEBHDFS_SCHEME);
+
+ Path trashPath = webFS.getTrashRoot(new Path("/"));
+ Path expectedPath = new Path(FileSystem.USER_HOME_PREFIX,
+ new Path(currentUser, FileSystem.TRASH_PREFIX));
+ assertEquals(expectedPath.toUri().getPath(), trashPath.toUri().getPath());
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org