You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2015/08/17 20:49:09 UTC
[19/50] hadoop git commit: HDFS-8622. Implement GETCONTENTSUMMARY
operation for WebImageViewer. Contributed by Jagadesh Kiran N.
HDFS-8622. Implement GETCONTENTSUMMARY operation for WebImageViewer. Contributed by Jagadesh Kiran N.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40f81513
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40f81513
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40f81513
Branch: refs/heads/YARN-1197
Commit: 40f815131e822f5b7a8e6a6827f4b85b31220c43
Parents: 6cc8e38
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Aug 13 11:31:56 2015 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Aug 13 11:31:56 2015 +0900
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../offlineImageViewer/FSImageHandler.java | 3 +
.../tools/offlineImageViewer/FSImageLoader.java | 90 +++++++
.../src/site/markdown/HdfsImageViewer.md | 1 +
...TestOfflineImageViewerForContentSummary.java | 248 +++++++++++++++++++
5 files changed, 345 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f81513/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index df9b742..c88368e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -780,6 +780,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8887. Expose storage type and storage ID in BlockLocation. (wang)
+ HDFS-8622. Implement GETCONTENTSUMMARY operation for WebImageViewer.
+ (Jagadesh Kiran N via aajisaka)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f81513/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java
index da02805..d1b61d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java
@@ -107,6 +107,9 @@ class FSImageHandler extends SimpleChannelInboundHandler<HttpRequest> {
case "LISTXATTRS":
content = image.listXAttrs(path);
break;
+ case "GETCONTENTSUMMARY":
+ content = image.getContentSummary(path);
+ break;
default:
throw new IllegalArgumentException("Invalid value for webhdfs parameter"
+ " \"op\"");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f81513/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
index 2a11734..4d8eb35 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf;
import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeId;
import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.hdfs.web.resources.XAttrEncodingParam;
@@ -311,6 +312,95 @@ class FSImageLoader {
}
/**
+ * Return the JSON formatted ContentSummary of the specified path.
+ * @param path a path specifies a file or directory
+ * @return JSON formatted ContentSummary
+ * @throws IOException if failed to serialize ContentSummary to JSON.
+ */
+ String getContentSummary(String path) throws IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ return "{\"ContentSummary\":\n"
+ + mapper.writeValueAsString(getContentSummaryMap(path)) + "\n}\n";
+ }
+
+ private Map<String, Object> getContentSummaryMap(String path)
+ throws IOException {
+ long id = lookup(path);
+ INode inode = fromINodeId(id);
+ long spaceQuota = 0;
+ long nsQuota = 0;
+ long[] data = new long[4];
+ FsImageProto.INodeSection.INodeFile f = inode.getFile();
+ switch (inode.getType()) {
+ case FILE:
+ data[0] = 0;
+ data[1] = 1;
+ data[2] = getFileSize(f);
+ nsQuota = -1;
+ data[3] = data[2] * f.getReplication();
+ spaceQuota = -1;
+ return fillSummaryMap(spaceQuota, nsQuota, data);
+ case DIRECTORY:
+ fillDirSummary(id, data);
+ nsQuota = inode.getDirectory().getNsQuota();
+ spaceQuota = inode.getDirectory().getDsQuota();
+ return fillSummaryMap(spaceQuota, nsQuota, data);
+ case SYMLINK:
+ data[0] = 0;
+ data[1] = 1;
+ data[2] = 0;
+ nsQuota = -1;
+ data[3] = 0;
+ spaceQuota = -1;
+ return fillSummaryMap(spaceQuota, nsQuota, data);
+ default:
+ return null;
+ }
+
+ }
+
+ private Map<String, Object> fillSummaryMap(long spaceQuota,
+ long nsQuota, long[] data) {
+ Map<String, Object> map = Maps.newHashMap();
+ map.put("directoryCount", data[0]);
+ map.put("fileCount", data[1]);
+ map.put("length", data[2]);
+ map.put("quota", nsQuota);
+ map.put("spaceConsumed", data[3]);
+ map.put("spaceQuota", spaceQuota);
+ return map;
+ }
+
+ private void fillDirSummary(long id, long[] data) throws IOException {
+ data[0]++;
+ long[] children = dirmap.get(id);
+ if (children == null) {
+ return;
+ }
+
+ for (long cid : children) {
+ INode node = fromINodeId(cid);
+ switch (node.getType()) {
+ case DIRECTORY:
+ fillDirSummary(cid, data);
+ break;
+ case FILE:
+ FsImageProto.INodeSection.INodeFile f = node.getFile();
+ long curLength = getFileSize(f);
+ data[1]++;
+ data[2] += curLength;
+ data[3] += (curLength) * (f.getReplication());
+ break;
+ case SYMLINK:
+ data[1]++;
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ /**
* Return the JSON formatted XAttrNames of the specified file.
*
* @param path
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f81513/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
index 9b9d80a..155d3f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
@@ -100,6 +100,7 @@ The Web processor now supports the following operations:
* [GETACLSTATUS](./WebHDFS.html#Get_ACL_Status)
* [GETXATTRS](./WebHDFS.html#Get_an_XAttr)
* [LISTXATTRS](./WebHDFS.html#List_all_XAttrs)
+* [CONTENTSUMMARY] (./WebHDFS.html#Get_Content_Summary_of_a_Directory)
### XML Processor
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f81513/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForContentSummary.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForContentSummary.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForContentSummary.java
new file mode 100644
index 0000000..b758c26
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForContentSummary.java
@@ -0,0 +1,248 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import static org.junit.Assert.assertEquals;
+import java.io.File;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
+import org.apache.hadoop.net.NetUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests GETCONTENTSUMMARY operation for WebImageViewer
+ */
+public class TestOfflineImageViewerForContentSummary {
+
+ private static final Log LOG = LogFactory
+ .getLog(TestOfflineImageViewerForContentSummary.class);
+
+ private static File originalFsimage = null;
+ private static ContentSummary summaryFromDFS = null;
+ private static ContentSummary emptyDirSummaryFromDFS = null;
+ private static ContentSummary fileSummaryFromDFS = null;
+ private static ContentSummary symLinkSummaryFromDFS = null;
+ private static ContentSummary symLinkSummaryForDirContainsFromDFS=null;
+ /**
+ * Create a populated namespace for later testing. Save its contents to a
+ * data structure and store its fsimage location. We only want to generate
+ * the fsimage file once and use it for multiple tests.
+ */
+ @BeforeClass
+ public static void createOriginalFSImage() throws IOException {
+ MiniDFSCluster cluster = null;
+ Configuration conf = new Configuration();
+
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster.waitActive();
+ DistributedFileSystem hdfs = cluster.getFileSystem();
+ Path parentDir = new Path("/parentDir");
+ Path childDir1 = new Path(parentDir, "childDir1");
+ Path childDir2 = new Path(parentDir, "childDir2");
+ Path dirForLinks = new Path("/dirForLinks");
+ hdfs.mkdirs(parentDir);
+ hdfs.mkdirs(childDir1);
+ hdfs.mkdirs(childDir2);
+ hdfs.mkdirs(dirForLinks);
+ hdfs.setQuota(parentDir, 10, 1024*1024*1024);
+
+ Path file1OnParentDir = new Path(parentDir, "file1");
+ try (FSDataOutputStream o = hdfs.create(file1OnParentDir)) {
+ o.write("123".getBytes());
+ }
+ try (FSDataOutputStream o = hdfs.create(new Path(parentDir, "file2"))) {
+ o.write("1234".getBytes());
+ }
+ try (FSDataOutputStream o = hdfs.create(new Path(childDir1, "file3"))) {
+ o.write("123".getBytes());
+ }
+ try (FSDataOutputStream o = hdfs.create(new Path(parentDir, "file4"))) {
+ o.write("123".getBytes());
+ }
+ Path link1 = new Path("/link1");
+ Path link2 = new Path("/dirForLinks/linkfordir1");
+ hdfs.createSymlink(new Path("/parentDir/file4"), link1, true);
+ summaryFromDFS = hdfs.getContentSummary(parentDir);
+ emptyDirSummaryFromDFS = hdfs.getContentSummary(childDir2);
+ fileSummaryFromDFS = hdfs.getContentSummary(file1OnParentDir);
+ symLinkSummaryFromDFS = hdfs.getContentSummary(link1);
+ hdfs.createSymlink(childDir1, link2, true);
+ symLinkSummaryForDirContainsFromDFS = hdfs.getContentSummary(new Path(
+ "/dirForLinks"));
+ // Write results to the fsimage file
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER, false);
+ hdfs.saveNamespace();
+ // Determine the location of the fsimage file
+ originalFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil
+ .getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0));
+ if (originalFsimage == null) {
+ throw new RuntimeException("Didn't generate or can't find fsimage");
+ }
+ LOG.debug("original FS image file is " + originalFsimage);
+ } finally {
+ if (cluster != null)
+ cluster.shutdown();
+ }
+ }
+
+ @AfterClass
+ public static void deleteOriginalFSImage() {
+ if (originalFsimage != null && originalFsimage.exists()) {
+ originalFsimage.delete();
+ }
+ }
+
+ @Test
+ public void testGetContentSummaryForEmptyDirectory() throws Exception {
+ try (WebImageViewer viewer = new WebImageViewer(
+ NetUtils.createSocketAddr("localhost:0"))) {
+ viewer.initServer(originalFsimage.getAbsolutePath());
+ int port = viewer.getPort();
+ URL url = new URL("http://localhost:" + port
+ + "/webhdfs/v1/parentDir/childDir2?op=GETCONTENTSUMMARY");
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestMethod("GET");
+ connection.connect();
+ assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
+ // create a WebHdfsFileSystem instance
+ URI uri = new URI("webhdfs://localhost:" + String.valueOf(port));
+ Configuration conf = new Configuration();
+ WebHdfsFileSystem webfs = (WebHdfsFileSystem) FileSystem.get(uri, conf);
+ ContentSummary summary = webfs.getContentSummary(new Path(
+ "/parentDir/childDir2"));
+ verifyContentSummary(emptyDirSummaryFromDFS, summary);
+ }
+ }
+
+ @Test
+ public void testGetContentSummaryForDirectory() throws Exception {
+ try (WebImageViewer viewer = new WebImageViewer(
+ NetUtils.createSocketAddr("localhost:0"))) {
+ viewer.initServer(originalFsimage.getAbsolutePath());
+ int port = viewer.getPort();
+ URL url = new URL("http://localhost:" + port
+ + "/webhdfs/v1/parentDir/?op=GETCONTENTSUMMARY");
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestMethod("GET");
+ connection.connect();
+ assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
+ // create a WebHdfsFileSystem instance
+ URI uri = new URI("webhdfs://localhost:" + String.valueOf(port));
+ Configuration conf = new Configuration();
+ WebHdfsFileSystem webfs = (WebHdfsFileSystem) FileSystem.get(uri, conf);
+ ContentSummary summary = webfs.getContentSummary(new Path("/parentDir/"));
+ verifyContentSummary(summaryFromDFS, summary);
+ }
+ }
+
+ @Test
+ public void testGetContentSummaryForFile() throws Exception {
+ try (WebImageViewer viewer = new WebImageViewer(
+ NetUtils.createSocketAddr("localhost:0"))) {
+ viewer.initServer(originalFsimage.getAbsolutePath());
+ int port = viewer.getPort();
+ URL url = new URL("http://localhost:" + port
+ + "/webhdfs/v1/parentDir/file1?op=GETCONTENTSUMMARY");
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestMethod("GET");
+ connection.connect();
+ assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
+ // create a WebHdfsFileSystem instance
+ URI uri = new URI("webhdfs://localhost:" + String.valueOf(port));
+ Configuration conf = new Configuration();
+ WebHdfsFileSystem webfs = (WebHdfsFileSystem) FileSystem.get(uri, conf);
+ ContentSummary summary = webfs.
+ getContentSummary(new Path("/parentDir/file1"));
+ verifyContentSummary(fileSummaryFromDFS, summary);
+ }
+ }
+
+ @Test
+ public void testGetContentSummaryForSymlink() throws Exception {
+ try (WebImageViewer viewer = new WebImageViewer(
+ NetUtils.createSocketAddr("localhost:0"))) {
+ viewer.initServer(originalFsimage.getAbsolutePath());
+ int port = viewer.getPort();
+ // create a WebHdfsFileSystem instance
+ URI uri = new URI("webhdfs://localhost:" + String.valueOf(port));
+ Configuration conf = new Configuration();
+ WebHdfsFileSystem webfs = (WebHdfsFileSystem) FileSystem.get(uri, conf);
+ ContentSummary summary = webfs.getContentSummary(new Path("/link1"));
+ verifyContentSummary(symLinkSummaryFromDFS, summary);
+ }
+ }
+
+ @Test
+ public void testGetContentSummaryForDirContainsSymlink() throws Exception {
+ try (WebImageViewer viewer = new WebImageViewer(
+ NetUtils.createSocketAddr("localhost:0"))) {
+ viewer.initServer(originalFsimage.getAbsolutePath());
+ int port = viewer.getPort();
+ // create a WebHdfsFileSystem instance
+ URI uri = new URI("webhdfs://localhost:" + String.valueOf(port));
+ Configuration conf = new Configuration();
+ WebHdfsFileSystem webfs = (WebHdfsFileSystem) FileSystem.get(uri, conf);
+ ContentSummary summary = webfs.getContentSummary(new Path(
+ "/dirForLinks/"));
+ verifyContentSummary(symLinkSummaryForDirContainsFromDFS, summary);
+ }
+ }
+
+ private void verifyContentSummary(ContentSummary expected,
+ ContentSummary actual) {
+ assertEquals(expected.getDirectoryCount(), actual.getDirectoryCount());
+ assertEquals(expected.getFileCount(), actual.getFileCount());
+ assertEquals(expected.getLength(), actual.getLength());
+ assertEquals(expected.getSpaceConsumed(), actual.getSpaceConsumed());
+ assertEquals(expected.getQuota(), actual.getQuota());
+ assertEquals(expected.getSpaceQuota(), actual.getSpaceQuota());
+ }
+
+ @Test
+ public void testGetContentSummaryResponseCode() throws Exception {
+ try (WebImageViewer viewer = new WebImageViewer(
+ NetUtils.createSocketAddr("localhost:0"))) {
+ viewer.initServer(originalFsimage.getAbsolutePath());
+ int port = viewer.getPort();
+ URL url = new URL("http://localhost:" + port
+ + "/webhdfs/v1/dir123/?op=GETCONTENTSUMMARY");
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestMethod("GET");
+ connection.connect();
+ assertEquals(HttpURLConnection.HTTP_NOT_FOUND,
+ connection.getResponseCode());
+ }
+ }
+}