You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ki...@apache.org on 2015/01/26 15:16:26 UTC
hadoop git commit: HDFS-7224. Allow reuse of NN connections via
webhdfs. Contributed by Eric Payne (cherry picked from commit
2b0fa20f69417326a92beac10ffa072db2616e73)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 2551cc6e6 -> c081fa6ae
HDFS-7224. Allow reuse of NN connections via webhdfs. Contributed by Eric Payne
(cherry picked from commit 2b0fa20f69417326a92beac10ffa072db2616e73)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c081fa6a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c081fa6a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c081fa6a
Branch: refs/heads/branch-2
Commit: c081fa6ae8fbd38eb3b6fb2ba0aa3ba1ac779240
Parents: 2551cc6
Author: Kihwal Lee <ki...@apache.org>
Authored: Mon Jan 26 08:15:37 2015 -0600
Committer: Kihwal Lee <ki...@apache.org>
Committed: Mon Jan 26 08:15:37 2015 -0600
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 20 ++++----
.../hdfs/web/TestFSMainOperationsWebHdfs.java | 49 ++++++++++++++++++++
3 files changed, 64 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c081fa6a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 82d933c..1220678 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -265,6 +265,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7623. Add htrace configuration properties to core-default.xml and
update user doc about how to enable htrace. (yliu)
+ HDFS-7224. Allow reuse of NN connections via webhdfs (Eric Payne via
+ kihwal)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c081fa6a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 7284b55..2a3faf7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -313,16 +313,20 @@ public class WebHdfsFileSystem extends FileSystem
if (in == null) {
throw new IOException("The " + (useErrorStream? "error": "input") + " stream is null.");
}
- final String contentType = c.getContentType();
- if (contentType != null) {
- final MediaType parsed = MediaType.valueOf(contentType);
- if (!MediaType.APPLICATION_JSON_TYPE.isCompatible(parsed)) {
- throw new IOException("Content-Type \"" + contentType
- + "\" is incompatible with \"" + MediaType.APPLICATION_JSON
- + "\" (parsed=\"" + parsed + "\")");
+ try {
+ final String contentType = c.getContentType();
+ if (contentType != null) {
+ final MediaType parsed = MediaType.valueOf(contentType);
+ if (!MediaType.APPLICATION_JSON_TYPE.isCompatible(parsed)) {
+ throw new IOException("Content-Type \"" + contentType
+ + "\" is incompatible with \"" + MediaType.APPLICATION_JSON
+ + "\" (parsed=\"" + parsed + "\")");
+ }
}
+ return (Map<?, ?>)JSON.parse(new InputStreamReader(in, Charsets.UTF_8));
+ } finally {
+ in.close();
}
- return (Map<?, ?>)JSON.parse(new InputStreamReader(in, Charsets.UTF_8));
}
private static Map<?, ?> validateResponse(final HttpOpParam.Op op,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c081fa6a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestFSMainOperationsWebHdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestFSMainOperationsWebHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestFSMainOperationsWebHdfs.java
index a5bb41d..295aff0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestFSMainOperationsWebHdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestFSMainOperationsWebHdfs.java
@@ -17,8 +17,14 @@
*/
package org.apache.hadoop.hdfs.web;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.doReturn;
+
import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
import java.net.URI;
+import java.net.URL;
import java.security.PrivilegedExceptionAction;
import org.apache.commons.logging.impl.Log4JLogger;
@@ -32,6 +38,8 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.web.resources.ExceptionHandler;
+import org.apache.hadoop.hdfs.web.resources.GetOpParam;
+import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.log4j.Level;
@@ -128,6 +136,47 @@ public class TestFSMainOperationsWebHdfs extends FSMainOperationsBaseTest {
Assert.assertEquals(1024*4, fileStatus.getLen());
}
+ // Test that WebHdfsFileSystem.jsonParse() closes the connection's input
+ // stream.
+ // Closing the inputstream in jsonParse will allow WebHDFS to reuse
+ // connections to the namenode rather than needing to always open new ones.
+ boolean closedInputStream = false;
+ @Test
+ public void testJsonParseClosesInputStream() throws Exception {
+ final WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)fileSystem;
+ Path file = getTestRootPath(fSys, "test/hadoop/file");
+ createFile(file);
+ final HttpOpParam.Op op = GetOpParam.Op.GETHOMEDIRECTORY;
+ final URL url = webhdfs.toUrl(op, file);
+ final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.setRequestMethod(op.getType().toString());
+ conn.connect();
+
+ InputStream myIn = new InputStream(){
+ private HttpURLConnection localConn = conn;
+ @Override
+ public void close() throws IOException {
+ closedInputStream = true;
+ localConn.getInputStream().close();
+ }
+ @Override
+ public int read() throws IOException {
+ return localConn.getInputStream().read();
+ }
+ };
+ final HttpURLConnection spyConn = spy(conn);
+ doReturn(myIn).when(spyConn).getInputStream();
+
+ try {
+ Assert.assertFalse(closedInputStream);
+ WebHdfsFileSystem.jsonParse(spyConn, false);
+ Assert.assertTrue(closedInputStream);
+ } catch(IOException ioe) {
+ junit.framework.TestCase.fail();
+ }
+ conn.disconnect();
+ }
+
@Override
@Test
public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception {