You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by jg...@apache.org on 2010/07/27 03:04:16 UTC
svn commit: r979489 - in /hadoop/hdfs/trunk: ./
src/java/org/apache/hadoop/hdfs/server/common/
src/java/org/apache/hadoop/hdfs/server/namenode/
Author: jghoman
Date: Tue Jul 27 01:04:15 2010
New Revision: 979489
URL: http://svn.apache.org/viewvc?rev=979489&view=rev
Log:
HDFS-1178. The NameNode servlets should not use RPC to connect to the NameNode. Contributed by Kan Zhang.
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryServlet.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/ListPathsServlet.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=979489&r1=979488&r2=979489&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Tue Jul 27 01:04:15 2010
@@ -91,6 +91,9 @@ Trunk (unreleased changes)
HDFS-1315. Add fsck event to audit log and remove other audit log events
corresponding to FSCK listStatus and open calls. (suresh)
+ HDFS-1178. The NameNode servlets should not use RPC to connect to the
+ NameNode. (Kan Zhang via jghoman)
+
OPTIMIZATIONS
HDFS-1140. Speedup INode.getPathComponents. (Dmytro Molkov via shv)
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/JspHelper.java?rev=979489&r1=979488&r2=979489&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/JspHelper.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/JspHelper.java Tue Jul 27 01:04:15 2010
@@ -18,6 +18,8 @@
package org.apache.hadoop.hdfs.server.common;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.InetSocketAddress;
@@ -470,7 +472,6 @@ public class JspHelper {
Configuration conf
) throws IOException {
UserGroupInformation ugi = null;
- final String RANDOM_USER = "webuser1234";
if(UserGroupInformation.isSecurityEnabled()) {
String user = request.getRemoteUser();
String tokenString = request.getParameter(DELEGATION_PARAMETER_NAME);
@@ -478,19 +479,18 @@ public class JspHelper {
Token<DelegationTokenIdentifier> token =
new Token<DelegationTokenIdentifier>();
token.decodeFromUrlString(tokenString);
- if (user == null) {
- //this really doesn't break any security since we use the
- //delegation token for authentication in
- //the back end.
- user = RANDOM_USER;
- }
InetSocketAddress serviceAddr = NameNode.getAddress(conf);
LOG.info("Setting service in token: "
+ new Text(serviceAddr.getAddress().getHostAddress() + ":"
+ serviceAddr.getPort()));
token.setService(new Text(serviceAddr.getAddress().getHostAddress()
+ ":" + serviceAddr.getPort()));
- ugi = UserGroupInformation.createRemoteUser(user);
+ ByteArrayInputStream buf = new ByteArrayInputStream(token
+ .getIdentifier());
+ DataInputStream in = new DataInputStream(buf);
+ DelegationTokenIdentifier id = new DelegationTokenIdentifier();
+ id.readFields(in);
+ ugi = id.getUser();
ugi.addToken(token);
ugi.setAuthenticationMethod(AuthenticationMethod.TOKEN);
} else {
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryServlet.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryServlet.java?rev=979489&r1=979488&r2=979489&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryServlet.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryServlet.java Tue Jul 27 01:04:15 2010
@@ -30,7 +30,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.server.common.JspHelper;
-import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.security.UserGroupInformation;
import org.znerd.xmlenc.XMLOutputter;
@@ -47,9 +46,9 @@ public class ContentSummaryServlet exten
(Configuration) getServletContext().getAttribute(JspHelper.CURRENT_CONF);
final UserGroupInformation ugi = getUGI(request, conf);
try {
- ugi.doAs(new PrivilegedExceptionAction<Object>() {
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
- public Object run() throws Exception {
+ public Void run() throws Exception {
final String path = request.getPathInfo();
final PrintWriter out = response.getWriter();
@@ -72,8 +71,7 @@ public class ContentSummaryServlet exten
}
xml.endTag();
} catch(IOException ioe) {
- new RemoteException(ioe.getClass().getName(), ioe.getMessage()
- ).writeXml(path, xml);
+ writeXml(ioe, path, xml);
}
xml.endDocument();
return null;
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java?rev=979489&r1=979488&r2=979489&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java Tue Jul 27 01:04:15 2010
@@ -36,7 +36,9 @@ import org.apache.hadoop.hdfs.protocol.D
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.security.UserGroupInformation;
+import org.znerd.xmlenc.XMLOutputter;
/**
* A base class for the servlets in DFS.
@@ -47,11 +49,36 @@ abstract class DfsServlet extends HttpSe
static final Log LOG = LogFactory.getLog(DfsServlet.class.getCanonicalName());
+ /** Write the object to XML format */
+ protected void writeXml(Exception except, String path, XMLOutputter doc)
+ throws IOException {
+ doc.startTag(RemoteException.class.getSimpleName());
+ doc.attribute("path", path);
+ if (except instanceof RemoteException) {
+ doc.attribute("class", ((RemoteException) except).getClassName());
+ } else {
+ doc.attribute("class", except.getClass().getName());
+ }
+ String msg = except.getLocalizedMessage();
+ int i = msg.indexOf("\n");
+ if (i >= 0) {
+ msg = msg.substring(0, i);
+ }
+ doc.attribute("message", msg.substring(msg.indexOf(":") + 1).trim());
+ doc.endTag();
+ }
+
/**
* Create a {@link NameNode} proxy from the current {@link ServletContext}.
*/
protected ClientProtocol createNameNodeProxy() throws IOException {
ServletContext context = getServletContext();
+ // if we are running in the Name Node, use it directly rather than via
+ // rpc
+ NameNode nn = (NameNode) context.getAttribute("name.node");
+ if (nn != null) {
+ return nn;
+ }
InetSocketAddress nnAddr = (InetSocketAddress)context.getAttribute("name.node.address");
Configuration conf = new HdfsConfiguration(
(Configuration)context.getAttribute(JspHelper.CURRENT_CONF));
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java?rev=979489&r1=979488&r2=979489&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java Tue Jul 27 01:04:15 2010
@@ -40,7 +40,6 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.znerd.xmlenc.XMLOutputter;
@@ -107,12 +106,9 @@ public class FileChecksumServlets {
filename, nnproxy, socketFactory, socketTimeout);
MD5MD5CRC32FileChecksum.write(xml, checksum);
} catch(IOException ioe) {
- new RemoteException(ioe.getClass().getName(), ioe.getMessage()
- ).writeXml(filename, xml);
+ writeXml(ioe, filename, xml);
} catch (InterruptedException e) {
- new RemoteException(e.getClass().getName(), e.getMessage()
- ).writeXml(filename, xml);
-
+ writeXml(e, filename, xml);
}
xml.endDocument();
}
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java?rev=979489&r1=979488&r2=979489&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java Tue Jul 27 01:04:15 2010
@@ -94,39 +94,40 @@ public class FileDataServlet extends Dfs
* GET http://<nn>:<port>/data[/<path>] HTTP/1.1
* }
*/
- public void doGet(HttpServletRequest request, HttpServletResponse response)
+ public void doGet(final HttpServletRequest request,
+ final HttpServletResponse response)
throws IOException {
final Configuration conf =
(Configuration) getServletContext().getAttribute(JspHelper.CURRENT_CONF);
final UserGroupInformation ugi = getUGI(request, conf);
try {
- final ClientProtocol nnproxy = ugi
- .doAs(new PrivilegedExceptionAction<ClientProtocol>() {
- @Override
- public ClientProtocol run() throws IOException {
- return createNameNodeProxy();
- }
- });
-
- final String path =
- request.getPathInfo() != null ? request.getPathInfo() : "/";
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws IOException {
+ ClientProtocol nn = createNameNodeProxy();
+ final String path = request.getPathInfo() != null ? request
+ .getPathInfo() : "/";
- String delegationToken =
- request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME);
+ String delegationToken = request
+ .getParameter(JspHelper.DELEGATION_PARAMETER_NAME);
- HdfsFileStatus info = nnproxy.getFileInfo(path);
- if ((info != null) && !info.isDir()) {
- String redirect = createUri(path, info, ugi, nnproxy,
- request, delegationToken).toURL().toString();
- response.sendRedirect(redirect);
- } else if (info == null){
- response.sendError(400, "cat: File not found " + path);
- } else {
- response.sendError(400, "cat: " + path + ": is a directory");
- }
- } catch (URISyntaxException e) {
- response.getWriter().println(e.toString());
+ HdfsFileStatus info = nn.getFileInfo(path);
+ if ((info != null) && !info.isDir()) {
+ try {
+ response.sendRedirect(createUri(path, info, ugi, nn, request,
+ delegationToken).toURL().toString());
+ } catch (URISyntaxException e) {
+ response.getWriter().println(e.toString());
+ }
+ } else if (info == null) {
+ response.sendError(400, "cat: File not found " + path);
+ } else {
+ response.sendError(400, "cat: " + path + ": is a directory");
+ }
+ return null;
+ }
+ });
} catch (IOException e) {
response.sendError(400, e.getMessage());
} catch (InterruptedException e) {
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/ListPathsServlet.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/ListPathsServlet.java?rev=979489&r1=979488&r2=979489&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/ListPathsServlet.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/ListPathsServlet.java Tue Jul 27 01:04:15 2010
@@ -25,7 +25,6 @@ import org.apache.hadoop.hdfs.protocol.C
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.server.common.JspHelper;
-import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.util.VersionInfo;
import org.znerd.xmlenc.*;
@@ -141,60 +140,61 @@ public class ListPathsServlet extends Df
final Configuration conf =
(Configuration) getServletContext().getAttribute(JspHelper.CURRENT_CONF);
- ClientProtocol nnproxy = getUGI(request, conf).doAs
- (new PrivilegedExceptionAction<ClientProtocol>() {
+ getUGI(request, conf).doAs(new PrivilegedExceptionAction<Void>() {
@Override
- public ClientProtocol run() throws IOException {
- return createNameNodeProxy();
+ public Void run() throws IOException {
+ ClientProtocol nn = createNameNodeProxy();
+ doc.declaration();
+ doc.startTag("listing");
+ for (Map.Entry<String, String> m : root.entrySet()) {
+ doc.attribute(m.getKey(), m.getValue());
+ }
+
+ HdfsFileStatus base = nn.getFileInfo(path);
+ if ((base != null) && base.isDir()) {
+ writeInfo(path, base, doc);
+ }
+
+ Stack<String> pathstack = new Stack<String>();
+ pathstack.push(path);
+ while (!pathstack.empty()) {
+ String p = pathstack.pop();
+ try {
+ byte[] lastReturnedName = HdfsFileStatus.EMPTY_NAME;
+ DirectoryListing thisListing;
+ do {
+ assert lastReturnedName != null;
+ thisListing = nn.getListing(p, lastReturnedName);
+ if (thisListing == null) {
+ if (lastReturnedName.length == 0) {
+ LOG
+ .warn("ListPathsServlet - Path " + p
+ + " does not exist");
+ }
+ break;
+ }
+ HdfsFileStatus[] listing = thisListing.getPartialListing();
+ for (HdfsFileStatus i : listing) {
+ String localName = i.getLocalName();
+ if (exclude.matcher(localName).matches()
+ || !filter.matcher(localName).matches()) {
+ continue;
+ }
+ if (recur && i.isDir()) {
+ pathstack.push(new Path(p, localName).toUri().getPath());
+ }
+ writeInfo(p, i, doc);
+ }
+ lastReturnedName = thisListing.getLastName();
+ } while (thisListing.hasMore());
+ } catch (IOException re) {
+ writeXml(re, p, doc);
+ }
+ }
+ doc.endDocument();
+ return null;
}
});
-
- doc.declaration();
- doc.startTag("listing");
- for (Map.Entry<String,String> m : root.entrySet()) {
- doc.attribute(m.getKey(), m.getValue());
- }
-
- HdfsFileStatus base = nnproxy.getFileInfo(path);
- if ((base != null) && base.isDir()) {
- writeInfo(path, base, doc);
- }
-
- Stack<String> pathstack = new Stack<String>();
- pathstack.push(path);
- while (!pathstack.empty()) {
- String p = pathstack.pop();
- try {
- byte[] lastReturnedName = HdfsFileStatus.EMPTY_NAME;
- DirectoryListing thisListing;
- do {
- assert lastReturnedName != null;
- thisListing = nnproxy.getListing(p, lastReturnedName);
- if (thisListing == null) {
- if (lastReturnedName.length == 0) {
- LOG.warn("ListPathsServlet - Path " + p + " does not exist");
- }
- break;
- }
- HdfsFileStatus[] listing = thisListing.getPartialListing();
- for (HdfsFileStatus i : listing) {
- String localName = i.getLocalName();
- if (exclude.matcher(localName).matches()
- || !filter.matcher(localName).matches()) {
- continue;
- }
- if (recur && i.isDir()) {
- pathstack.push(new Path(p, localName).toUri().getPath());
- }
- writeInfo(p, i, doc);
- }
- lastReturnedName = thisListing.getLastName();
- } while (thisListing.hasMore());
- } catch(RemoteException re) {re.writeXml(p, doc);}
- }
- if (doc != null) {
- doc.endDocument();
- }
} catch (InterruptedException e) {
LOG.warn("ListPathServlet encountered InterruptedException", e);
response.sendError(400, e.getMessage());