You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2008/03/19 02:44:53 UTC
svn commit: r638677 - in /hadoop/core/trunk: ./ conf/
src/java/org/apache/hadoop/dfs/ src/java/org/apache/hadoop/mapred/
Author: cdouglas
Date: Tue Mar 18 18:44:50 2008
New Revision: 638677
URL: http://svn.apache.org/viewvc?rev=638677&view=rev
Log:
HADOOP-2239. Add HsftpFileSystem to permit transferring files over ssl.
Added:
hadoop/core/trunk/conf/sslinfo.xml.example
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HsftpFileSystem.java
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/conf/hadoop-default.xml
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FileDataServlet.java
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/mapred/StatusHttpServer.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=638677&r1=638676&r2=638677&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Mar 18 18:44:50 2008
@@ -108,6 +108,9 @@
HADOOP-2939. Make the automated patch testing process an executable
Ant target, test-patch. (nigel)
+ HADOOP-2239. Add HsftpFileSystem to permit transferring files over ssl.
+ (cdouglas)
+
OPTIMIZATIONS
HADOOP-2790. Fixed inefficient method hasSpeculativeTask by removing
Modified: hadoop/core/trunk/conf/hadoop-default.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/conf/hadoop-default.xml?rev=638677&r1=638676&r2=638677&view=diff
==============================================================================
--- hadoop/core/trunk/conf/hadoop-default.xml (original)
+++ hadoop/core/trunk/conf/hadoop-default.xml Tue Mar 18 18:44:50 2008
@@ -175,6 +175,11 @@
</property>
<property>
+ <name>fs.hsftp.impl</name>
+ <value>org.apache.hadoop.dfs.HsftpFileSystem</value>
+</property>
+
+<property>
<name>fs.ramfs.impl</name>
<value>org.apache.hadoop.fs.InMemoryFileSystem</value>
<description>The FileSystem for ramfs: uris.</description>
@@ -242,6 +247,24 @@
<description>
The address and the base port where the dfs namenode web ui will listen on.
If the port is 0 then the server will start on a free port.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.https.address</name>
+ <value>0.0.0.0:50475</value>
+</property>
+
+<property>
+ <name>dfs.https.address</name>
+ <value>0.0.0.0:50470</value>
+</property>
+
+<property>
+ <name>https.keystore.info.rsrc</name>
+ <value>sslinfo.xml</value>
+ <description>The name of the resource from which ssl keystore information
+ will be extracted
</description>
</property>
Added: hadoop/core/trunk/conf/sslinfo.xml.example
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/conf/sslinfo.xml.example?rev=638677&view=auto
==============================================================================
--- hadoop/core/trunk/conf/sslinfo.xml.example (added)
+++ hadoop/core/trunk/conf/sslinfo.xml.example Tue Mar 18 18:44:50 2008
@@ -0,0 +1,21 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+<property>
+ <name>https.keystore.location</name>
+ <value>${user.home}/.keystore</value>
+</property>
+
+<property>
+ <name>https.keystore.password</name>
+ <value>changeme</value>
+</property>
+
+<property>
+ <name>https.keystore.keypassword</name>
+ <value>changeme</value>
+</property>
+
+</configuration>
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java?rev=638677&r1=638676&r2=638677&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java Tue Mar 18 18:44:50 2008
@@ -311,6 +311,16 @@
String infoHost = infoSocAddr.getHostName();
int tmpInfoPort = infoSocAddr.getPort();
this.infoServer = new StatusHttpServer("datanode", infoHost, tmpInfoPort, tmpInfoPort == 0);
+ InetSocketAddress secInfoSocAddr = NetUtils.createSocketAddr(
+ conf.get("dfs.datanode.https.address", infoHost + ":" + 0));
+ Configuration sslConf = new Configuration(conf);
+ sslConf.addResource(conf.get("https.keystore.info.rsrc", "sslinfo.xml"));
+ String keyloc = sslConf.get("https.keystore.location");
+ if (null != keyloc) {
+ this.infoServer.addSslListener(secInfoSocAddr, keyloc,
+ sslConf.get("https.keystore.password", ""),
+ sslConf.get("https.keystore.keypassword", ""));
+ }
this.infoServer.addServlet(null, "/streamFile/*", StreamFile.class);
this.infoServer.setAttribute("datanode.blockScanner", blockScanner);
this.infoServer.addServlet(null, "/blockScannerReport",
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java?rev=638677&r1=638676&r2=638677&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Tue Mar 18 18:44:50 2008
@@ -310,6 +310,21 @@
int tmpInfoPort = infoSocAddr.getPort();
this.infoServer = new StatusHttpServer("dfs", infoHost, tmpInfoPort,
tmpInfoPort == 0);
+ InetSocketAddress secInfoSocAddr = NetUtils.createSocketAddr(
+ conf.get("dfs.https.address", infoHost + ":" + 0));
+ Configuration sslConf = new Configuration(conf);
+ sslConf.addResource(conf.get("https.keystore.info.rsrc", "sslinfo.xml"));
+ String keyloc = sslConf.get("https.keystore.location");
+ if (null != keyloc) {
+ this.infoServer.addSslListener(secInfoSocAddr, keyloc,
+ sslConf.get("https.keystore.password", ""),
+ sslConf.get("https.keystore.keypassword", ""));
+ }
+ // assume same ssl port for all datanodes
+ InetSocketAddress datanodeSslPort = NetUtils.createSocketAddr(
+ conf.get("dfs.datanode.https.address", infoHost + ":" + 50475));
+ this.infoServer.setAttribute("datanode.https.port",
+ datanodeSslPort.getPort());
this.infoServer.setAttribute("name.system", this);
this.infoServer.setAttribute("name.node", nn);
this.infoServer.setAttribute("name.conf", conf);
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FileDataServlet.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FileDataServlet.java?rev=638677&r1=638676&r2=638677&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FileDataServlet.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FileDataServlet.java Tue Mar 18 18:44:50 2008
@@ -34,11 +34,16 @@
* @see org.apache.hadoop.dfs.HftpFileSystem
*/
public class FileDataServlet extends DfsServlet {
- private static URI createUri(DFSFileInfo i, UnixUserGroupInformation ugi,
- ClientProtocol nnproxy) throws IOException, URISyntaxException {
+
+ private URI createUri(DFSFileInfo i, UnixUserGroupInformation ugi,
+ ClientProtocol nnproxy, String scheme)
+ throws IOException, URISyntaxException {
final DatanodeInfo host = pickSrcDatanode(i, nnproxy);
- return new URI("http", null, host.getHostName(), host.getInfoPort(),
- "/streamFile", "filename=" + i.getPath() + "&ugi=" + ugi, null);
+ return new URI(scheme, null, host.getHostName(),
+ "https".equals(scheme)
+ ? (Integer)getServletContext().getAttribute("datanode.https.port")
+ : host.getInfoPort(),
+ "/streamFile", "filename=" + i.getPath() + "&ugi=" + ugi, null);
}
private final static int BLOCK_SAMPLE = 5;
@@ -97,7 +102,8 @@
? request.getPathInfo() : "/";
DFSFileInfo info = nnproxy.getFileInfo(path);
if (!info.isDir()) {
- response.sendRedirect(createUri(info, ugi, nnproxy).toURL().toString());
+ response.sendRedirect(createUri(info, ugi, nnproxy,
+ request.getScheme()).toURL().toString());
} else {
response.sendError(400, "cat: " + path + ": is a directory");
}
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java?rev=638677&r1=638676&r2=638677&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java Tue Mar 18 18:44:50 2008
@@ -23,6 +23,7 @@
import java.io.IOException;
import java.net.HttpURLConnection;
+import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
@@ -64,10 +65,10 @@
HttpURLConnection.setFollowRedirects(true);
}
- private String fshostname = "";
- private int fsport = -1;
+ protected InetSocketAddress nnAddr;
+ protected UserGroupInformation ugi;
+
protected static final SimpleDateFormat df = ListPathsServlet.df;
- private UserGroupInformation ugi;
@Override
public void initialize(URI name, Configuration conf) throws IOException {
@@ -76,43 +77,43 @@
this.ugi = UnixUserGroupInformation.login(conf);
} catch (LoginException le) {
throw new IOException(StringUtils.stringifyException(le));
- }
+ }
- this.fshostname = name.getHost();
- this.fsport = name.getPort();
- if(fsport >= 0)
- return;
- String infoAddr =
- NetUtils.getServerAddress(conf,
- "dfs.info.bindAddress",
- "dfs.info.port",
- "dfs.http.address");
- this.fsport = NetUtils.createSocketAddr(infoAddr).getPort();
+ nnAddr = NetUtils.createSocketAddr(name.toString());
}
@Override
public URI getUri() {
try {
- return new URI("hftp", null, fshostname, fsport, null, null, null);
+ return new URI("hftp", null, nnAddr.getHostName(), nnAddr.getPort(),
+ null, null, null);
} catch (URISyntaxException e) {
return null;
}
}
- @Override
- public FSDataInputStream open(Path f, int buffersize) throws IOException {
- HttpURLConnection connection = null;
+ /**
+ * Open an HTTP connection to the namenode to read file data and metadata.
+ * @param path The path component of the URL
+ * @param query The query component of the URL
+ */
+ protected HttpURLConnection openConnection(String path, String query)
+ throws IOException {
try {
- final URL url = new URI("http", null, fshostname, fsport,
- "/data" + f.toUri().getPath(), "ugi=" + ugi, null).toURL();
- connection = (HttpURLConnection)url.openConnection();
- connection.setRequestMethod("GET");
- connection.connect();
+ final URL url = new URI("http", null, nnAddr.getHostName(),
+ nnAddr.getPort(), path, query, null).toURL();
+ return (HttpURLConnection)url.openConnection();
} catch (URISyntaxException e) {
- IOException ie = new IOException("invalid url");
- ie.initCause(e);
- throw ie;
+ throw (IOException)new IOException().initCause(e);
}
+ }
+
+ @Override
+ public FSDataInputStream open(Path f, int buffersize) throws IOException {
+ HttpURLConnection connection = null;
+ connection = openConnection("/data" + f.toUri().getPath(), "ugi=" + ugi);
+ connection.setRequestMethod("GET");
+ connection.connect();
final InputStream in = connection.getInputStream();
return new FSDataInputStream(new FSInputStream() {
public int read() throws IOException {
@@ -160,13 +161,11 @@
Long.valueOf(attrs.getValue("blocksize")).longValue(),
modif, FsPermission.valueOf(attrs.getValue("permission")),
attrs.getValue("owner"), attrs.getValue("group"),
- new Path("hftp", fshostname + ":" + fsport,
- attrs.getValue("path")))
+ new Path(getUri().toString(), attrs.getValue("path")))
: new FileStatus(0L, true, 0, 0L,
modif, FsPermission.valueOf(attrs.getValue("permission")),
attrs.getValue("owner"), attrs.getValue("group"),
- new Path("hftp", fshostname + ":" + fsport,
- attrs.getValue("path")));
+ new Path(getUri().toString(), attrs.getValue("path")));
fslist.add(fs);
}
@@ -174,10 +173,8 @@
try {
XMLReader xr = XMLReaderFactory.createXMLReader();
xr.setContentHandler(this);
- final URL url = new URI("http", null, fshostname, fsport,
- "/listPaths" + path, "ugi=" + ugi + (recur? "&recursive=yes" : ""),
- null).toURL();
- HttpURLConnection connection = (HttpURLConnection)url.openConnection();
+ HttpURLConnection connection = openConnection("/listPaths" + path,
+ "ugi=" + ugi + (recur? "&recursive=yes" : ""));
connection.setRequestMethod("GET");
connection.connect();
@@ -185,10 +182,6 @@
xr.parse(new InputSource(resp));
} catch (SAXException e) {
IOException ie = new IOException("invalid xml directory content");
- ie.initCause(e);
- throw ie;
- } catch (URISyntaxException e) {
- IOException ie = new IOException("invalid url");
ie.initCause(e);
throw ie;
}
Added: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HsftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HsftpFileSystem.java?rev=638677&view=auto
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HsftpFileSystem.java (added)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HsftpFileSystem.java Tue Mar 18 18:44:50 2008
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.dfs;
+
+import java.io.IOException;
+
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+import org.apache.hadoop.fs.Path;
+
+/** An implementation of a protocol for accessing filesystems over HTTPS.
+ * The following implementation provides a limited, read-only interface
+ * to a filesystem over HTTPS.
+ * @see org.apache.hadoop.dfs.ListPathsServlet
+ * @see org.apache.hadoop.dfs.FileDataServlet
+ */
+public class HsftpFileSystem extends HftpFileSystem {
+
+ @Override
+ protected HttpURLConnection openConnection(String path, String query)
+ throws IOException {
+ try {
+ final URL url = new URI("https", null, nnAddr.getHostName(),
+ nnAddr.getPort(), path, query, null).toURL();
+ return (HttpURLConnection)url.openConnection();
+ } catch (URISyntaxException e) {
+ throw (IOException)new IOException().initCause(e);
+ }
+ }
+
+ @Override
+ public URI getUri() {
+ try {
+ return new URI("hsftp", null, nnAddr.getHostName(), nnAddr.getPort(),
+ null, null, null);
+ } catch (URISyntaxException e) {
+ return null;
+ }
+ }
+
+}
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/mapred/StatusHttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/mapred/StatusHttpServer.java?rev=638677&r1=638676&r2=638677&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/mapred/StatusHttpServer.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/mapred/StatusHttpServer.java Tue Mar 18 18:44:50 2008
@@ -21,6 +21,7 @@
import java.io.OutputStream;
import java.io.PrintWriter;
import java.net.URL;
+import java.net.InetSocketAddress;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
@@ -32,6 +33,7 @@
import org.apache.hadoop.util.ReflectionUtils;
import org.mortbay.http.HttpContext;
import org.mortbay.http.SocketListener;
+import org.mortbay.http.SslListener;
import org.mortbay.http.handler.ResourceHandler;
import org.mortbay.jetty.servlet.WebApplicationContext;
@@ -46,6 +48,7 @@
public class StatusHttpServer {
private org.mortbay.jetty.Server webServer;
private SocketListener listener;
+ private SslListener sslListener;
private boolean findPort;
private WebApplicationContext webAppContext;
private static final Log LOG =
@@ -172,6 +175,28 @@
listener.setMinThreads(min);
listener.setMaxThreads(max);
}
+
+ /**
+ * Configure an ssl listener on the server.
+ * @param addr address to listen on
+ * @param keystore location of the keystore
+ * @param storPass password for the keystore
+ * @param keyPass password for the key
+ */
+ public void addSslListener(InetSocketAddress addr, String keystore,
+ String storPass, String keyPass) throws IOException {
+ if (sslListener != null || webServer.isStarted()) {
+ throw new IOException("Failed to add ssl listener");
+ }
+ sslListener = new SslListener();
+ sslListener.setHost(addr.getHostName());
+ sslListener.setPort(addr.getPort());
+ sslListener.setKeystore(keystore);
+ sslListener.setPassword(storPass);
+ sslListener.setKeyPassword(keyPass);
+ webServer.addListener(sslListener);
+ }
+
/**
* Start the server. Does not wait for the server to start.
*/