You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-dev@hadoop.apache.org by Tom White <to...@gmail.com> on 2009/06/29 10:09:24 UTC
Re: svn commit: r788898 - in /hadoop/hdfs/trunk: ./
src/contrib/hdfsproxy/ src/contrib/hdfsproxy/conf/ src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/
src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/
src/contrib/hdfsproxy/src/t
On Fri, Jun 26, 2009 at 11:48 PM, <cd...@apache.org> wrote:
> Author: cdouglas
> Date: Fri Jun 26 22:48:23 2009
> New Revision: 788898
>
> URL: http://svn.apache.org/viewvc?rev=788898&view=rev
> Log:
> HDFS-447. Add LDAP lookup to hdfsproxy. Contributed by Zhiyong Zhang
>
> Added:
> hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/LdapIpDirFilter.java
> hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/DummyLdapContext.java
> hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestLdapIpDirFilter.java
> hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/resources/tomcat-web.xml
> Modified:
> hadoop/hdfs/trunk/CHANGES.txt
> hadoop/hdfs/trunk/src/contrib/hdfsproxy/build.xml
> hadoop/hdfs/trunk/src/contrib/hdfsproxy/conf/hdfsproxy-default.xml
> hadoop/hdfs/trunk/src/contrib/hdfsproxy/conf/tomcat-forward-web.xml
> hadoop/hdfs/trunk/src/contrib/hdfsproxy/conf/tomcat-web.xml
> hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFileDataServlet.java
> hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyForwardServlet.java
> hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyListPathsServlet.java
> hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyStreamFile.java
> hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyUtil.java
> hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/resources/proxy-config/hdfsproxy-default.xml
> hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/resources/proxy-config/user-certs.xml
> hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/HsftpFileSystem.java
>
> Modified: hadoop/hdfs/trunk/CHANGES.txt
> URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=788898&r1=788897&r2=788898&view=diff
> ==============================================================================
> --- hadoop/hdfs/trunk/CHANGES.txt (original)
> +++ hadoop/hdfs/trunk/CHANGES.txt Fri Jun 26 22:48:23 2009
> @@ -7,6 +7,8 @@
> HDFS-436. Introduce AspectJ framework for HDFS code and tests.
> (Konstantin Boudnik via szetszwo)
>
> + HDFS-447. Add LDAP lookup to hdfsproxy. (Zhiyong Zhang via cdouglas)
> +
> IMPROVEMENTS
>
> HDFS-381. Remove blocks from DataNode maps when corresponding file
>
> Modified: hadoop/hdfs/trunk/src/contrib/hdfsproxy/build.xml
> URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/contrib/hdfsproxy/build.xml?rev=788898&r1=788897&r2=788898&view=diff
> ==============================================================================
> --- hadoop/hdfs/trunk/src/contrib/hdfsproxy/build.xml (original)
> +++ hadoop/hdfs/trunk/src/contrib/hdfsproxy/build.xml Fri Jun 26 22:48:23 2009
> @@ -60,6 +60,7 @@
> <and>
> <or>
> <equals arg1="${testcase}" arg2="TestProxyFilter" />
> + <equals arg1="${testcase}" arg2="TestLdapIpDirFilter" />
> <equals arg1="${testcase}" arg2="TestProxyUtil" />
> <equals arg1="${testcase}" arg2="TestProxyForwardServlet" />
> <not>
> @@ -118,22 +119,21 @@
> <war destfile="${build.dir}/${final.name}.war" webxml="${basedir}/conf/tomcat-web.xml">
> <lib dir="${common.ivy.lib.dir}">
> <include name="commons-logging-${commons-logging.version}.jar"/>
> - <include name="junit-${junit.version}.jar"/>
> - <include name="log4j-${log4j.version}.jar"/>
> - <include name="slf4j-api-${slf4j-api.version}.jar"/>
> - <include name="slf4j-log4j12-${slf4j-log4j12.version}.jar"/>
> - <include name="xmlenc-${xmlenc.version}.jar"/>
> - <include name="core-${core.version}.jar"/>
> + <include name="junit-${junit.version}.jar"/>
> + <include name="log4j-${log4j.version}.jar"/>
> + <include name="slf4j-api-${slf4j-api.version}.jar"/>
> + <include name="slf4j-log4j12-${slf4j-log4j12.version}.jar"/>
> + <include name="xmlenc-${xmlenc.version}.jar"/>
> + <include name="core-${core.vesion}.jar"/>
> + </lib>
> + <lib dir="${hadoop.root}/lib">
> + <include name="hadoop-core-${hadoop-version}.jar"/>
> </lib>
> - <lib dir="${hadoop.root}/lib">
> - <include name="hadoop-mapred-tools-${hadoop-version}.jar"/>
> - <include name="hadoop-mapred-examples-${hadoop-version}.jar"/>
> - <include name="hadoop-mapred-test-${hadoop-version}.jar"/>
> - <include name="hadoop-core-test-${hadoop-version}.jar"/>
> - <include name="hadoop-core-${hadoop-version}.jar"/>
> - <include name="hadoop-mapred-test-${hadoop-version}.jar"/>
> - </lib>
> - <classes dir="${proxy.conf.dir}" excludes="**/*.example **/*.template **/*.sh hadoop-site.xml"/>
> + <classes dir="${proxy.conf.dir}">
> + <include name="hdfsproxy-default.xml"/>
> + <include name="user-certs.xml"/>
> + <include name="user-permissions.xml"/>
> + </classes>
> <classes dir="${build.classes}"/>
> <classes dir="${hadoop.root}/build/classes"/>
> </war>
> @@ -153,40 +153,69 @@
> <include name="xmlenc-${xmlenc.version}.jar"/>
> <include name="core-${core.vesion}.jar"/>
> </lib>
> - <classes dir="${proxy.conf.dir}" excludes="**/*.example **/*.template **/*.sh hadoop-site.xml"/>
> + <lib dir="${hadoop.root}/lib">
> + <include name="hadoop-core-${hadoop-version}.jar"/>
> + </lib>
> + <classes dir="${proxy.conf.dir}">
> + <include name="hdfsproxy-default.xml"/>
> + <include name="hdfsproxy-site.xml"/>
> + <include name="user-certs.xml"/>
> + <include name="user-permissions.xml"/>
> + </classes>
> + <classes dir="${build.classes}"/>
> + <classes dir="${hadoop.root}/build/classes"/>
> + </war>
> + </target>
> +
> + <target name="testwar" depends="compile" description="Create testing war">
> + <echo>
> + Building the testing .war file
> + </echo>
> + <war destfile="${build.dir}/${final.name}-test.war" webxml="${src.test.resources}/tomcat-web.xml">
> + <lib dir="${common.ivy.lib.dir}">
> + <include name="commons-logging-${commons-logging.version}.jar"/>
> + <include name="junit-${junit.version}.jar"/>
> + <include name="log4j-${log4j.version}.jar"/>
> + <include name="slf4j-api-${slf4j-api.version}.jar"/>
> + <include name="slf4j-log4j12-${slf4j-log4j12.version}.jar"/>
> + <include name="xmlenc-${xmlenc.version}.jar"/>
> + <include name="core-${core.vesion}.jar"/>
> + </lib>
> + <lib dir="${hadoop.root}/lib">
> + <include name="hadoop-core-${hadoop-version}.jar"/>
> + </lib>
> + <classes dir="${proxy.conf.test}" excludes="**/*.template **/*.sh"/>
> <classes dir="${build.classes}"/>
> <classes dir="${hadoop.root}/build/classes"/>
> </war>
> </target>
>
> - <target name="cactifywar" depends="war,load-tasks,cactifywar-pure,cactifywar-clover" description="To include clover coverage test use -Dclover.home ..."/>
> + <target name="cactifywar" depends="testwar,load-tasks,cactifywar-pure,cactifywar-clover" description="To include clover coverage test use -Dclover.home ..."/>
>
> - <target name="cactifywar-pure" depends="war,load-tasks" unless="useClover">
> + <target name="cactifywar-pure" depends="testwar,load-tasks" unless="useClover">
> <mkdir dir="${target.dir}" />
> <echo> no clover found ...</echo>
> - <cactifywar srcfile="${build.dir}/${final.name}.war"
> + <cactifywar srcfile="${build.dir}/${final.name}-test.war"
> destfile="${target.dir}/${cactus.warfile.name}.war"
> mergewebxml="${src.test.resources}/cactus-web.xml">
> <servletredirector/>
> <servletredirector name="ServletRedirectorSecure"
> mapping="/ServletRedirectorSecure" roles="test"/>
> - <filterredirector mapping="/test/filterRedirector.jsp"/>
> - <classes dir="${proxy.conf.test}" excludes="**/*.template **/*.sh"/>
> + <filterredirector mapping="/test/filterRedirector.jsp"/>
> <classes dir="${test.build.dir}"/>
> </cactifywar>
> </target>
>
> - <target name="cactifywar-clover" depends="war,load-tasks" if="useClover">
> + <target name="cactifywar-clover" depends="testwar,load-tasks" if="useClover">
> <mkdir dir="${target.dir}" />
> <echo> Including clover.jar in the war file ...</echo>
> - <cactifywar srcfile="${build.dir}/${final.name}.war"
> + <cactifywar srcfile="${build.dir}/${final.name}-test.war"
> destfile="${target.dir}/${cactus.warfile.name}.war"
> mergewebxml="${src.test.resources}/cactus-web.xml">
> <servletredirector/>
> <servletredirector name="ServletRedirectorSecure"
> mapping="/ServletRedirectorSecure" roles="test"/>
> <filterredirector mapping="/test/filterRedirector.jsp"/>
> - <classes dir="${proxy.conf.test}" excludes="**/*.template **/*.sh"/>
> <classes dir="${test.build.dir}"/>
> <lib dir="${clover.home}/lib">
> <include name="clover.jar"/>
> @@ -224,6 +253,7 @@
> <fileset dir="${src.test}">
> <include name="**/${testcase}.java"/>
> <exclude name="**/TestProxyFilter.java"/>
> + <exclude name="**/TestLdapIpDirFilter.java"/>
> <exclude name="**/TestProxyUtil.java"/>
> <exclude name="**/TestProxyForwardServlet.java"/>
> </fileset>
> @@ -267,7 +297,6 @@
> <classpath>
> <path refid="cactus.classpath"/>
> <pathelement location="${build.classes}"/>
> - <pathelement location="${proxy.conf.dir}"/>
> <pathelement location="${src.test.resources}"/>
> <pathelement location="${src.test.resources}/proxy-config"/>
> </classpath>
> @@ -448,15 +477,12 @@
> <pathelement location="${proxy.conf.test}" />
> <pathelement location="${test.build.dir}" />
> <pathelement location="${hadoop.root}/build/test/classes"/>
> - <pathelement location="${hadoop.root}/lib/hadoop-core-test-${hadoop-version}.jar"/>
> - <pathelement location="${hadoop.root}/build/test/hdfs/classes"/>
> - <pathelement location="${hadoop.root}/lib/hadoop-mapred-test-${hadoop-version}.jar"/>
> <!--<pathelement location="${hadoop.root}/src/contrib/test"/>-->
> <pathelement location="${hadoop.root}/conf"/>
> <pathelement location="${hadoop.root}/build"/>
> <pathelement location="${hadoop.root}/build/classes"/>
> + <pathelement location="${hadoop.root}/build/tools"/>
> <pathelement location="${build.examples}"/>
> - <pathelement location="${hadoop.root}/lib/hadoop-mapred-tools-${hadoop-version}.jar"/>
> <pathelement path="${clover.jar}"/>
> <path refid="contrib-classpath"/>
> </path>
>
> Modified: hadoop/hdfs/trunk/src/contrib/hdfsproxy/conf/hdfsproxy-default.xml
> URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/contrib/hdfsproxy/conf/hdfsproxy-default.xml?rev=788898&r1=788897&r2=788898&view=diff
> ==============================================================================
> --- hadoop/hdfs/trunk/src/contrib/hdfsproxy/conf/hdfsproxy-default.xml (original)
> +++ hadoop/hdfs/trunk/src/contrib/hdfsproxy/conf/hdfsproxy-default.xml Fri Jun 26 22:48:23 2009
> @@ -55,5 +55,51 @@
> </description>
> </property>
>
> +<property>
> + <name>hdfsproxy.ldap.initial.context.factory</name>
> + <value>com.sun.jndi.ldap.LdapCtxFactory</value>
> + <description> ldap initial context factory
> + </description>
> +</property>
> +
> +<property>
> + <name>hdfsproxy.ldap.provider.url</name>
> + <value>ldap://localhost:389</value>
> + <description> ldap server address
> + </description>
> +</property>
> +
> +<property>
> + <name>hdfsproxy.ldap.role.base</name>
> + <value>ou=proxyroles,dc=mycompany,dc=com</value>
> + <description> ldap role base
> + </description>
> +</property>
> +
> +<property>
> + <name>fs.default.name</name>
> + <!-- cluster variant -->
> + <value>hdfs://localhost:54321</value>
> + <description>The name of the default file system. Either the
> + literal string "local" or a host:port for NDFS.</description>
> + <final>true</final>
> + </property>
> +
> +<property>
> + <name>dfs.block.size</name>
> + <value>134217728</value>
> + <description>The default block size for new files.</description>
> +</property>
> +
> +<property>
> + <name>io.file.buffer.size</name>
> + <value>131072</value>
> + <description>The size of buffer for use in sequence files.
> + The size of this buffer should probably be a multiple of hardware
> + page size (4096 on Intel x86), and it determines how much data is
> + buffered during read and write operations.</description>
> +</property>
> +
> +
> </configuration>
>
>
> Modified: hadoop/hdfs/trunk/src/contrib/hdfsproxy/conf/tomcat-forward-web.xml
> URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/contrib/hdfsproxy/conf/tomcat-forward-web.xml?rev=788898&r1=788897&r2=788898&view=diff
> ==============================================================================
> --- hadoop/hdfs/trunk/src/contrib/hdfsproxy/conf/tomcat-forward-web.xml (original)
> +++ hadoop/hdfs/trunk/src/contrib/hdfsproxy/conf/tomcat-forward-web.xml Fri Jun 26 22:48:23 2009
> @@ -38,21 +38,19 @@
> and comments about this application should be addressed.
> </description>
> </context-param>
> -
> +
> <filter>
> - <filter-name>proxyFilter</filter-name>
> - <filter-class>org.apache.hadoop.hdfsproxy.ProxyFilter</filter-class>
> - <init-param>
> - <param-name>filteraddress</param-name>
> - <param-value>10</param-value>
> - </init-param>
> + <filter-name>ldapIpDirFilter</filter-name>
> + <filter-class>org.apache.hadoop.hdfsproxy.LdapIpDirFilter</filter-class>
> </filter>
>
> <filter-mapping>
> - <filter-name>proxyFilter</filter-name>
> + <filter-name>ldapIpDirFilter</filter-name>
> <url-pattern>/*</url-pattern>
> </filter-mapping>
>
> +
> +
>
> <servlet>
> <servlet-name>proxyForward</servlet-name>
> @@ -84,6 +82,7 @@
> <url-pattern>/file/*</url-pattern>
> </servlet-mapping>
>
> +
>
> <welcome-file-list>
> <welcome-file>index.html</welcome-file>
>
> Modified: hadoop/hdfs/trunk/src/contrib/hdfsproxy/conf/tomcat-web.xml
> URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/contrib/hdfsproxy/conf/tomcat-web.xml?rev=788898&r1=788897&r2=788898&view=diff
> ==============================================================================
> --- hadoop/hdfs/trunk/src/contrib/hdfsproxy/conf/tomcat-web.xml (original)
> +++ hadoop/hdfs/trunk/src/contrib/hdfsproxy/conf/tomcat-web.xml Fri Jun 26 22:48:23 2009
> @@ -58,21 +58,16 @@
> </context-param>
>
> <filter>
> - <filter-name>proxyFilter</filter-name>
> - <filter-class>org.apache.hadoop.hdfsproxy.ProxyFilter</filter-class>
> - <init-param>
> - <param-name>filteraddress</param-name>
> - <param-value>10</param-value>
> - </init-param>
> + <filter-name>ldapIpDirFilter</filter-name>
> + <filter-class>org.apache.hadoop.hdfsproxy.LdapIpDirFilter</filter-class>
> </filter>
>
> <filter-mapping>
> - <filter-name>proxyFilter</filter-name>
> + <filter-name>ldapIpDirFilter</filter-name>
> <url-pattern>/*</url-pattern>
> </filter-mapping>
>
>
> -
> <!-- Servlet definitions for the servlets that make up
> your web application, including initialization
> parameters. With Tomcat, you can also send requests
>
> Added: hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/LdapIpDirFilter.java
> URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/LdapIpDirFilter.java?rev=788898&view=auto
> ==============================================================================
> --- hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/LdapIpDirFilter.java (added)
> +++ hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/LdapIpDirFilter.java Fri Jun 26 22:48:23 2009
> @@ -0,0 +1,262 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.hadoop.hdfsproxy;
> +
> +import java.io.IOException;
> +import java.net.InetSocketAddress;
> +import java.util.ArrayList;
> +import java.util.Hashtable;
> +import java.util.regex.Pattern;
> +
> +import javax.naming.NamingEnumeration;
> +import javax.naming.NamingException;
> +import javax.naming.directory.Attribute;
> +import javax.naming.directory.Attributes;
> +import javax.naming.directory.BasicAttribute;
> +import javax.naming.directory.BasicAttributes;
> +import javax.naming.directory.SearchResult;
> +import javax.naming.ldap.InitialLdapContext;
> +import javax.servlet.Filter;
> +import javax.servlet.FilterChain;
> +import javax.servlet.FilterConfig;
> +import javax.servlet.ServletContext;
> +import javax.servlet.ServletException;
> +import javax.servlet.ServletRequest;
> +import javax.servlet.ServletResponse;
> +import javax.servlet.http.HttpServletRequest;
> +import javax.servlet.http.HttpServletResponse;
> +
> +import org.apache.commons.logging.Log;
> +import org.apache.commons.logging.LogFactory;
> +import org.apache.hadoop.conf.Configuration;
> +import org.apache.hadoop.fs.Path;
> +import org.apache.hadoop.net.NetUtils;
> +import org.apache.hadoop.security.UnixUserGroupInformation;
> +
> +public class LdapIpDirFilter implements Filter {
> + public static final Log LOG = LogFactory.getLog(LdapIpDirFilter.class);
> +
> + private static String baseName;
> + private static String hdfsIpSchemaStr;
> + private static String hdfsIpSchemaStrPrefix;
> + private static String hdfsUidSchemaStr;
> + private static String hdfsGroupSchemaStr;
> + private static String hdfsPathSchemaStr;
> +
> + private InitialLdapContext lctx;
> + private String userId;
> + private String groupName;
> + private ArrayList<String> paths;
> +
> + /** Pattern for a filter to find out if a request is HFTP/HSFTP request */
> + protected static final Pattern HFTP_PATTERN = Pattern
> + .compile("^(/listPaths|/data|/streamFile|/file)$");
> + /**
> + * Pattern for a filter to find out if an HFTP/HSFTP request stores its file
> + * path in the extra path information associated with the URL; if not, the
> + * file path is stored in request parameter "filename"
> + */
> + protected static final Pattern FILEPATH_PATTERN = Pattern
> + .compile("^(/listPaths|/data|/file)$");
> +
> + public void initialize(String bName, InitialLdapContext ctx) {
> + // hook to cooperate unit test
> + baseName = bName;
> + hdfsIpSchemaStr = "uniqueMember";
> + hdfsIpSchemaStrPrefix = "cn=";
> + hdfsUidSchemaStr = "uid";
> + hdfsGroupSchemaStr = "userClass";
> + hdfsPathSchemaStr = "documentLocation";
> + lctx = ctx;
> + paths = new ArrayList<String>();
> + }
> +
> + /** {@inheritDoc} */
> + public void init(FilterConfig filterConfig) throws ServletException {
> + ServletContext context = filterConfig.getServletContext();
> + Configuration conf = new Configuration(false);
> + conf.addResource("hdfsproxy-default.xml");
> + conf.addResource("hdfsproxy-site.xml");
> + // extract namenode from source conf.
> + String nn = conf.get("fs.default.name");
> + if (nn == null) {
> + throw new ServletException(
> + "Proxy source cluster name node address not speficied");
> + }
> + InetSocketAddress nAddr = NetUtils.createSocketAddr(nn);
> + context.setAttribute("name.node.address", nAddr);
> + context.setAttribute("name.conf", conf);
> +
> + // for storing hostname <--> cluster mapping to decide which source cluster
> + // to forward
> + context.setAttribute("org.apache.hadoop.hdfsproxy.conf", conf);
> +
> + if (lctx == null) {
> + Hashtable<String, String> env = new Hashtable<String, String>();
> + env.put(InitialLdapContext.INITIAL_CONTEXT_FACTORY, conf.get(
> + "hdfsproxy.ldap.initial.context.factory",
> + "com.sun.jndi.ldap.LdapCtxFactory"));
> + env.put(InitialLdapContext.PROVIDER_URL, conf
> + .get("hdfsproxy.ldap.provider.url"));
> +
> + try {
> + lctx = new InitialLdapContext(env, null);
> + } catch (NamingException ne) {
> + throw new ServletException("NamingException in initializing ldap"
> + + ne.toString());
> + }
> +
> + baseName = conf.get("hdfsproxy.ldap.role.base");
> + hdfsIpSchemaStr = conf.get("hdfsproxy.ldap.ip.schema.string",
> + "uniqueMember");
> + hdfsIpSchemaStrPrefix = conf.get(
> + "hdfsproxy.ldap.ip.schema.string.prefix", "cn=");
> + hdfsUidSchemaStr = conf.get("hdfsproxy.ldap.uid.schema.string", "uid");
> + hdfsGroupSchemaStr = conf.get("hdfsproxy.ldap.group.schema.string",
> + "userClass");
> + hdfsPathSchemaStr = conf.get("hdfsproxy.ldap.hdfs.path.schema.string",
> + "documentLocation");
> + paths = new ArrayList<String>();
> + }
> + LOG.info("LdapIpDirFilter initialization success: " + nn);
> + }
> +
> + /** {@inheritDoc} */
> + public void destroy() {
> + }
> +
> + /** {@inheritDoc} */
> + public void doFilter(ServletRequest request, ServletResponse response,
> + FilterChain chain) throws IOException, ServletException {
> +
> + HttpServletRequest rqst = (HttpServletRequest) request;
> + HttpServletResponse rsp = (HttpServletResponse) response;
> +
> + if (LOG.isDebugEnabled()) {
> + StringBuilder b = new StringBuilder("Request from ").append(
> + rqst.getRemoteHost()).append("/").append(rqst.getRemoteAddr())
> + .append(":").append(rqst.getRemotePort());
> + b.append("\n The Scheme is " + rqst.getScheme());
> + b.append("\n The Path Info is " + rqst.getPathInfo());
> + b.append("\n The Translated Path Info is " + rqst.getPathTranslated());
> + b.append("\n The Context Path is " + rqst.getContextPath());
> + b.append("\n The Query String is " + rqst.getQueryString());
> + b.append("\n The Request URI is " + rqst.getRequestURI());
> + b.append("\n The Request URL is " + rqst.getRequestURL());
> + b.append("\n The Servlet Path is " + rqst.getServletPath());
> + LOG.debug(b.toString());
> + }
> + // check ip address
> + String userIp = rqst.getRemoteAddr();
> + boolean isAuthorized = false;
> + try {
> + isAuthorized = checkUserIp(userIp);
> + if (!isAuthorized) {
> + rsp.sendError(HttpServletResponse.SC_FORBIDDEN,
> + "IP not authorized to access");
> + return;
> + }
> + } catch (NamingException ne) {
> + throw new IOException("NameingException in searching ldap"
> + + ne.toString());
> + }
> + // check request path
> + String servletPath = rqst.getServletPath();
> + if (HFTP_PATTERN.matcher(servletPath).matches()) {
> + // request is an HSFTP request
> + if (FILEPATH_PATTERN.matcher(servletPath).matches()) {
> + // file path as part of the URL
> + isAuthorized = checkHdfsPath(rqst.getPathInfo() != null ? rqst
> + .getPathInfo() : "/");
> + } else {
> + // file path is stored in "filename" parameter
> + isAuthorized = checkHdfsPath(rqst.getParameter("filename"));
> + }
> + }
> + if (!isAuthorized) {
> + rsp.sendError(HttpServletResponse.SC_FORBIDDEN,
> + "User not authorized to access path");
> + return;
> + }
> + UnixUserGroupInformation ugi = new UnixUserGroupInformation(userId,
> + groupName.split(","));
> + rqst.setAttribute("authorized.ugi", ugi);
> + // since we cannot pass ugi object cross context as they are from different
> + // classloaders in different war file, we have to use String attribute.
> + rqst.setAttribute("org.apache.hadoop.hdfsproxy.authorized.userID", userId);
> + rqst.setAttribute("org.apache.hadoop.hdfsproxy.authorized.role", groupName);
> + LOG.info("User: " + userId + " (" + groupName + ") Request: "
> + + rqst.getPathInfo() + " From: " + rqst.getRemoteAddr());
> + chain.doFilter(request, response);
> + }
> +
> + /** check that client's ip is listed in the Ldap Roles */
> + @SuppressWarnings("unchecked")
> + private boolean checkUserIp(String userIp) throws NamingException {
> + String ipMember = hdfsIpSchemaStrPrefix + userIp;
> + Attributes matchAttrs = new BasicAttributes(true);
> + matchAttrs.put(new BasicAttribute(hdfsIpSchemaStr, ipMember));
> + matchAttrs.put(new BasicAttribute(hdfsUidSchemaStr));
> + matchAttrs.put(new BasicAttribute(hdfsPathSchemaStr));
> +
> + String[] attrIDs = { hdfsUidSchemaStr, hdfsGroupSchemaStr,
> + hdfsPathSchemaStr };
> +
> + NamingEnumeration<SearchResult> results = lctx.search(baseName, matchAttrs,
> + attrIDs);
> + if (results.hasMore()) {
> + SearchResult sr = results.next();
> + Attributes attrs = sr.getAttributes();
> + for (NamingEnumeration ne = attrs.getAll(); ne.hasMore();) {
> + Attribute attr = (Attribute) ne.next();
> + if (hdfsUidSchemaStr.equalsIgnoreCase(attr.getID())) {
> + userId = (String) attr.get();
> + } else if (hdfsGroupSchemaStr.equalsIgnoreCase(attr.getID())) {
> + groupName = (String) attr.get();
> + } else if (hdfsPathSchemaStr.equalsIgnoreCase(attr.getID())) {
> + for (NamingEnumeration e = attr.getAll(); e.hasMore();) {
> + paths.add((String) e.next());
> + }
> + }
> + }
> + return true;
> + }
> + LOG.info("Ip address " + userIp
> + + " is not authorized to access the proxy server");
> + return false;
> + }
> +
> + /** check that the requested path is listed in the ldap entry */
> + private boolean checkHdfsPath(String pathInfo) {
> + if (pathInfo == null || pathInfo.length() == 0) {
> + LOG.info("Can't get file path from the request");
> + return false;
> + }
> + Path userPath = new Path(pathInfo);
> + while (userPath != null) {
> + if (LOG.isDebugEnabled()) {
> + LOG.debug("\n Checking file path " + userPath);
> + }
> + if (paths.contains(userPath.toString()))
> + return true;
> + userPath = userPath.getParent();
> + }
> + LOG.info("User " + userId + " is not authorized to access " + pathInfo);
> + return false;
> + }
> +}
>
> Modified: hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFileDataServlet.java
> URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFileDataServlet.java?rev=788898&r1=788897&r2=788898&view=diff
> ==============================================================================
> --- hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFileDataServlet.java (original)
> +++ hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyFileDataServlet.java Fri Jun 26 22:48:23 2009
> @@ -24,7 +24,6 @@
> import javax.servlet.ServletContext;
> import javax.servlet.ServletException;
> import javax.servlet.http.HttpServletRequest;
> -import javax.servlet.http.HttpServletResponse;
>
> import org.apache.hadoop.conf.Configuration;
> import org.apache.hadoop.fs.FileStatus;
> @@ -36,14 +35,14 @@
> public class ProxyFileDataServlet extends FileDataServlet {
> /** For java.io.Serializable */
> private static final long serialVersionUID = 1L;
> -
> +
> /** {@inheritDoc} */
> @Override
> public void init() throws ServletException {
> ServletContext context = getServletContext();
> - if (context.getAttribute("name.conf") == null) {
> + if (context.getAttribute("name.conf") == null) {
> context.setAttribute("name.conf", new Configuration());
> - }
> + }
> }
>
> /** {@inheritDoc} */
> @@ -59,8 +58,17 @@
> /** {@inheritDoc} */
> @Override
> protected UnixUserGroupInformation getUGI(HttpServletRequest request) {
> - String userID = (String) request.getAttribute("org.apache.hadoop.hdfsproxy.authorized.userID");
> - UnixUserGroupInformation ugi = ProxyUgiManager.getUgiForUser(userID);
> + String userID = (String) request
> + .getAttribute("org.apache.hadoop.hdfsproxy.authorized.userID");
> + String groupName = (String) request
> + .getAttribute("org.apache.hadoop.hdfsproxy.authorized.role");
> + UnixUserGroupInformation ugi;
> + if (groupName != null) {
> + // get group info from ldap
> + ugi = new UnixUserGroupInformation(userID, groupName.split(","));
> + } else {// stronger ugi management
> + ugi = ProxyUgiManager.getUgiForUser(userID);
> + }
> return ugi;
> }
> }
>
> Modified: hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyForwardServlet.java
> URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyForwardServlet.java?rev=788898&r1=788897&r2=788898&view=diff
> ==============================================================================
> --- hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyForwardServlet.java (original)
> +++ hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyForwardServlet.java Fri Jun 26 22:48:23 2009
> @@ -17,17 +17,18 @@
> */
> package org.apache.hadoop.hdfsproxy;
>
> +import java.io.IOException;
> +
> +import javax.servlet.RequestDispatcher;
> +import javax.servlet.ServletContext;
> +import javax.servlet.ServletException;
> import javax.servlet.http.HttpServlet;
> import javax.servlet.http.HttpServletRequest;
> import javax.servlet.http.HttpServletResponse;
> -import java.io.IOException;
> -import javax.servlet.ServletException;
> -import javax.servlet.ServletContext;
> -import javax.servlet.RequestDispatcher;
> +
> import org.apache.commons.logging.Log;
> import org.apache.commons.logging.LogFactory;
> import org.apache.hadoop.conf.Configuration;
> -import org.apache.hadoop.security.UnixUserGroupInformation;
>
> /**
> *
> @@ -40,51 +41,55 @@
> private static final long serialVersionUID = 1L;
> private static Configuration configuration = null;
> public static final Log LOG = LogFactory.getLog(ProxyForwardServlet.class);
> -
> +
> /** {@inheritDoc} */
> @Override
> public void init() throws ServletException {
> ServletContext context = getServletContext();
> - configuration = (Configuration) context.getAttribute("org.apache.hadoop.hdfsproxy.conf");
> + configuration = (Configuration) context
> + .getAttribute("org.apache.hadoop.hdfsproxy.conf");
> }
> -
> +
> /** {@inheritDoc} */
> @Override
> public void doGet(HttpServletRequest request, HttpServletResponse response)
> - throws IOException, ServletException {
> - String hostname = request.getServerName();
> -
> + throws IOException, ServletException {
> + String hostname = request.getServerName();
> +
> String version = configuration.get(hostname);
> - if (version != null) {
> - ServletContext curContext = getServletContext();
> - ServletContext dstContext = curContext.getContext(version);
> -
> - if (dstContext == null) {
> - LOG.info("Context non-exist or restricted from access: " + version);
> - response.sendError(HttpServletResponse.SC_NOT_FOUND);
> - return;
> - }
> - LOG.debug("Request to " + hostname + " is forwarded to version " + version);
> - forwardRequest(request, response, dstContext, request.getServletPath());
> -
> - } else {
> - LOG.info("not a valid context path");
> - response.sendError(HttpServletResponse.SC_NOT_IMPLEMENTED);
> + if (version == null) {
> + // extract from hostname directly
> + String[] strs = hostname.split("[-\\.]");
> + version = "/" + strs[0];
> + }
> +
> + ServletContext curContext = getServletContext();
> + ServletContext dstContext = curContext.getContext(version);
> +
> + if (dstContext == null) {
> + LOG.info("Context non-exist or restricted from access: " + version);
> + response.sendError(HttpServletResponse.SC_NOT_FOUND);
> + return;
> }
> - }
> + LOG.debug("Request to " + hostname + " is forwarded to version " + version);
> + forwardRequest(request, response, dstContext, request.getServletPath());
> +
> + }
> +
> /** {@inheritDoc} */
> - public void forwardRequest(HttpServletRequest request, HttpServletResponse response, ServletContext context, String pathInfo)
> - throws IOException, ServletException{
> - String path = buildForwardPath(request, pathInfo);
> + public void forwardRequest(HttpServletRequest request,
> + HttpServletResponse response, ServletContext context, String pathInfo)
> + throws IOException, ServletException {
> + String path = buildForwardPath(request, pathInfo);
> RequestDispatcher dispatcher = context.getRequestDispatcher(path);
> if (dispatcher == null) {
> - LOG.info("There was no such dispatcher");
> + LOG.info("There was no such dispatcher: " + path);
> response.sendError(HttpServletResponse.SC_NO_CONTENT);
> return;
> }
> dispatcher.forward(request, response);
> }
> -
> +
> /** {@inheritDoc} */
> protected String buildForwardPath(HttpServletRequest request, String pathInfo) {
> String path = pathInfo;
>
> Modified: hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyListPathsServlet.java
> URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyListPathsServlet.java?rev=788898&r1=788897&r2=788898&view=diff
> ==============================================================================
> --- hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyListPathsServlet.java (original)
> +++ hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyListPathsServlet.java Fri Jun 26 22:48:23 2009
> @@ -29,21 +29,30 @@
> public class ProxyListPathsServlet extends ListPathsServlet {
> /** For java.io.Serializable */
> private static final long serialVersionUID = 1L;
> -
> +
> /** {@inheritDoc} */
> @Override
> public void init() throws ServletException {
> ServletContext context = getServletContext();
> - if (context.getAttribute("name.conf") == null) {
> + if (context.getAttribute("name.conf") == null) {
> context.setAttribute("name.conf", new Configuration());
> - }
> + }
> }
>
> /** {@inheritDoc} */
> @Override
> protected UnixUserGroupInformation getUGI(HttpServletRequest request) {
> - String userID = (String) request.getAttribute("org.apache.hadoop.hdfsproxy.authorized.userID");
> - UnixUserGroupInformation ugi = ProxyUgiManager.getUgiForUser(userID);
> + String userID = (String) request
> + .getAttribute("org.apache.hadoop.hdfsproxy.authorized.userID");
> + String groupName = (String) request
> + .getAttribute("org.apache.hadoop.hdfsproxy.authorized.role");
> + UnixUserGroupInformation ugi;
> + if (groupName != null) {
> + // group info stored in ldap
> + ugi = new UnixUserGroupInformation(userID, groupName.split(","));
> + } else {// stronger ugi management
> + ugi = ProxyUgiManager.getUgiForUser(userID);
> + }
> return ugi;
> }
> }
>
> Modified: hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyStreamFile.java
> URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyStreamFile.java?rev=788898&r1=788897&r2=788898&view=diff
> ==============================================================================
> --- hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyStreamFile.java (original)
> +++ hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyStreamFile.java Fri Jun 26 22:48:23 2009
> @@ -24,22 +24,23 @@
> import javax.servlet.ServletException;
> import javax.servlet.http.HttpServletRequest;
>
> +import org.apache.hadoop.conf.Configuration;
> import org.apache.hadoop.hdfs.DFSClient;
> import org.apache.hadoop.hdfs.server.namenode.StreamFile;
> import org.apache.hadoop.security.UnixUserGroupInformation;
> -import org.apache.hadoop.conf.Configuration;
>
> /** {@inheritDoc} */
> public class ProxyStreamFile extends StreamFile {
> /** For java.io.Serializable */
> private static final long serialVersionUID = 1L;
> +
> /** {@inheritDoc} */
> @Override
> public void init() throws ServletException {
> ServletContext context = getServletContext();
> - if (context.getAttribute("name.conf") == null) {
> + if (context.getAttribute("name.conf") == null) {
> context.setAttribute("name.conf", new Configuration());
> - }
> + }
> }
>
> /** {@inheritDoc} */
> @@ -59,8 +60,18 @@
> /** {@inheritDoc} */
> @Override
> protected UnixUserGroupInformation getUGI(HttpServletRequest request) {
> - String userID = (String) request.getAttribute("org.apache.hadoop.hdfsproxy.authorized.userID");
> - UnixUserGroupInformation ugi = ProxyUgiManager.getUgiForUser(userID);
> + String userID = (String) request
> + .getAttribute("org.apache.hadoop.hdfsproxy.authorized.userID");
> + String groupName = (String) request
> + .getAttribute("org.apache.hadoop.hdfsproxy.authorized.role");
> + UnixUserGroupInformation ugi;
> + if (groupName != null) {
> + // get group info from ldap
> + ugi = new UnixUserGroupInformation(userID, groupName.split(","));
> + } else {// stronger ugi management
> + ugi = ProxyUgiManager.getUgiForUser(userID);
> + }
> return ugi;
> }
> +
> }
>
> Modified: hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyUtil.java
> URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyUtil.java?rev=788898&r1=788897&r2=788898&view=diff
> ==============================================================================
> --- hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyUtil.java (original)
> +++ hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/java/org/apache/hadoop/hdfsproxy/ProxyUtil.java Fri Jun 26 22:48:23 2009
> @@ -18,6 +18,7 @@
>
> package org.apache.hadoop.hdfsproxy;
>
> +import java.io.FileInputStream;
> import java.io.IOException;
> import java.io.InputStream;
> import java.net.HttpURLConnection;
> @@ -25,13 +26,20 @@
> import java.net.URI;
> import java.net.URISyntaxException;
> import java.net.URL;
> +import java.security.KeyStore;
> import java.security.cert.X509Certificate;
> import java.util.Date;
> import java.util.Set;
>
> -import javax.net.ssl.HttpsURLConnection;
> import javax.net.ssl.HostnameVerifier;
> +import javax.net.ssl.HttpsURLConnection;
> +import javax.net.ssl.KeyManager;
> +import javax.net.ssl.KeyManagerFactory;
> +import javax.net.ssl.SSLContext;
> import javax.net.ssl.SSLSession;
> +import javax.net.ssl.TrustManager;
> +import javax.net.ssl.TrustManagerFactory;
> +import javax.net.ssl.X509TrustManager;
> import javax.servlet.http.HttpServletResponse;
>
> import org.apache.commons.logging.Log;
> @@ -43,17 +51,19 @@
> import org.apache.hadoop.net.NetUtils;
> import org.apache.hadoop.util.HostsFileReader;
>
> -
> /**
> * Proxy Utility .
> */
> public class ProxyUtil {
> public static final Log LOG = LogFactory.getLog(ProxyUtil.class);
> private static final long MM_SECONDS_PER_DAY = 1000 * 60 * 60 * 24;
> - private static final int CERT_EXPIRATION_WARNING_THRESHOLD = 30; // 30 days warning
> -
> + private static final int CERT_EXPIRATION_WARNING_THRESHOLD = 30; // 30 days
> +
> + // warning
> +
> private static enum UtilityOption {
> - RELOAD("-reloadPermFiles"), CLEAR("-clearUgiCache"), GET("-get"), CHECKCERTS("-checkcerts");
> + RELOAD("-reloadPermFiles"), CLEAR("-clearUgiCache"), GET("-get"), CHECKCERTS(
> + "-checkcerts");
>
> private String name = null;
>
> @@ -65,7 +75,7 @@
> return name;
> }
> }
> -
> +
> /**
> * Dummy hostname verifier that is used to bypass hostname checking
> */
> @@ -75,6 +85,21 @@
> }
> }
>
> + /**
> + * Dummy trustmanager that is used to bypass server certificate checking
> + */
> + private static class DummyTrustManager implements X509TrustManager {
> + public void checkClientTrusted(X509Certificate[] chain, String authType) {
> + }
> +
> + public void checkServerTrusted(X509Certificate[] chain, String authType) {
> + }
> +
> + public X509Certificate[] getAcceptedIssuers() {
> + return null;
> + }
> + }
> +
> private static HttpsURLConnection openConnection(String hostname, int port,
> String path) throws IOException {
> try {
> @@ -90,21 +115,53 @@
> }
> }
>
> - private static void setupSslProps(Configuration conf) {
> - System.setProperty("javax.net.ssl.trustStore", conf
> - .get("ssl.client.truststore.location"));
> - System.setProperty("javax.net.ssl.trustStorePassword", conf.get(
> - "ssl.client.truststore.password", ""));
> - System.setProperty("javax.net.ssl.trustStoreType", conf.get(
> - "ssl.client.truststore.type", "jks"));
> - System.setProperty("javax.net.ssl.keyStore", conf
> - .get("ssl.client.keystore.location"));
> - System.setProperty("javax.net.ssl.keyStorePassword", conf.get(
> - "ssl.client.keystore.password", ""));
> - System.setProperty("javax.net.ssl.keyPassword", conf.get(
> - "ssl.client.keystore.keypassword", ""));
> - System.setProperty("javax.net.ssl.keyStoreType", conf.get(
> - "ssl.client.keystore.type", "jks"));
> + private static void setupSslProps(Configuration conf) throws IOException {
> + FileInputStream fis = null;
> + try {
> + SSLContext sc = SSLContext.getInstance("SSL");
> + KeyManager[] kms = null;
> + TrustManager[] tms = null;
> + if (conf.get("ssl.client.keystore.location") != null) {
> + // initialize default key manager with keystore file and pass
> + KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
> + KeyStore ks = KeyStore.getInstance(conf.get("ssl.client.keystore.type",
> + "JKS"));
> + char[] ksPass = conf.get("ssl.client.keystore.password", "changeit")
> + .toCharArray();
> + fis = new FileInputStream(conf.get("ssl.client.keystore.location",
> + "keystore.jks"));
> + ks.load(fis, ksPass);
> + kmf.init(ks, conf.get("ssl.client.keystore.keypassword", "changeit")
> + .toCharArray());
> + kms = kmf.getKeyManagers();
> + fis.close();
> + fis = null;
> + }
> + // initialize default trust manager with keystore file and pass
> + if (conf.getBoolean("ssl.client.do.not.authenticate.server", false)) {
> + // by pass trustmanager validation
> + tms = new DummyTrustManager[] { new DummyTrustManager() };
> + } else {
> + TrustManagerFactory tmf = TrustManagerFactory.getInstance("PKIX");
> + KeyStore ts = KeyStore.getInstance(conf.get(
> + "ssl.client.truststore.type", "JKS"));
> + char[] tsPass = conf.get("ssl.client.truststore.password", "changeit")
> + .toCharArray();
> + fis = new FileInputStream(conf.get("ssl.client.truststore.location",
> + "truststore.jks"));
> + ts.load(fis, tsPass);
> + tmf.init(ts);
> + tms = tmf.getTrustManagers();
> + }
> + sc.init(kms, tms, new java.security.SecureRandom());
> + HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
> + } catch (Exception e) {
> + throw new IOException("Could not initialize SSLContext", e);
> + } finally {
> + if (fis != null) {
> + fis.close();
> + }
> + }
> }
>
> static InetSocketAddress getSslAddr(Configuration conf) throws IOException {
> @@ -121,31 +178,33 @@
> int err = 0;
> StringBuilder b = new StringBuilder();
>
> - HostsFileReader hostsReader = new HostsFileReader(conf.get("hdfsproxy.hosts",
> - "hdfsproxy-hosts"), "");
> + HostsFileReader hostsReader = new HostsFileReader(conf.get(
> + "hdfsproxy.hosts", "hdfsproxy-hosts"), "");
> Set<String> hostsList = hostsReader.getHosts();
> for (String hostname : hostsList) {
> HttpsURLConnection connection = null;
> try {
> - connection = openConnection(hostname, sslPort, path);
> - connection.connect();
> + connection = openConnection(hostname, sslPort, path);
> + connection.connect();
> if (LOG.isDebugEnabled()) {
> StringBuffer sb = new StringBuffer();
> - X509Certificate[] clientCerts = (X509Certificate[]) connection.getLocalCertificates();
> + X509Certificate[] clientCerts = (X509Certificate[]) connection
> + .getLocalCertificates();
> if (clientCerts != null) {
> for (X509Certificate cert : clientCerts)
> sb.append("\n Client certificate Subject Name is "
> + cert.getSubjectX500Principal().getName());
> } else {
> - sb.append("\n No client certificates were found");
> + sb.append("\n No client certificates were found");
> }
> - X509Certificate[] serverCerts = (X509Certificate[]) connection.getServerCertificates();
> + X509Certificate[] serverCerts = (X509Certificate[]) connection
> + .getServerCertificates();
> if (serverCerts != null) {
> for (X509Certificate cert : serverCerts)
> sb.append("\n Server certificate Subject Name is "
> + cert.getSubjectX500Principal().getName());
> } else {
> - sb.append("\n No server certificates were found");
> + sb.append("\n No server certificates were found");
> }
> LOG.debug(sb.toString());
> }
> @@ -156,7 +215,8 @@
> }
> } catch (IOException e) {
> b.append("\n\t" + hostname + ": " + e.getLocalizedMessage());
> - if (LOG.isDebugEnabled()) e.printStackTrace();
> + if (LOG.isDebugEnabled())
> + LOG.debug("Exception happend for host " + hostname, e);
> err++;
> } finally {
> if (connection != null)
> @@ -164,65 +224,73 @@
> }
> }
> if (err > 0) {
> - System.err.print("Command failed on the following "
> - + err + " host" + (err==1?":":"s:") + b.toString() + "\n");
> + System.err.print("Command failed on the following " + err + " host"
> + + (err == 1 ? ":" : "s:") + b.toString() + "\n");
> return false;
> }
> return true;
> }
> -
> -
> - static FSDataInputStream open(Configuration conf, String hostname, int port, String path) throws IOException {
> +
> + static FSDataInputStream open(Configuration conf, String hostname, int port,
> + String path) throws IOException {
> setupSslProps(conf);
> HttpURLConnection connection = null;
> connection = openConnection(hostname, port, path);
> connection.connect();
> final InputStream in = connection.getInputStream();
> return new FSDataInputStream(new FSInputStream() {
> - public int read() throws IOException {
> - return in.read();
> - }
> - public int read(byte[] b, int off, int len) throws IOException {
> - return in.read(b, off, len);
> - }
> + public int read() throws IOException {
> + return in.read();
> + }
>
> - public void close() throws IOException {
> - in.close();
> - }
> + public int read(byte[] b, int off, int len) throws IOException {
> + return in.read(b, off, len);
> + }
>
> - public void seek(long pos) throws IOException {
> - throw new IOException("Can't seek!");
> - }
> - public long getPos() throws IOException {
> - throw new IOException("Position unknown!");
> - }
> - public boolean seekToNewSource(long targetPos) throws IOException {
> - return false;
> - }
> - });
> + public void close() throws IOException {
> + in.close();
> + }
> +
> + public void seek(long pos) throws IOException {
> + throw new IOException("Can't seek!");
> + }
> +
> + public long getPos() throws IOException {
> + throw new IOException("Position unknown!");
> + }
> +
> + public boolean seekToNewSource(long targetPos) throws IOException {
> + return false;
> + }
> + });
> }
> -
> - static void checkServerCertsExpirationDays(Configuration conf, String hostname, int port) throws IOException {
> +
> + static void checkServerCertsExpirationDays(Configuration conf,
> + String hostname, int port) throws IOException {
> setupSslProps(conf);
> HttpsURLConnection connection = null;
> connection = openConnection(hostname, port, null);
> connection.connect();
> - X509Certificate[] serverCerts = (X509Certificate[]) connection.getServerCertificates();
> + X509Certificate[] serverCerts = (X509Certificate[]) connection
> + .getServerCertificates();
> Date curDate = new Date();
> long curTime = curDate.getTime();
> if (serverCerts != null) {
> for (X509Certificate cert : serverCerts) {
> StringBuffer sb = new StringBuffer();
> - sb.append("\n Server certificate Subject Name: " + cert.getSubjectX500Principal().getName());
> + sb.append("\n Server certificate Subject Name: "
> + + cert.getSubjectX500Principal().getName());
> Date expDate = cert.getNotAfter();
> long expTime = expDate.getTime();
> - int dayOffSet = (int) ((expTime - curTime)/MM_SECONDS_PER_DAY);
> + int dayOffSet = (int) ((expTime - curTime) / MM_SECONDS_PER_DAY);
> sb.append(" have " + dayOffSet + " days to expire");
> - if (dayOffSet < CERT_EXPIRATION_WARNING_THRESHOLD) LOG.warn(sb.toString());
> - else LOG.info(sb.toString());
> + if (dayOffSet < CERT_EXPIRATION_WARNING_THRESHOLD)
> + LOG.warn(sb.toString());
> + else
> + LOG.info(sb.toString());
> }
> } else {
> - LOG.info("\n No Server certs was found");
> + LOG.info("\n No Server certs was found");
> }
>
> if (connection != null) {
> @@ -231,24 +299,23 @@
> }
>
> public static void main(String[] args) throws Exception {
> - if(args.length < 1 ||
> - (!UtilityOption.RELOAD.getName().equalsIgnoreCase(args[0])
> + if (args.length < 1
> + || (!UtilityOption.RELOAD.getName().equalsIgnoreCase(args[0])
> && !UtilityOption.CLEAR.getName().equalsIgnoreCase(args[0])
> - && !UtilityOption.GET.getName().equalsIgnoreCase(args[0])
> - && !UtilityOption.CHECKCERTS.getName().equalsIgnoreCase(args[0])) ||
> - (UtilityOption.GET.getName().equalsIgnoreCase(args[0]) && args.length != 4) ||
> - (UtilityOption.CHECKCERTS.getName().equalsIgnoreCase(args[0]) && args.length != 3)) {
> - System.err.println("Usage: ProxyUtil ["
> - + UtilityOption.RELOAD.getName() + "] | ["
> - + UtilityOption.CLEAR.getName() + "] | ["
> + && !UtilityOption.GET.getName().equalsIgnoreCase(args[0]) && !UtilityOption.CHECKCERTS
> + .getName().equalsIgnoreCase(args[0]))
> + || (UtilityOption.GET.getName().equalsIgnoreCase(args[0]) && args.length != 4)
> + || (UtilityOption.CHECKCERTS.getName().equalsIgnoreCase(args[0]) && args.length != 3)) {
> + System.err.println("Usage: ProxyUtil [" + UtilityOption.RELOAD.getName()
> + + "] | [" + UtilityOption.CLEAR.getName() + "] | ["
> + UtilityOption.GET.getName() + " <hostname> <#port> <path> ] | ["
> + UtilityOption.CHECKCERTS.getName() + " <hostname> <#port> ]");
> - System.exit(0);
> + System.exit(0);
> }
> - Configuration conf = new Configuration(false);
> + Configuration conf = new Configuration(false);
> conf.addResource("ssl-client.xml");
> conf.addResource("hdfsproxy-default.xml");
> -
> +
> if (UtilityOption.RELOAD.getName().equalsIgnoreCase(args[0])) {
> // reload user-certs.xml and user-permissions.xml files
> sendCommand(conf, "/reloadPermFiles");
> @@ -266,5 +333,5 @@
> in.close();
> }
> }
> -
> +
> }
>
> Added: hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/DummyLdapContext.java
> URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/DummyLdapContext.java?rev=788898&view=auto
> ==============================================================================
> --- hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/DummyLdapContext.java (added)
> +++ hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/DummyLdapContext.java Fri Jun 26 22:48:23 2009
> @@ -0,0 +1,128 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +package org.apache.hadoop.hdfsproxy;
> +
> +import java.util.ArrayList;
> +import java.util.Hashtable;
> +
> +import javax.naming.NamingEnumeration;
> +import javax.naming.NamingException;
> +import javax.naming.directory.Attribute;
> +import javax.naming.directory.Attributes;
> +import javax.naming.directory.BasicAttribute;
> +import javax.naming.directory.BasicAttributes;
> +import javax.naming.directory.SearchResult;
> +import javax.naming.ldap.Control;
> +import javax.naming.ldap.InitialLdapContext;
> +
> +class DummyLdapContext extends InitialLdapContext {
> + class ResultEnum<T> implements NamingEnumeration<T> {
> + private ArrayList<T> rl;
> +
> + public ResultEnum() {
> + rl = new ArrayList<T>();
> + }
> +
> + public ResultEnum(ArrayList<T> al) {
> + rl = al;
> + }
> +
> + public boolean hasMoreElements() {
> + return !rl.isEmpty();
> + }
> +
> + public T nextElement() {
> + T t = rl.get(0);
> + rl.remove(0);
> + return t;
> + }
> +
> + public boolean hasMore() throws NamingException {
> + return !rl.isEmpty();
> + }
> +
> + public T next() throws NamingException {
> + T t = rl.get(0);
> + rl.remove(0);
> + return t;
> + }
> +
> + public void close() throws NamingException {
> + }
> + }
> +
> + public DummyLdapContext() throws NamingException {
> + }
> +
> + public DummyLdapContext(Hashtable<?, ?> environment, Control[] connCtls)
> + throws NamingException {
> + }
> +
> + public NamingEnumeration<SearchResult> search(String name,
> + Attributes matchingAttributes, String[] attributesToReturn)
> + throws NamingException {
> + System.out.println("Searching Dummy LDAP Server Results:");
> + if (!"ou=proxyroles,dc=mycompany,dc=com".equalsIgnoreCase(name)) {
> + System.out.println("baseName mismatch");
> + return new ResultEnum<SearchResult>();
> + }
> + if (!"cn=127.0.0.1".equals((String) matchingAttributes.get("uniqueMember")
> + .get())) {
> + System.out.println("Ip address mismatch");
> + return new ResultEnum<SearchResult>();
> + }
> + BasicAttributes attrs = new BasicAttributes();
> + BasicAttribute uidAttr = new BasicAttribute("uid", "testuser");
> + attrs.put(uidAttr);
> + BasicAttribute groupAttr = new BasicAttribute("userClass", "testgroup");
> + attrs.put(groupAttr);
> + BasicAttribute locAttr = new BasicAttribute("documentLocation", "/testdir");
> + attrs.put(locAttr);
> + SearchResult sr = new SearchResult(null, null, attrs);
> + ArrayList<SearchResult> al = new ArrayList<SearchResult>();
> + al.add(sr);
> + NamingEnumeration<SearchResult> ne = new ResultEnum<SearchResult>(al);
> + return ne;
> + }
> +
> + @SuppressWarnings("unchecked")
> + public static void main(String[] args) throws Exception {
> + DummyLdapContext dlc = new DummyLdapContext();
> + String baseName = "ou=proxyroles,dc=mycompany,dc=com";
> + Attributes matchAttrs = new BasicAttributes(true);
> + String[] attrIDs = { "uid", "documentLocation" };
> + NamingEnumeration<SearchResult> results = dlc.search(baseName, matchAttrs,
> + attrIDs);
> + if (results.hasMore()) {
> + SearchResult sr = results.next();
> + Attributes attrs = sr.getAttributes();
> + for (NamingEnumeration ne = attrs.getAll(); ne.hasMore();) {
> + Attribute attr = (Attribute) ne.next();
> + if ("uid".equalsIgnoreCase(attr.getID())) {
> + System.out.println("User ID = " + attr.get());
> + } else if ("documentLocation".equalsIgnoreCase(attr.getID())) {
> + System.out.println("Document Location = ");
> + for (NamingEnumeration e = attr.getAll(); e.hasMore();) {
> + System.out.println(e.next());
> + }
> + }
> + }
> + }
> + }
> +}
>
> Added: hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestLdapIpDirFilter.java
> URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestLdapIpDirFilter.java?rev=788898&view=auto
> ==============================================================================
> --- hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestLdapIpDirFilter.java (added)
> +++ hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/org/apache/hadoop/hdfsproxy/TestLdapIpDirFilter.java Fri Jun 26 22:48:23 2009
> @@ -0,0 +1,139 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements. See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to You under the Apache License, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +package org.apache.hadoop.hdfsproxy;
> +
> +import java.io.IOException;
> +import java.io.PrintWriter;
> +
> +import javax.naming.NamingException;
> +import javax.servlet.FilterChain;
> +import javax.servlet.FilterConfig;
> +import javax.servlet.ServletContext;
> +import javax.servlet.ServletException;
> +import javax.servlet.ServletRequest;
> +import javax.servlet.ServletResponse;
> +
> +import org.apache.cactus.FilterTestCase;
> +import org.apache.cactus.WebRequest;
> +import org.apache.cactus.WebResponse;
> +import org.apache.commons.logging.Log;
> +import org.apache.commons.logging.LogFactory;
> +
> +public class TestLdapIpDirFilter extends FilterTestCase {
> +
> + public static final Log LOG = LogFactory.getLog(TestLdapIpDirFilter.class);
> +
> + private class DummyFilterChain implements FilterChain {
> + public void doFilter(ServletRequest theRequest, ServletResponse theResponse)
> + throws IOException, ServletException {
> + PrintWriter writer = theResponse.getWriter();
> +
> + writer.print("<p>some content</p>");
> + writer.close();
> + }
> +
> + public void init(FilterConfig theConfig) {
> + }
> +
> + public void destroy() {
> + }
> + }
> +
> + public void testIpRestriction() throws ServletException, IOException,
> + NamingException {
> + LdapIpDirFilter filter = new LdapIpDirFilter();
> + String baseName = "ou=proxyroles,dc=mycompany,dc=com";
> + DummyLdapContext dlc = new DummyLdapContext();
> + filter.initialize(baseName, dlc);
> + request.setRemoteIPAddress("127.0.0.2");
> + request.removeAttribute("org.apache.hadoop.hdfsproxy.authorized.userID");
> + FilterChain mockFilterChain = new DummyFilterChain();
> + filter.doFilter(request, response, mockFilterChain);
> + assertNull(request
> + .getAttribute("org.apache.hadoop.hdfsproxy.authorized.userID"));
> + }
> +
> + public void endIpRestriction(WebResponse theResponse) {
> + assertEquals(theResponse.getStatusCode(), 403);
> + assertTrue("Text missing 'IP not authorized to access' : : ["
> + + theResponse.getText() + "]", theResponse.getText().indexOf(
> + "IP not authorized to access") > 0);
> + }
> +
> + public void beginPathRestriction(WebRequest theRequest) {
> + theRequest.setURL("proxy-test:0", null, "/streamFile", null,
> + "filename=/nontestdir");
> + }
> +
> + public void testPathRestriction() throws ServletException, IOException,
> + NamingException {
> + LdapIpDirFilter filter = new LdapIpDirFilter();
> + String baseName = "ou=proxyroles,dc=mycompany,dc=com";
> + DummyLdapContext dlc = new DummyLdapContext();
> + filter.initialize(baseName, dlc);
> + request.setRemoteIPAddress("127.0.0.1");
> + request.removeAttribute("org.apache.hadoop.hdfsproxy.authorized.userID");
> + FilterChain mockFilterChain = new DummyFilterChain();
> + filter.doFilter(request, response, mockFilterChain);
> + assertNull(request
> + .getAttribute("org.apache.hadoop.hdfsproxy.authorized.userID"));
> + }
> +
> + public void endPathRestriction(WebResponse theResponse) {
> + assertEquals(theResponse.getStatusCode(), 403);
> + assertTrue("Text missing 'User not authorized to access path' : : ["
> + + theResponse.getText() + "]", theResponse.getText().indexOf(
> + "User not authorized to access path") > 0);
> + }
> +
> + public void beginDoFilter(WebRequest theRequest) {
> + theRequest.setURL("proxy-test:0", null, "/streamFile", null,
> + "filename=/testdir");
> + }
> +
> + public void testDoFilter() throws ServletException, IOException,
> + NamingException {
> + LdapIpDirFilter filter = new LdapIpDirFilter();
> + String baseName = "ou=proxyroles,dc=mycompany,dc=com";
> + DummyLdapContext dlc = new DummyLdapContext();
> + filter.initialize(baseName, dlc);
> + request.setRemoteIPAddress("127.0.0.1");
> +
> + ServletContext context = config.getServletContext();
> + context.removeAttribute("name.node.address");
> + context.removeAttribute("name.conf");
> + assertNull(context.getAttribute("name.node.address"));
> + assertNull(context.getAttribute("name.conf"));
> + filter.init(config);
> + assertNotNull(context.getAttribute("name.node.address"));
> + assertNotNull(context.getAttribute("name.conf"));
> +
> + request.removeAttribute("org.apache.hadoop.hdfsproxy.authorized.userID");
> + FilterChain mockFilterChain = new DummyFilterChain();
> + filter.doFilter(request, response, mockFilterChain);
> + assertEquals(request
> + .getAttribute("org.apache.hadoop.hdfsproxy.authorized.userID"),
> + "testuser");
> +
> + }
> +
> + public void endDoFilter(WebResponse theResponse) {
> + assertEquals("<p>some content</p>", theResponse.getText());
> + }
> +
> +}
>
> Modified: hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/resources/proxy-config/hdfsproxy-default.xml
> URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/resources/proxy-config/hdfsproxy-default.xml?rev=788898&r1=788897&r2=788898&view=diff
> ==============================================================================
> --- hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/resources/proxy-config/hdfsproxy-default.xml (original)
> +++ hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/resources/proxy-config/hdfsproxy-default.xml Fri Jun 26 22:48:23 2009
> @@ -70,5 +70,35 @@
> </description>
> </property>
>
> +<property>
> + <name>hdfsproxy.ldap.initial.context.factory</name>
> + <value>com.sun.jndi.ldap.LdapCtxFactory</value>
> + <description> ldap initial context factory
> + </description>
> +</property>
> +
> +<property>
> + <name>hdfsproxy.ldap.provider.url</name>
> + <value>ldap://ldapserver:389</value>
> + <description> ldap server address
> + </description>
> +</property>
> +
> +<property>
> + <name>hdfsproxy.ldap.role.base</name>
> + <value>ou=proxyroles,dc=mycompany,dc=com</value>
> + <description> ldap role base
> + </description>
> +</property>
> +
> +<property>
> + <name>fs.default.name</name>
> + <!-- cluster variant -->
> + <value>hdfs://localhost:8020</value>
> + <description>The name of the default file system. Either the
> + literal string "local" or a host:port for NDFS.</description>
> + <final>true</final>
> +</property>
> +
> </configuration>
>
>
> Modified: hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/resources/proxy-config/user-certs.xml
> URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/resources/proxy-config/user-certs.xml?rev=788898&r1=788897&r2=788898&view=diff
> ==============================================================================
> --- hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/resources/proxy-config/user-certs.xml (original)
> +++ hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/resources/proxy-config/user-certs.xml Fri Jun 26 22:48:23 2009
> @@ -75,7 +75,7 @@
>
> <property>
> <name> Admin </name>
> - <value>, 5, ,, 3 , 9a2cf0be9ddf8280
> + <value>, 6, ,, 3 , 9a2cf0be9ddf8280
>
>
>
>
> Added: hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/resources/tomcat-web.xml
> URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/resources/tomcat-web.xml?rev=788898&view=auto
> ==============================================================================
> --- hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/resources/tomcat-web.xml (added)
> +++ hadoop/hdfs/trunk/src/contrib/hdfsproxy/src/test/resources/tomcat-web.xml Fri Jun 26 22:48:23 2009
> @@ -0,0 +1,154 @@
> +<?xml version="1.0" encoding="ISO-8859-1"?>
> +<!--
> + Licensed to the Apache Software Foundation (ASF) under one or more
> + contributor license agreements. See the NOTICE file distributed with
> + this work for additional information regarding copyright ownership.
> + The ASF licenses this file to You under the Apache License, Version 2.0
> + (the "License"); you may not use this file except in compliance with
> + the License. You may obtain a copy of the License at
> +
> + http://www.apache.org/licenses/LICENSE-2.0
> +
> + Unless required by applicable law or agreed to in writing, software
> + distributed under the License is distributed on an "AS IS" BASIS,
> + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + See the License for the specific language governing permissions and
> + limitations under the License.
> +-->
> +
> +<!DOCTYPE web-app
> + PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN"
> + "http://java.sun.com/dtd/web-app_2_3.dtd">
> +
> +<web-app>
> +
> +
> + <!-- General description of your web application -->
> +
> + <display-name>HDFS Proxy</display-name>
> + <description>
> + get data from grid
> + </description>
> +
> +
> + <!-- Context initialization parameters that define shared
> + String constants used within your application, which
> + can be customized by the system administrator who is
> + installing your application. The values actually
> + assigned to these parameters can be retrieved in a
> + servlet or JSP page by calling:
> +
> + String value =
> + getServletContext().getInitParameter("name");
> +
> + where "name" matches the <param-name> element of
> + one of these initialization parameters.
> +
> + You can define any number of context initialization
> + parameters, including zero.
> + -->
> +
> + <context-param>
> + <param-name>webmaster</param-name>
> + <param-value>zhiyong1@yahoo-inc.com</param-value>
> + <description>
> + The EMAIL address of the administrator to whom questions
> + and comments about this application should be addressed.
> + </description>
> + </context-param>
> +
> +
> +
> +
> + <!-- Servlet definitions for the servlets that make up
> + your web application, including initialization
> + parameters. With Tomcat, you can also send requests
> + to servlets not listed here with a request like this:
> +
> + http://localhost:8080/{context-path}/servlet/{classname}
> +
> + but this usage is not guaranteed to be portable. It also
> + makes relative references to images and other resources
> + required by your servlet more complicated, so defining
> + all of your servlets (and defining a mapping to them with
> + a servlet-mapping element) is recommended.
> +
> + Servlet initialization parameters can be retrieved in a
> + servlet or JSP page by calling:
> +
> + String value =
> + getServletConfig().getInitParameter("name");
> +
> + where "name" matches the <param-name> element of
> + one of these initialization parameters.
> +
> + You can define any number of servlets, including zero.
> + -->
> +
> + <filter>
> + <filter-name>proxyFilter</filter-name>
> + <filter-class>org.apache.hadoop.hdfsproxy.ProxyFilter</filter-class>
> + </filter>
> +
> + <filter-mapping>
> + <filter-name>proxyFilter</filter-name>
> + <url-pattern>/*</url-pattern>
> + </filter-mapping>
> +
> + <servlet>
> + <servlet-name>listPaths</servlet-name>
> + <description>list paths data access</description>
> + <servlet-class>org.apache.hadoop.hdfsproxy.ProxyListPathsServlet</servlet-class>
> + </servlet>
> +
> + <servlet-mapping>
> + <servlet-name>listPaths</servlet-name>
> + <url-pattern>/listPaths/*</url-pattern>
> + </servlet-mapping>
> +
> + <servlet>
> + <servlet-name>data</servlet-name>
> + <description>data access</description>
> + <servlet-class>org.apache.hadoop.hdfsproxy.ProxyFileDataServlet</servlet-class>
> + </servlet>
> +
> + <servlet-mapping>
> + <servlet-name>data</servlet-name>
> + <url-pattern>/data/*</url-pattern>
> + </servlet-mapping>
> +
> + <servlet>
> + <servlet-name>streamFile</servlet-name>
> + <description>stream file access</description>
> + <servlet-class>org.apache.hadoop.hdfsproxy.ProxyStreamFile</servlet-class>
> + </servlet>
> +
> + <servlet-mapping>
> + <servlet-name>streamFile</servlet-name>
> + <url-pattern>/streamFile/*</url-pattern>
> + </servlet-mapping>
> +
> +
> + <welcome-file-list>
> + <welcome-file>index.html</welcome-file>
> + </welcome-file-list>
> +
> + <!-- Define the default session timeout for your application,
> + in minutes. From a servlet or JSP page, you can modify
> + the timeout for a particular session dynamically by using
> + HttpSession.getMaxInactiveInterval(). -->
> +
> + <session-config>
> + <session-timeout>30</session-timeout> <!-- 30 minutes -->
> + </session-config>
> +
> +
> +</web-app>
> +
> +
> +
> +
> +
> +
> +
> +
>
> Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/HsftpFileSystem.java
> URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/HsftpFileSystem.java?rev=788898&r1=788897&r2=788898&view=diff
> ==============================================================================
> --- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/HsftpFileSystem.java (original)
> +++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/HsftpFileSystem.java Fri Jun 26 22:48:23 2009
> @@ -18,33 +18,40 @@
>
> package org.apache.hadoop.hdfs;
>
> +import java.io.FileInputStream;
> import java.io.IOException;
> import java.net.HttpURLConnection;
> import java.net.URI;
> import java.net.URISyntaxException;
> import java.net.URL;
> +import java.security.KeyStore;
> import java.security.cert.X509Certificate;
>
> import javax.net.ssl.HostnameVerifier;
> import javax.net.ssl.HttpsURLConnection;
> +import javax.net.ssl.KeyManager;
> +import javax.net.ssl.KeyManagerFactory;
> +import javax.net.ssl.SSLContext;
> import javax.net.ssl.SSLSession;
> +import javax.net.ssl.TrustManager;
> +import javax.net.ssl.TrustManagerFactory;
> +import javax.net.ssl.X509TrustManager;
>
> import org.apache.hadoop.conf.Configuration;
>
> -
> -
> -/** An implementation of a protocol for accessing filesystems over HTTPS.
> - * The following implementation provides a limited, read-only interface
> - * to a filesystem over HTTPS.
> +/**
> + * An implementation of a protocol for accessing filesystems over HTTPS. The
> + * following implementation provides a limited, read-only interface to a
> + * filesystem over HTTPS.
> + *
> * @see org.apache.hadoop.hdfs.server.namenode.ListPathsServlet
> * @see org.apache.hadoop.hdfs.server.namenode.FileDataServlet
> */
> public class HsftpFileSystem extends HftpFileSystem {
>
> private static final long MM_SECONDS_PER_DAY = 1000 * 60 * 60 * 24;
> - private volatile int ExpWarnDays = 0;
> -
> -
> + private volatile int ExpWarnDays = 0;
> +
> @Override
> public void initialize(URI name, Configuration conf) throws IOException {
> super.initialize(name, conf);
> @@ -52,40 +59,76 @@
> ExpWarnDays = conf.getInt("ssl.expiration.warn.days", 30);
> }
>
> - /** Set up SSL resources */
> - private static void setupSsl(Configuration conf) {
> + /**
> + * Set up SSL resources
> + *
> + * @throws IOException
> + */
> + private static void setupSsl(Configuration conf) throws IOException {
> Configuration sslConf = new Configuration(false);
> sslConf.addResource(conf.get("dfs.https.client.keystore.resource",
> "ssl-client.xml"));
> - System.setProperty("javax.net.ssl.trustStore", sslConf.get(
> - "ssl.client.truststore.location", ""));
> - System.setProperty("javax.net.ssl.trustStorePassword", sslConf.get(
> - "ssl.client.truststore.password", ""));
> - System.setProperty("javax.net.ssl.trustStoreType", sslConf.get(
> - "ssl.client.truststore.type", "jks"));
> - System.setProperty("javax.net.ssl.keyStore", sslConf.get(
> - "ssl.client.keystore.location", ""));
> - System.setProperty("javax.net.ssl.keyStorePassword", sslConf.get(
> - "ssl.client.keystore.password", ""));
> - System.setProperty("javax.net.ssl.keyPassword", sslConf.get(
> - "ssl.client.keystore.keypassword", ""));
> - System.setProperty("javax.net.ssl.keyStoreType", sslConf.get(
> - "ssl.client.keystore.type", "jks"));
> + FileInputStream fis = null;
> + try {
> + SSLContext sc = SSLContext.getInstance("SSL");
> + KeyManager[] kms = null;
> + TrustManager[] tms = null;
> + if (sslConf.get("ssl.client.keystore.location") != null) {
> + // initialize default key manager with keystore file and pass
> + KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
> + KeyStore ks = KeyStore.getInstance(sslConf.get(
> + "ssl.client.keystore.type", "JKS"));
> + char[] ksPass = sslConf.get("ssl.client.keystore.password", "changeit")
> + .toCharArray();
> + fis = new FileInputStream(sslConf.get("ssl.client.keystore.location",
> + "keystore.jks"));
> + ks.load(fis, ksPass);
> + kmf.init(ks, sslConf.get("ssl.client.keystore.keypassword", "changeit")
> + .toCharArray());
> + kms = kmf.getKeyManagers();
> + fis.close();
> + fis = null;
> + }
> + // initialize default trust manager with truststore file and pass
> + if (conf.getBoolean("ssl.client.do.not.authenticate.server", false)) {
> + // by pass trustmanager validation
> + tms = new DummyTrustManager[] { new DummyTrustManager() };
> + } else {
> + TrustManagerFactory tmf = TrustManagerFactory.getInstance("PKIX");
> + KeyStore ts = KeyStore.getInstance(sslConf.get(
> + "ssl.client.truststore.type", "JKS"));
> + char[] tsPass = sslConf.get("ssl.client.truststore.password",
> + "changeit").toCharArray();
> + fis = new FileInputStream(sslConf.get("ssl.client.truststore.location",
> + "truststore.jks"));
> + ts.load(fis, tsPass);
> + tmf.init(ts);
> + tms = tmf.getTrustManagers();
> + }
> + sc.init(kms, tms, new java.security.SecureRandom());
> + HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
> + } catch (Exception e) {
> + throw new IOException("Could not initialize SSLContext", e);
> + } finally {
> + if (fis != null) {
> + fis.close();
> + }
> + }
> }
> -
> +
> @Override
> protected HttpURLConnection openConnection(String path, String query)
> throws IOException {
> try {
> - final URL url = new URI("https", null, nnAddr.getHostName(),
> - nnAddr.getPort(), path, query, null).toURL();
> - HttpsURLConnection conn = (HttpsURLConnection)url.openConnection();
> + final URL url = new URI("https", null, nnAddr.getHostName(), nnAddr
> + .getPort(), path, query, null).toURL();
> + HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
> // bypass hostname verification
> conn.setHostnameVerifier(new DummyHostnameVerifier());
> conn.setRequestMethod("GET");
> conn.connect();
> -
> - // check cert expiration date
> +
> + // check cert expiration date
> final int warnDays = ExpWarnDays;
> if (warnDays > 0) { // make sure only check once
> ExpWarnDays = 0;
> @@ -100,16 +143,16 @@
> StringBuffer sb = new StringBuffer();
> sb.append("\n Client certificate "
> + cert.getSubjectX500Principal().getName());
> - int dayOffSet = (int) ((expTime - System.currentTimeMillis())/MM_SECONDS_PER_DAY);
> + int dayOffSet = (int) ((expTime - System.currentTimeMillis()) / MM_SECONDS_PER_DAY);
> sb.append(" have " + dayOffSet + " days to expire");
> LOG.warn(sb.toString());
> }
> }
> - }
> + }
> }
> - return (HttpURLConnection)conn;
> + return (HttpURLConnection) conn;
> } catch (URISyntaxException e) {
> - throw (IOException)new IOException().initCause(e);
> + throw (IOException) new IOException().initCause(e);
> }
> }
>
> @@ -117,10 +160,10 @@
> public URI getUri() {
> try {
> return new URI("hsftp", null, nnAddr.getHostName(), nnAddr.getPort(),
> - null, null, null);
> + null, null, null);
> } catch (URISyntaxException e) {
> return null;
> - }
> + }
> }
>
> /**
> @@ -132,4 +175,19 @@
> }
> }
>
> + /**
> + * Dummy trustmanager that is used to trust all server certificates
> + */
> + protected static class DummyTrustManager implements X509TrustManager {
> + public void checkClientTrusted(X509Certificate[] chain, String authType) {
> + }
> +
> + public void checkServerTrusted(X509Certificate[] chain, String authType) {
> + }
> +
> + public X509Certificate[] getAcceptedIssuers() {
> + return null;
> + }
> + }
> +
> }
>
>
>