You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zh...@apache.org on 2015/03/02 18:15:28 UTC
[12/50] [abbrv] hadoop git commit: HDFS-7460. Rewrite httpfs to use
new shell framework (John Smith via aw)
HDFS-7460. Rewrite httpfs to use new shell framework (John Smith via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/21e9e91c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/21e9e91c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/21e9e91c
Branch: refs/heads/HDFS-7285
Commit: 21e9e91c8ab1160a9a756b5f155c62492661bc78
Parents: 746bc37
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed Feb 25 18:57:41 2015 -0800
Committer: Zhe Zhang <zh...@cloudera.com>
Committed: Mon Mar 2 09:13:51 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 2 +-
.../src/main/conf/httpfs-env.sh | 51 +++--
.../src/main/libexec/httpfs-config.sh | 208 ++++---------------
.../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 116 +++++++----
.../src/main/tomcat/ssl-server.xml | 135 ------------
.../src/main/tomcat/ssl-server.xml.conf | 135 ++++++++++++
.../src/site/apt/ServerSetup.apt.vm | 159 --------------
.../src/site/apt/UsingHttpTools.apt.vm | 87 --------
.../src/site/apt/index.apt.vm | 83 --------
.../src/site/markdown/ServerSetup.md.vm | 121 +++++++++++
.../src/site/markdown/UsingHttpTools.md | 62 ++++++
.../src/site/markdown/index.md | 52 +++++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
13 files changed, 533 insertions(+), 680 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21e9e91c/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index 4c42ef9..ddc6033 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -524,7 +524,7 @@
<copy file="${basedir}/src/main/tomcat/server.xml"
toDir="${httpfs.tomcat.dist.dir}/conf"/>
<delete file="${httpfs.tomcat.dist.dir}/conf/ssl-server.xml"/>
- <copy file="${basedir}/src/main/tomcat/ssl-server.xml"
+ <copy file="${basedir}/src/main/tomcat/ssl-server.xml.conf"
toDir="${httpfs.tomcat.dist.dir}/conf"/>
<delete file="${httpfs.tomcat.dist.dir}/conf/logging.properties"/>
<copy file="${basedir}/src/main/tomcat/logging.properties"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21e9e91c/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
index a2701d4..0e8cc40 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
@@ -14,40 +14,59 @@
#
# Set httpfs specific environment variables here.
-
-# Settings for the Embedded Tomcat that runs HttpFS
-# Java System properties for HttpFS should be specified in this variable
#
-# export CATALINA_OPTS=
-
-# HttpFS logs directory
+# hadoop-env.sh is read prior to this file.
#
-# export HTTPFS_LOG=${HTTPFS_HOME}/logs
-# HttpFS temporary directory
+# HTTPFS temporary directory
#
-# export HTTPFS_TEMP=${HTTPFS_HOME}/temp
+# export HTTPFS_TEMP=${HADOOP_PREFIX}/temp
-# The HTTP port used by HttpFS
+# The HTTP port used by HTTPFS
#
# export HTTPFS_HTTP_PORT=14000
-# The Admin port used by HttpFS
+# The Admin port used by HTTPFS
#
-# export HTTPFS_ADMIN_PORT=`expr ${HTTPFS_HTTP_PORT} + 1`
+# export HTTPFS_ADMIN_PORT=$((HTTPFS_HTTP_PORT + 1))
-# The hostname HttpFS server runs on
+# The maximum number of Tomcat handler threads
#
-# export HTTPFS_HTTP_HOSTNAME=`hostname -f`
+# export HTTPFS_MAX_THREADS=1000
-# Indicates if HttpFS is using SSL
+# The hostname HttpFS server runs on
#
-# export HTTPFS_SSL_ENABLED=false
+# export HTTPFS_HTTP_HOSTNAME=$(hostname -f)
# The location of the SSL keystore if using SSL
#
# export HTTPFS_SSL_KEYSTORE_FILE=${HOME}/.keystore
+#
# The password of the SSL keystore if using SSL
#
# export HTTPFS_SSL_KEYSTORE_PASS=password
+
+##
+## Tomcat specific settings
+##
+#
+# Location of tomcat
+#
+# export HTTPFS_CATALINA_HOME=${HADOOP_PREFIX}/share/hadoop/httpfs/tomcat
+
+# Java System properties for HTTPFS should be specified in this variable.
+# The java.library.path and hadoop.home.dir properties are automatically
+# configured. In order to supplement java.library.path,
+# one should add to the JAVA_LIBRARY_PATH env var.
+#
+# export CATALINA_OPTS=
+
+# PID file
+#
+# export CATALINA_PID=${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-httpfs.pid
+
+# Output file
+#
+# export CATALINA_OUT=${HTTPFS_LOG}/hadoop-${HADOOP_IDENT_STRING}-httpfs-${HOSTNAME}.out
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21e9e91c/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
index d4db28f..e520e08 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,183 +13,63 @@
# limitations under the License.
#
-# resolve links - $0 may be a softlink
-PRG="${0}"
-
-while [ -h "${PRG}" ]; do
- ls=`ls -ld "${PRG}"`
- link=`expr "$ls" : '.*-> \(.*\)$'`
- if expr "$link" : '/.*' > /dev/null; then
- PRG="$link"
- else
- PRG=`dirname "${PRG}"`/"$link"
+function hadoop_subproject_init
+{
+ local this
+ local binparent
+ local varlist
+
+ if [[ -z "${HADOOP_HTTPFS_ENV_PROCESSED}" ]]; then
+ if [[ -e "${HADOOP_CONF_DIR}/httpfs-env.sh" ]]; then
+ . "${HADOOP_CONF_DIR}/httpfs-env.sh"
+ export HADOOP_HTTPFS_ENV_PROCESSED=true
+ fi
fi
-done
-BASEDIR=`dirname ${PRG}`
-BASEDIR=`cd ${BASEDIR}/..;pwd`
+ export HADOOP_CATALINA_PREFIX=httpfs
+ export HADOOP_CATALINA_TEMP="${HTTPFS_TEMP:-${HADOOP_PREFIX}/temp}"
-function print() {
- if [ "${HTTPFS_SILENT}" != "true" ]; then
- echo "$@"
- fi
-}
+ hadoop_deprecate_envvar HTTPFS_CONFIG HADOOP_CONF_DIR
-# if HTTPFS_HOME is already set warn it will be ignored
-#
-if [ "${HTTPFS_HOME}" != "" ]; then
- echo "WARNING: current setting of HTTPFS_HOME ignored"
-fi
+ hadoop_deprecate_envvar HTTPFS_LOG HADOOP_LOG_DIR
-print
+ export HADOOP_CATALINA_CONFIG="${HADOOP_CONF_DIR}"
+ export HADOOP_CATALINA_LOG="${HADOOP_LOG_DIR}"
-# setting HTTPFS_HOME to the installation dir, it cannot be changed
-#
-export HTTPFS_HOME=${BASEDIR}
-httpfs_home=${HTTPFS_HOME}
-print "Setting HTTPFS_HOME: ${HTTPFS_HOME}"
+ export HTTPFS_HTTP_HOSTNAME=${HTTPFS_HTTP_HOSTNAME:-$(hostname -f)}
-# if the installation has a env file, source it
-# this is for native packages installations
-#
-if [ -e "${HTTPFS_HOME}/bin/httpfs-env.sh" ]; then
- print "Sourcing: ${HTTPFS_HOME}/bin/httpfs-env.sh"
- source ${HTTPFS_HOME}/bin/httpfs-env.sh
- grep "^ *export " ${HTTPFS_HOME}/bin/httpfs-env.sh | sed 's/ *export/ setting/'
-fi
+ export HADOOP_CATALINA_HTTP_PORT="${HTTPFS_HTTP_PORT:-14000}"
+ export HADOOP_CATALINA_ADMIN_PORT="${HTTPFS_ADMIN_PORT:-$((HADOOP_CATALINA_HTTP_PORT+1))}"
+ export HADOOP_CATALINA_MAX_THREADS="${HTTPFS_MAX_THREADS:-150}"
-# verify that the sourced env file didn't change HTTPFS_HOME
-# if so, warn and revert
-#
-if [ "${HTTPFS_HOME}" != "${httpfs_home}" ]; then
- print "WARN: HTTPFS_HOME resetting to ''${HTTPFS_HOME}'' ignored"
- export HTTPFS_HOME=${httpfs_home}
- print " using HTTPFS_HOME: ${HTTPFS_HOME}"
-fi
+ export HTTPFS_SSL_ENABLED=${HTTPFS_SSL_ENABLED:-false}
-if [ "${HTTPFS_CONFIG}" = "" ]; then
- export HTTPFS_CONFIG=${HTTPFS_HOME}/etc/hadoop
- print "Setting HTTPFS_CONFIG: ${HTTPFS_CONFIG}"
-else
- print "Using HTTPFS_CONFIG: ${HTTPFS_CONFIG}"
-fi
-httpfs_config=${HTTPFS_CONFIG}
+ export HADOOP_CATALINA_SSL_KEYSTORE_FILE="${HTTPFS_SSL_KEYSTORE_FILE:-${HOME}/.keystore}"
-# if the configuration dir has a env file, source it
-#
-if [ -e "${HTTPFS_CONFIG}/httpfs-env.sh" ]; then
- print "Sourcing: ${HTTPFS_CONFIG}/httpfs-env.sh"
- source ${HTTPFS_CONFIG}/httpfs-env.sh
- grep "^ *export " ${HTTPFS_CONFIG}/httpfs-env.sh | sed 's/ *export/ setting/'
-fi
+ export CATALINA_BASE="${CATALINA_BASE:-${HADOOP_PREFIX}/share/hadoop/httpfs/tomcat}"
+ export HADOOP_CATALINA_HOME="${HTTPFS_CATALINA_HOME:-${CATALINA_BASE}}"
-# verify that the sourced env file didn't change HTTPFS_HOME
-# if so, warn and revert
-#
-if [ "${HTTPFS_HOME}" != "${httpfs_home}" ]; then
- echo "WARN: HTTPFS_HOME resetting to ''${HTTPFS_HOME}'' ignored"
- export HTTPFS_HOME=${httpfs_home}
-fi
-
-# verify that the sourced env file didn't change HTTPFS_CONFIG
-# if so, warn and revert
-#
-if [ "${HTTPFS_CONFIG}" != "${httpfs_config}" ]; then
- echo "WARN: HTTPFS_CONFIG resetting to ''${HTTPFS_CONFIG}'' ignored"
- export HTTPFS_CONFIG=${httpfs_config}
-fi
-
-if [ "${HTTPFS_LOG}" = "" ]; then
- export HTTPFS_LOG=${HTTPFS_HOME}/logs
- print "Setting HTTPFS_LOG: ${HTTPFS_LOG}"
-else
- print "Using HTTPFS_LOG: ${HTTPFS_LOG}"
-fi
+ export CATALINA_OUT="${CATALINA_OUT:-${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-httpfs-${HOSTNAME}.out}"
-if [ ! -f ${HTTPFS_LOG} ]; then
- mkdir -p ${HTTPFS_LOG}
-fi
-
-if [ "${HTTPFS_TEMP}" = "" ]; then
- export HTTPFS_TEMP=${HTTPFS_HOME}/temp
- print "Setting HTTPFS_TEMP: ${HTTPFS_TEMP}"
-else
- print "Using HTTPFS_TEMP: ${HTTPFS_TEMP}"
-fi
+ export CATALINA_PID="${CATALINA_PID:-${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-httpfs.pid}"
-if [ ! -f ${HTTPFS_TEMP} ]; then
- mkdir -p ${HTTPFS_TEMP}
-fi
-
-if [ "${HTTPFS_HTTP_PORT}" = "" ]; then
- export HTTPFS_HTTP_PORT=14000
- print "Setting HTTPFS_HTTP_PORT: ${HTTPFS_HTTP_PORT}"
-else
- print "Using HTTPFS_HTTP_PORT: ${HTTPFS_HTTP_PORT}"
-fi
-
-if [ "${HTTPFS_ADMIN_PORT}" = "" ]; then
- export HTTPFS_ADMIN_PORT=`expr $HTTPFS_HTTP_PORT + 1`
- print "Setting HTTPFS_ADMIN_PORT: ${HTTPFS_ADMIN_PORT}"
-else
- print "Using HTTPFS_ADMIN_PORT: ${HTTPFS_ADMIN_PORT}"
-fi
-
-if [ "${HTTPFS_HTTP_HOSTNAME}" = "" ]; then
- export HTTPFS_HTTP_HOSTNAME=`hostname -f`
- print "Setting HTTPFS_HTTP_HOSTNAME: ${HTTPFS_HTTP_HOSTNAME}"
-else
- print "Using HTTPFS_HTTP_HOSTNAME: ${HTTPFS_HTTP_HOSTNAME}"
-fi
-
-if [ "${HTTPFS_SSL_ENABLED}" = "" ]; then
- export HTTPFS_SSL_ENABLED="false"
- print "Setting HTTPFS_SSL_ENABLED: ${HTTPFS_SSL_ENABLED}"
-else
- print "Using HTTPFS_SSL_ENABLED: ${HTTPFS_SSL_ENABLED}"
-fi
-
-if [ "${HTTPFS_SSL_KEYSTORE_FILE}" = "" ]; then
- export HTTPFS_SSL_KEYSTORE_FILE=${HOME}/.keystore
- print "Setting HTTPFS_SSL_KEYSTORE_FILE: ${HTTPFS_SSL_KEYSTORE_FILE}"
-else
- print "Using HTTPFS_SSL_KEYSTORE_FILE: ${HTTPFS_SSL_KEYSTORE_FILE}"
-fi
-
-if [ "${HTTPFS_SSL_KEYSTORE_PASS}" = "" ]; then
- export HTTPFS_SSL_KEYSTORE_PASS=password
- print "Setting HTTPFS_SSL_KEYSTORE_PASS: ${HTTPFS_SSL_KEYSTORE_PASS}"
-else
- print "Using HTTPFS_SSL_KEYSTORE_PASS: ${HTTPFS_SSL_KEYSTORE_PASS}"
-fi
-
-if [ "${CATALINA_BASE}" = "" ]; then
- export CATALINA_BASE=${HTTPFS_HOME}/share/hadoop/httpfs/tomcat
- print "Setting CATALINA_BASE: ${CATALINA_BASE}"
-else
- print "Using CATALINA_BASE: ${CATALINA_BASE}"
-fi
-
-if [ "${HTTPFS_CATALINA_HOME}" = "" ]; then
- export HTTPFS_CATALINA_HOME=${CATALINA_BASE}
- print "Setting HTTPFS_CATALINA_HOME: ${HTTPFS_CATALINA_HOME}"
-else
- print "Using HTTPFS_CATALINA_HOME: ${HTTPFS_CATALINA_HOME}"
-fi
-
-if [ "${CATALINA_OUT}" = "" ]; then
- export CATALINA_OUT=${HTTPFS_LOG}/httpfs-catalina.out
- print "Setting CATALINA_OUT: ${CATALINA_OUT}"
-else
- print "Using CATALINA_OUT: ${CATALINA_OUT}"
-fi
+ if [[ -n "${HADOOP_SHELL_SCRIPT_DEBUG}" ]]; then
+ varlist=$(env | egrep '(^HTTPFS|^CATALINA)' | cut -f1 -d= | grep -v _PASS)
+ for i in ${varlist}; do
+ hadoop_debug "Setting ${i} to ${!i}"
+ done
+ fi
+}
-if [ "${CATALINA_PID}" = "" ]; then
- export CATALINA_PID=/tmp/httpfs.pid
- print "Setting CATALINA_PID: ${CATALINA_PID}"
+if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
+ [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then
+ . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh"
+elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
+ . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
+elif [[ -e "${HADOOP_PREFIX}/libexec/hadoop-config.sh" ]]; then
+ . "${HADOOP_PREFIX}/libexec/hadoop-config.sh"
else
- print "Using CATALINA_PID: ${CATALINA_PID}"
+ echo "ERROR: Hadoop common not found." 2>&1
+ exit 1
fi
-
-print
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21e9e91c/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
index 65903dc..72dca93 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,53 +13,99 @@
# limitations under the License.
#
-# resolve links - $0 may be a softlink
-PRG="${0}"
+function hadoop_usage()
+{
+ echo "Usage: httpfs.sh [--config confdir] [--debug] --daemon start|status|stop"
+ echo " httpfs.sh [--config confdir] [--debug] COMMAND"
+ echo " where COMMAND is one of:"
+ echo " run Start httpfs in the current window"
+ echo " run -security Start in the current window with security manager"
+ echo " start Start httpfs in a separate window"
+ echo " start -security Start in a separate window with security manager"
+ echo " status Return the LSB compliant status"
+ echo " stop Stop httpfs, waiting up to 5 seconds for the process to end"
+ echo " stop n Stop httpfs, waiting up to n seconds for the process to end"
+ echo " stop -force Stop httpfs, wait up to 5 seconds and then use kill -KILL if still running"
+ echo " stop n -force Stop httpfs, wait up to n seconds and then use kill -KILL if still running"
+}
-while [ -h "${PRG}" ]; do
- ls=`ls -ld "${PRG}"`
- link=`expr "$ls" : '.*-> \(.*\)$'`
- if expr "$link" : '/.*' > /dev/null; then
- PRG="$link"
- else
- PRG=`dirname "${PRG}"`/"$link"
- fi
-done
-
-BASEDIR=`dirname ${PRG}`
-BASEDIR=`cd ${BASEDIR}/..;pwd`
+# let's locate libexec...
+if [[ -n "${HADOOP_PREFIX}" ]]; then
+ DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+else
+ this="${BASH_SOURCE-$0}"
+ bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+ DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
+fi
-source ${HADOOP_LIBEXEC_DIR:-${BASEDIR}/libexec}/httpfs-config.sh
+HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
+# shellcheck disable=SC2034
+HADOOP_NEW_CONFIG=true
+if [[ -f "${HADOOP_LIBEXEC_DIR}/httpfs-config.sh" ]]; then
+ . "${HADOOP_LIBEXEC_DIR}/httpfs-config.sh"
+else
+ echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/httpfs-config.sh." 2>&1
+ exit 1
+fi
-# The Java System property 'httpfs.http.port' it is not used by HttpFS,
+# The Java System property 'httpfs.http.port' it is not used by Kms,
# it is used in Tomcat's server.xml configuration file
#
-print "Using CATALINA_OPTS: ${CATALINA_OPTS}"
-catalina_opts="-Dhttpfs.home.dir=${HTTPFS_HOME}";
-catalina_opts="${catalina_opts} -Dhttpfs.config.dir=${HTTPFS_CONFIG}";
-catalina_opts="${catalina_opts} -Dhttpfs.log.dir=${HTTPFS_LOG}";
-catalina_opts="${catalina_opts} -Dhttpfs.temp.dir=${HTTPFS_TEMP}";
-catalina_opts="${catalina_opts} -Dhttpfs.admin.port=${HTTPFS_ADMIN_PORT}";
-catalina_opts="${catalina_opts} -Dhttpfs.http.port=${HTTPFS_HTTP_PORT}";
-catalina_opts="${catalina_opts} -Dhttpfs.http.hostname=${HTTPFS_HTTP_HOSTNAME}";
-catalina_opts="${catalina_opts} -Dhttpfs.ssl.enabled=${HTTPFS_SSL_ENABLED}";
-catalina_opts="${catalina_opts} -Dhttpfs.ssl.keystore.file=${HTTPFS_SSL_KEYSTORE_FILE}";
-catalina_opts="${catalina_opts} -Dhttpfs.ssl.keystore.pass=${HTTPFS_SSL_KEYSTORE_PASS}";
+# Mask the trustStorePassword
+# shellcheck disable=SC2086
+CATALINA_OPTS_DISP="$(echo ${CATALINA_OPTS} | sed -e 's/trustStorePassword=[^ ]*/trustStorePassword=***/')"
-print "Adding to CATALINA_OPTS: ${catalina_opts}"
+hadoop_debug "Using CATALINA_OPTS: ${CATALINA_OPTS_DISP}"
-export CATALINA_OPTS="${CATALINA_OPTS} ${catalina_opts}"
+# We're using hadoop-common, so set up some stuff it might need:
+hadoop_finalize
+
+hadoop_verify_logdir
+
+if [[ $# = 0 ]]; then
+ case "${HADOOP_DAEMON_MODE}" in
+ status)
+ hadoop_status_daemon "${CATALINA_PID}"
+ exit
+ ;;
+ start)
+ set -- "start"
+ ;;
+ stop)
+ set -- "stop"
+ ;;
+ esac
+fi
+
+hadoop_finalize_catalina_opts
+export CATALINA_OPTS
# A bug in catalina.sh script does not use CATALINA_OPTS for stopping the server
#
-if [ "${1}" = "stop" ]; then
+if [[ "${1}" = "stop" ]]; then
export JAVA_OPTS=${CATALINA_OPTS}
fi
-if [ "${HTTPFS_SILENT}" != "true" ]; then
- exec ${HTTPFS_CATALINA_HOME}/bin/catalina.sh "$@"
-else
- exec ${HTTPFS_CATALINA_HOME}/bin/catalina.sh "$@" > /dev/null
+# If ssl, the populate the passwords into ssl-server.xml before starting tomcat
+#
+# HTTPFS_SSL_KEYSTORE_PASS is a bit odd.
+# if undefined, then the if test will not enable ssl on its own
+# if "", set it to "password".
+# if custom, use provided password
+#
+if [[ -f "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml.conf" ]]; then
+ if [[ -n "${HTTPFS_SSL_KEYSTORE_PASS+x}" ]] || [[ -n "${HTTPFS_SSL_TRUSTSTORE_PASS}" ]]; then
+ export HTTPFS_SSL_KEYSTORE_PASS=${HTTPFS_SSL_KEYSTORE_PASS:-password}
+ sed -e 's/_httpfs_ssl_keystore_pass_/'${HTTPFS_SSL_KEYSTORE_PASS}'/g' \
+ -e 's/_httpfs_ssl_truststore_pass_/'${HTTPFS_SSL_TRUSTSTORE_PASS}'/g' \
+ "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml.conf" \
+ > "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml"
+ chmod 700 "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml" >/dev/null 2>&1
+ fi
fi
+hadoop_add_param CATALINA_OPTS -Dhttpfs.http.hostname "-Dhttpfs.http.hostname=${HTTPFS_HOST_NAME}"
+hadoop_add_param CATALINA_OPTS -Dhttpfs.ssl.enabled "-Dhttpfs.ssl.enabled=${HTTPFS_SSL_ENABLED}"
+
+exec "${HADOOP_CATALINA_HOME}/bin/catalina.sh" "$@"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21e9e91c/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml
deleted file mode 100644
index c91c2e2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml
+++ /dev/null
@@ -1,135 +0,0 @@
-<?xml version='1.0' encoding='utf-8'?>
-<!--
-
- All Rights Reserved.
-
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!-- Note: A "Server" is not itself a "Container", so you may not
- define subcomponents such as "Valves" at this level.
- Documentation at /docs/config/server.html
- -->
-<Server port="${httpfs.admin.port}" shutdown="SHUTDOWN">
-
- <!--APR library loader. Documentation at /docs/apr.html -->
- <Listener className="org.apache.catalina.core.AprLifecycleListener"
- SSLEngine="on"/>
- <!--Initialize Jasper prior to webapps are loaded. Documentation at /docs/jasper-howto.html -->
- <Listener className="org.apache.catalina.core.JasperListener"/>
- <!-- Prevent memory leaks due to use of particular java/javax APIs-->
- <Listener
- className="org.apache.catalina.core.JreMemoryLeakPreventionListener"/>
- <!-- JMX Support for the Tomcat server. Documentation at /docs/non-existent.html -->
- <Listener className="org.apache.catalina.mbeans.ServerLifecycleListener"/>
- <Listener
- className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener"/>
-
- <!-- Global JNDI resources
- Documentation at /docs/jndi-resources-howto.html
- -->
- <GlobalNamingResources>
- <!-- Editable user database that can also be used by
- UserDatabaseRealm to authenticate users
- -->
- <Resource name="UserDatabase" auth="Container"
- type="org.apache.catalina.UserDatabase"
- description="User database that can be updated and saved"
- factory="org.apache.catalina.users.MemoryUserDatabaseFactory"
- pathname="conf/tomcat-users.xml"/>
- </GlobalNamingResources>
-
- <!-- A "Service" is a collection of one or more "Connectors" that share
- a single "Container" Note: A "Service" is not itself a "Container",
- so you may not define subcomponents such as "Valves" at this level.
- Documentation at /docs/config/service.html
- -->
- <Service name="Catalina">
-
- <!--The connectors can use a shared executor, you can define one or more named thread pools-->
- <!--
- <Executor name="tomcatThreadPool" namePrefix="catalina-exec-"
- maxThreads="150" minSpareThreads="4"/>
- -->
-
- <!-- Define a SSL HTTP/1.1 Connector on port 8443
- This connector uses the JSSE configuration, when using APR, the
- connector should be using the OpenSSL style configuration
- described in the APR documentation -->
- <Connector port="${httpfs.http.port}" protocol="HTTP/1.1" SSLEnabled="true"
- maxThreads="150" scheme="https" secure="true"
- clientAuth="false" sslEnabledProtocols="TLSv1,SSLv2Hello"
- keystoreFile="${httpfs.ssl.keystore.file}"
- keystorePass="${httpfs.ssl.keystore.pass}"/>
-
- <!-- Define an AJP 1.3 Connector on port 8009 -->
-
-
- <!-- An Engine represents the entry point (within Catalina) that processes
- every request. The Engine implementation for Tomcat stand alone
- analyzes the HTTP headers included with the request, and passes them
- on to the appropriate Host (virtual host).
- Documentation at /docs/config/engine.html -->
-
- <!-- You should set jvmRoute to support load-balancing via AJP ie :
- <Engine name="Catalina" defaultHost="localhost" jvmRoute="jvm1">
- -->
- <Engine name="Catalina" defaultHost="localhost">
-
- <!--For clustering, please take a look at documentation at:
- /docs/cluster-howto.html (simple how to)
- /docs/config/cluster.html (reference documentation) -->
- <!--
- <Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"/>
- -->
-
- <!-- The request dumper valve dumps useful debugging information about
- the request and response data received and sent by Tomcat.
- Documentation at: /docs/config/valve.html -->
- <!--
- <Valve className="org.apache.catalina.valves.RequestDumperValve"/>
- -->
-
- <!-- This Realm uses the UserDatabase configured in the global JNDI
- resources under the key "UserDatabase". Any edits
- that are performed against this UserDatabase are immediately
- available for use by the Realm. -->
- <Realm className="org.apache.catalina.realm.UserDatabaseRealm"
- resourceName="UserDatabase"/>
-
- <!-- Define the default virtual host
- Note: XML Schema validation will not work with Xerces 2.2.
- -->
- <Host name="localhost" appBase="webapps"
- unpackWARs="true" autoDeploy="true"
- xmlValidation="false" xmlNamespaceAware="false">
-
- <!-- SingleSignOn valve, share authentication between web applications
- Documentation at: /docs/config/valve.html -->
- <!--
- <Valve className="org.apache.catalina.authenticator.SingleSignOn" />
- -->
-
- <!-- Access log processes all example.
- Documentation at: /docs/config/valve.html -->
- <!--
- <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
- prefix="localhost_access_log." suffix=".txt" pattern="common" resolveHosts="false"/>
- -->
-
- </Host>
- </Engine>
- </Service>
-</Server>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21e9e91c/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml.conf
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml.conf b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml.conf
new file mode 100644
index 0000000..4a90532
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml.conf
@@ -0,0 +1,135 @@
+<?xml version='1.0' encoding='utf-8'?>
+<!--
+
+ All Rights Reserved.
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- Note: A "Server" is not itself a "Container", so you may not
+ define subcomponents such as "Valves" at this level.
+ Documentation at /docs/config/server.html
+ -->
+<Server port="${httpfs.admin.port}" shutdown="SHUTDOWN">
+
+ <!--APR library loader. Documentation at /docs/apr.html -->
+ <Listener className="org.apache.catalina.core.AprLifecycleListener"
+ SSLEngine="on"/>
+ <!--Initialize Jasper prior to webapps are loaded. Documentation at /docs/jasper-howto.html -->
+ <Listener className="org.apache.catalina.core.JasperListener"/>
+ <!-- Prevent memory leaks due to use of particular java/javax APIs-->
+ <Listener
+ className="org.apache.catalina.core.JreMemoryLeakPreventionListener"/>
+ <!-- JMX Support for the Tomcat server. Documentation at /docs/non-existent.html -->
+ <Listener className="org.apache.catalina.mbeans.ServerLifecycleListener"/>
+ <Listener
+ className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener"/>
+
+ <!-- Global JNDI resources
+ Documentation at /docs/jndi-resources-howto.html
+ -->
+ <GlobalNamingResources>
+ <!-- Editable user database that can also be used by
+ UserDatabaseRealm to authenticate users
+ -->
+ <Resource name="UserDatabase" auth="Container"
+ type="org.apache.catalina.UserDatabase"
+ description="User database that can be updated and saved"
+ factory="org.apache.catalina.users.MemoryUserDatabaseFactory"
+ pathname="conf/tomcat-users.xml"/>
+ </GlobalNamingResources>
+
+ <!-- A "Service" is a collection of one or more "Connectors" that share
+ a single "Container" Note: A "Service" is not itself a "Container",
+ so you may not define subcomponents such as "Valves" at this level.
+ Documentation at /docs/config/service.html
+ -->
+ <Service name="Catalina">
+
+ <!--The connectors can use a shared executor, you can define one or more named thread pools-->
+ <!--
+ <Executor name="tomcatThreadPool" namePrefix="catalina-exec-"
+ maxThreads="httpfs.max.threads" minSpareThreads="4"/>
+ -->
+
+ <!-- Define a SSL HTTP/1.1 Connector on port 8443
+ This connector uses the JSSE configuration, when using APR, the
+ connector should be using the OpenSSL style configuration
+ described in the APR documentation -->
+ <Connector port="${httpfs.http.port}" protocol="HTTP/1.1" SSLEnabled="true"
+ maxThreads="150" scheme="https" secure="true"
+ clientAuth="false" sslEnabledProtocols="TLSv1,SSLv2Hello"
+ keystoreFile="${httpfs.ssl.keystore.file}"
+ keystorePass="_httpfs_ssl_keystore_pass_"/>
+
+ <!-- Define an AJP 1.3 Connector on port 8009 -->
+
+
+ <!-- An Engine represents the entry point (within Catalina) that processes
+ every request. The Engine implementation for Tomcat stand alone
+ analyzes the HTTP headers included with the request, and passes them
+ on to the appropriate Host (virtual host).
+ Documentation at /docs/config/engine.html -->
+
+ <!-- You should set jvmRoute to support load-balancing via AJP ie :
+ <Engine name="Catalina" defaultHost="localhost" jvmRoute="jvm1">
+ -->
+ <Engine name="Catalina" defaultHost="localhost">
+
+ <!--For clustering, please take a look at documentation at:
+ /docs/cluster-howto.html (simple how to)
+ /docs/config/cluster.html (reference documentation) -->
+ <!--
+ <Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"/>
+ -->
+
+ <!-- The request dumper valve dumps useful debugging information about
+ the request and response data received and sent by Tomcat.
+ Documentation at: /docs/config/valve.html -->
+ <!--
+ <Valve className="org.apache.catalina.valves.RequestDumperValve"/>
+ -->
+
+ <!-- This Realm uses the UserDatabase configured in the global JNDI
+ resources under the key "UserDatabase". Any edits
+ that are performed against this UserDatabase are immediately
+ available for use by the Realm. -->
+ <Realm className="org.apache.catalina.realm.UserDatabaseRealm"
+ resourceName="UserDatabase"/>
+
+ <!-- Define the default virtual host
+ Note: XML Schema validation will not work with Xerces 2.2.
+ -->
+ <Host name="localhost" appBase="webapps"
+ unpackWARs="true" autoDeploy="true"
+ xmlValidation="false" xmlNamespaceAware="false">
+
+ <!-- SingleSignOn valve, share authentication between web applications
+ Documentation at: /docs/config/valve.html -->
+ <!--
+ <Valve className="org.apache.catalina.authenticator.SingleSignOn" />
+ -->
+
+ <!-- Access log processes all example.
+ Documentation at: /docs/config/valve.html -->
+ <!--
+ <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
+ prefix="localhost_access_log." suffix=".txt" pattern="common" resolveHosts="false"/>
+ -->
+
+ </Host>
+ </Engine>
+ </Service>
+</Server>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21e9e91c/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/ServerSetup.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/ServerSetup.apt.vm b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/ServerSetup.apt.vm
deleted file mode 100644
index 878ab1f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/ServerSetup.apt.vm
+++ /dev/null
@@ -1,159 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License.
-
- ---
- Hadoop HDFS over HTTP ${project.version} - Server Setup
- ---
- ---
- ${maven.build.timestamp}
-
-Hadoop HDFS over HTTP ${project.version} - Server Setup
-
- This page explains how to quickly setup HttpFS with Pseudo authentication
- against a Hadoop cluster with Pseudo authentication.
-
-* Requirements
-
- * Java 6+
-
- * Maven 3+
-
-* Install HttpFS
-
-+---+
-~ $ tar xzf httpfs-${project.version}.tar.gz
-+---+
-
-* Configure HttpFS
-
- By default, HttpFS assumes that Hadoop configuration files
- (<<<core-site.xml & hdfs-site.xml>>>) are in the HttpFS
- configuration directory.
-
- If this is not the case, add to the <<<httpfs-site.xml>>> file the
- <<<httpfs.hadoop.config.dir>>> property set to the location
- of the Hadoop configuration directory.
-
-* Configure Hadoop
-
- Edit Hadoop <<<core-site.xml>>> and defined the Unix user that will
- run the HttpFS server as a proxyuser. For example:
-
-+---+
- ...
- <property>
- <name>hadoop.proxyuser.#HTTPFSUSER#.hosts</name>
- <value>httpfs-host.foo.com</value>
- </property>
- <property>
- <name>hadoop.proxyuser.#HTTPFSUSER#.groups</name>
- <value>*</value>
- </property>
- ...
-+---+
-
- IMPORTANT: Replace <<<#HTTPFSUSER#>>> with the Unix user that will
- start the HttpFS server.
-
-* Restart Hadoop
-
- You need to restart Hadoop for the proxyuser configuration ot become
- active.
-
-* Start/Stop HttpFS
-
- To start/stop HttpFS use HttpFS's bin/httpfs.sh script. For example:
-
-+---+
-httpfs-${project.version} $ bin/httpfs.sh start
-+---+
-
- NOTE: Invoking the script without any parameters list all possible
- parameters (start, stop, run, etc.). The <<<httpfs.sh>>> script is a wrapper
- for Tomcat's <<<catalina.sh>>> script that sets the environment variables
- and Java System properties required to run HttpFS server.
-
-* Test HttpFS is working
-
-+---+
-~ $ curl -i "http://<HTTPFSHOSTNAME>:14000?user.name=babu&op=homedir"
-HTTP/1.1 200 OK
-Content-Type: application/json
-Transfer-Encoding: chunked
-
-{"homeDir":"http:\/\/<HTTPFS_HOST>:14000\/user\/babu"}
-+---+
-
-* Embedded Tomcat Configuration
-
- To configure the embedded Tomcat go to the <<<tomcat/conf>>>.
-
- HttpFS preconfigures the HTTP and Admin ports in Tomcat's <<<server.xml>>> to
- 14000 and 14001.
-
- Tomcat logs are also preconfigured to go to HttpFS's <<<logs/>>> directory.
-
- The following environment variables (which can be set in HttpFS's
- <<<conf/httpfs-env.sh>>> script) can be used to alter those values:
-
- * HTTPFS_HTTP_PORT
-
- * HTTPFS_ADMIN_PORT
-
- * HTTPFS_LOG
-
-* HttpFS Configuration
-
- HttpFS supports the following {{{./httpfs-default.html}configuration properties}}
- in the HttpFS's <<<conf/httpfs-site.xml>>> configuration file.
-
-* HttpFS over HTTPS (SSL)
-
- To configure HttpFS to work over SSL edit the {{httpfs-env.sh}} script in the
- configuration directory setting the {{HTTPFS_SSL_ENABLED}} to {{true}}.
-
- In addition, the following 2 properties may be defined (shown with default
- values):
-
- * HTTPFS_SSL_KEYSTORE_FILE=${HOME}/.keystore
-
- * HTTPFS_SSL_KEYSTORE_PASS=password
-
- In the HttpFS <<<tomcat/conf>>> directory, replace the <<<server.xml>>> file
- with the <<<ssl-server.xml>>> file.
-
-
- You need to create an SSL certificate for the HttpFS server. As the
- <<<httpfs>>> Unix user, using the Java <<<keytool>>> command to create the
- SSL certificate:
-
-+---+
-$ keytool -genkey -alias tomcat -keyalg RSA
-+---+
-
- You will be asked a series of questions in an interactive prompt. It will
- create the keystore file, which will be named <<.keystore>> and located in the
- <<<httpfs>>> user home directory.
-
- The password you enter for "keystore password" must match the value of the
- <<<HTTPFS_SSL_KEYSTORE_PASS>>> environment variable set in the
- <<<httpfs-env.sh>>> script in the configuration directory.
-
- The answer to "What is your first and last name?" (i.e. "CN") must be the
- hostname of the machine where the HttpFS Server will be running.
-
- Start HttpFS. It should work over HTTPS.
-
- Using the Hadoop <<<FileSystem>>> API or the Hadoop FS shell, use the
- <<<swebhdfs://>>> scheme. Make sure the JVM is picking up the truststore
- containing the public key of the SSL certificate if using a self-signed
- certificate.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21e9e91c/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/UsingHttpTools.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/UsingHttpTools.apt.vm b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/UsingHttpTools.apt.vm
deleted file mode 100644
index c93e20b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/UsingHttpTools.apt.vm
+++ /dev/null
@@ -1,87 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License.
-
- ---
- Hadoop HDFS over HTTP ${project.version} - Using HTTP Tools
- ---
- ---
- ${maven.build.timestamp}
-
-Hadoop HDFS over HTTP ${project.version} - Using HTTP Tools
-
-* Security
-
- Out of the box HttpFS supports both pseudo authentication and Kerberos HTTP
- SPNEGO authentication.
-
-** Pseudo Authentication
-
- With pseudo authentication the user name must be specified in the
- <<<user.name=\<USERNAME\>>>> query string parameter of a HttpFS URL.
- For example:
-
-+---+
-$ curl "http://<HTTFS_HOST>:14000/webhdfs/v1?op=homedir&user.name=babu"
-+---+
-
-** Kerberos HTTP SPNEGO Authentication
-
- Kerberos HTTP SPNEGO authentication requires a tool or library supporting
- Kerberos HTTP SPNEGO protocol.
-
- IMPORTANT: If using <<<curl>>>, the <<<curl>>> version being used must support
- GSS (<<<curl -V>>> prints out 'GSS' if it supports it).
-
- For example:
-
-+---+
-$ kinit
-Please enter the password for tucu@LOCALHOST:
-$ curl --negotiate -u foo "http://<HTTPFS_HOST>:14000/webhdfs/v1?op=homedir"
-Enter host password for user 'foo':
-+---+
-
- NOTE: the <<<-u USER>>> option is required by the <<<--negotiate>>> but it is
- not used. Use any value as <<<USER>>> and when asked for the password press
- [ENTER] as the password value is ignored.
-
-** {Remembering Who I Am} (Establishing an Authenticated Session)
-
- As most authentication mechanisms, Hadoop HTTP authentication authenticates
- users once and issues a short-lived authentication token to be presented in
- subsequent requests. This authentication token is a signed HTTP Cookie.
-
- When using tools like <<<curl>>>, the authentication token must be stored on
- the first request doing authentication, and submitted in subsequent requests.
- To do this with curl the <<<-b>>> and <<<-c>>> options to save and send HTTP
- Cookies must be used.
-
- For example, the first request doing authentication should save the received
- HTTP Cookies.
-
- Using Pseudo Authentication:
-
-+---+
-$ curl -c ~/.httpfsauth "http://<HTTPFS_HOST>:14000/webhdfs/v1?op=homedir&user.name=babu"
-+---+
-
- Using Kerberos HTTP SPNEGO authentication:
-
-+---+
-$ curl --negotiate -u foo -c ~/.httpfsauth "http://<HTTPFS_HOST>:14000/webhdfs/v1?op=homedir"
-+---+
-
- Then, subsequent requests forward the previously received HTTP Cookie:
-
-+---+
-$ curl -b ~/.httpfsauth "http://<HTTPFS_HOST>:14000/webhdfs/v1?op=liststatus"
-+---+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21e9e91c/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/index.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/index.apt.vm b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/index.apt.vm
deleted file mode 100644
index f51e743..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/index.apt.vm
+++ /dev/null
@@ -1,83 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License.
-
- ---
- Hadoop HDFS over HTTP - Documentation Sets ${project.version}
- ---
- ---
- ${maven.build.timestamp}
-
-Hadoop HDFS over HTTP - Documentation Sets ${project.version}
-
- HttpFS is a server that provides a REST HTTP gateway supporting all HDFS
- File System operations (read and write). And it is inteoperable with the
- <<webhdfs>> REST HTTP API.
-
- HttpFS can be used to transfer data between clusters running different
- versions of Hadoop (overcoming RPC versioning issues), for example using
- Hadoop DistCP.
-
- HttpFS can be used to access data in HDFS on a cluster behind of a firewall
- (the HttpFS server acts as a gateway and is the only system that is allowed
- to cross the firewall into the cluster).
-
- HttpFS can be used to access data in HDFS using HTTP utilities (such as curl
- and wget) and HTTP libraries Perl from other languages than Java.
-
- The <<webhdfs>> client FileSytem implementation can be used to access HttpFS
- using the Hadoop filesystem command (<<<hadoop fs>>>) line tool as well as
- from Java aplications using the Hadoop FileSystem Java API.
-
- HttpFS has built-in security supporting Hadoop pseudo authentication and
- HTTP SPNEGO Kerberos and other pluggable authentication mechanims. It also
- provides Hadoop proxy user support.
-
-* How Does HttpFS Works?
-
- HttpFS is a separate service from Hadoop NameNode.
-
- HttpFS itself is Java web-application and it runs using a preconfigured Tomcat
- bundled with HttpFS binary distribution.
-
- HttpFS HTTP web-service API calls are HTTP REST calls that map to a HDFS file
- system operation. For example, using the <<<curl>>> Unix command:
-
- * <<<$ curl http://httpfs-host:14000/webhdfs/v1/user/foo/README.txt>>> returns
- the contents of the HDFS <<</user/foo/README.txt>>> file.
-
- * <<<$ curl http://httpfs-host:14000/webhdfs/v1/user/foo?op=list>>> returns the
- contents of the HDFS <<</user/foo>>> directory in JSON format.
-
- * <<<$ curl -X POST http://httpfs-host:14000/webhdfs/v1/user/foo/bar?op=mkdirs>>>
- creates the HDFS <<</user/foo.bar>>> directory.
-
-* How HttpFS and Hadoop HDFS Proxy differ?
-
- HttpFS was inspired by Hadoop HDFS proxy.
-
- HttpFS can be seen as a full rewrite of Hadoop HDFS proxy.
-
- Hadoop HDFS proxy provides a subset of file system operations (read only),
- HttpFS provides support for all file system operations.
-
- HttpFS uses a clean HTTP REST API making its use with HTTP tools more
- intuitive.
-
- HttpFS supports Hadoop pseudo authentication, Kerberos SPNEGOS authentication
- and Hadoop proxy users. Hadoop HDFS proxy did not.
-
-* User and Developer Documentation
-
- * {{{./ServerSetup.html}HttpFS Server Setup}}
-
- * {{{./UsingHttpTools.html}Using HTTP Tools}}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21e9e91c/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
new file mode 100644
index 0000000..3c7f9d3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
@@ -0,0 +1,121 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Hadoop HDFS over HTTP - Server Setup
+====================================
+
+This page explains how to quickly setup HttpFS with Pseudo authentication against a Hadoop cluster with Pseudo authentication.
+
+Install HttpFS
+--------------
+
+ ~ $ tar xzf httpfs-${project.version}.tar.gz
+
+Configure HttpFS
+----------------
+
+By default, HttpFS assumes that Hadoop configuration files (`core-site.xml & hdfs-site.xml`) are in the HttpFS configuration directory.
+
+If this is not the case, add to the `httpfs-site.xml` file the `httpfs.hadoop.config.dir` property set to the location of the Hadoop configuration directory.
+
+Configure Hadoop
+----------------
+
+Edit Hadoop `core-site.xml` and defined the Unix user that will run the HttpFS server as a proxyuser. For example:
+
+```xml
+ <property>
+ <name>hadoop.proxyuser.#HTTPFSUSER#.hosts</name>
+ <value>httpfs-host.foo.com</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.#HTTPFSUSER#.groups</name>
+ <value>*</value>
+ </property>
+```
+
+IMPORTANT: Replace `#HTTPFSUSER#` with the Unix user that will start the HttpFS server.
+
+Restart Hadoop
+--------------
+
+You need to restart Hadoop for the proxyuser configuration ot become active.
+
+Start/Stop HttpFS
+-----------------
+
+To start/stop HttpFS use HttpFS's sbin/httpfs.sh script. For example:
+
+ $ sbin/httpfs.sh start
+
+NOTE: Invoking the script without any parameters list all possible parameters (start, stop, run, etc.). The `httpfs.sh` script is a wrapper for Tomcat's `catalina.sh` script that sets the environment variables and Java System properties required to run HttpFS server.
+
+Test HttpFS is working
+----------------------
+
+ ~ $ curl -i "http://<HTTPFSHOSTNAME>:14000?user.name=babu&op=homedir"
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+
+ {"homeDir":"http:\/\/<HTTPFS_HOST>:14000\/user\/babu"}
+
+Embedded Tomcat Configuration
+-----------------------------
+
+To configure the embedded Tomcat go to the `tomcat/conf`.
+
+HttpFS preconfigures the HTTP and Admin ports in Tomcat's `server.xml` to 14000 and 14001.
+
+Tomcat logs are also preconfigured to go to HttpFS's `logs/` directory.
+
+The following environment variables (which can be set in HttpFS's `etc/hadoop/httpfs-env.sh` script) can be used to alter those values:
+
+* HTTPFS\_HTTP\_PORT
+
+* HTTPFS\_ADMIN\_PORT
+
+* HADOOP\_LOG\_DIR
+
+HttpFS Configuration
+--------------------
+
+HttpFS supports the following [configuration properties](./httpfs-default.html) in the HttpFS's `etc/hadoop/httpfs-site.xml` configuration file.
+
+HttpFS over HTTPS (SSL)
+-----------------------
+
+To configure HttpFS to work over SSL edit the [httpfs-env.sh](#httpfs-env.sh) script in the configuration directory setting the [HTTPFS\_SSL\_ENABLED](#HTTPFS_SSL_ENABLED) to [true](#true).
+
+In addition, the following 2 properties may be defined (shown with default values):
+
+* HTTPFS\_SSL\_KEYSTORE\_FILE=$HOME/.keystore
+
+* HTTPFS\_SSL\_KEYSTORE\_PASS=password
+
+In the HttpFS `tomcat/conf` directory, replace the `server.xml` file with the `ssl-server.xml` file.
+
+You need to create an SSL certificate for the HttpFS server. As the `httpfs` Unix user, using the Java `keytool` command to create the SSL certificate:
+
+ $ keytool -genkey -alias tomcat -keyalg RSA
+
+You will be asked a series of questions in an interactive prompt. It will create the keystore file, which will be named **.keystore** and located in the `httpfs` user home directory.
+
+The password you enter for "keystore password" must match the value of the `HTTPFS_SSL_KEYSTORE_PASS` environment variable set in the `httpfs-env.sh` script in the configuration directory.
+
+The answer to "What is your first and last name?" (i.e. "CN") must be the hostname of the machine where the HttpFS Server will be running.
+
+Start HttpFS. It should work over HTTPS.
+
+Using the Hadoop `FileSystem` API or the Hadoop FS shell, use the `swebhdfs://` scheme. Make sure the JVM is picking up the truststore containing the public key of the SSL certificate if using a self-signed certificate.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21e9e91c/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/UsingHttpTools.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/UsingHttpTools.md b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/UsingHttpTools.md
new file mode 100644
index 0000000..3045ad6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/UsingHttpTools.md
@@ -0,0 +1,62 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Hadoop HDFS over HTTP - Using HTTP Tools
+========================================
+
+Security
+--------
+
+Out of the box HttpFS supports both pseudo authentication and Kerberos HTTP SPNEGO authentication.
+
+### Pseudo Authentication
+
+With pseudo authentication the user name must be specified in the `user.name=<USERNAME>` query string parameter of a HttpFS URL. For example:
+
+ $ curl "http://<HTTFS_HOST>:14000/webhdfs/v1?op=homedir&user.name=babu"
+
+### Kerberos HTTP SPNEGO Authentication
+
+Kerberos HTTP SPNEGO authentication requires a tool or library supporting Kerberos HTTP SPNEGO protocol.
+
+IMPORTANT: If using `curl`, the `curl` version being used must support GSS (`curl -V` prints out 'GSS' if it supports it).
+
+For example:
+
+ $ kinit
+ Please enter the password for user@LOCALHOST:
+ $ curl --negotiate -u foo "http://<HTTPFS_HOST>:14000/webhdfs/v1?op=homedir"
+ Enter host password for user 'foo':
+
+NOTE: the `-u USER` option is required by the `--negotiate` but it is not used. Use any value as `USER` and when asked for the password press [ENTER] as the password value is ignored.
+
+### Remembering Who I Am (Establishing an Authenticated Session)
+
+As most authentication mechanisms, Hadoop HTTP authentication authenticates users once and issues a short-lived authentication token to be presented in subsequent requests. This authentication token is a signed HTTP Cookie.
+
+When using tools like `curl`, the authentication token must be stored on the first request doing authentication, and submitted in subsequent requests. To do this with curl the `-b` and `-c` options to save and send HTTP Cookies must be used.
+
+For example, the first request doing authentication should save the received HTTP Cookies.
+
+Using Pseudo Authentication:
+
+ $ curl -c ~/.httpfsauth "http://<HTTPFS_HOST>:14000/webhdfs/v1?op=homedir&user.name=foo"
+
+Using Kerberos HTTP SPNEGO authentication:
+
+ $ curl --negotiate -u foo -c ~/.httpfsauth "http://<HTTPFS_HOST>:14000/webhdfs/v1?op=homedir"
+
+Then, subsequent requests forward the previously received HTTP Cookie:
+
+ $ curl -b ~/.httpfsauth "http://<HTTPFS_HOST>:14000/webhdfs/v1?op=liststatus"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21e9e91c/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
new file mode 100644
index 0000000..ac8f4ca
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
@@ -0,0 +1,52 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Hadoop HDFS over HTTP - Documentation Sets
+==========================================
+
+HttpFS is a server that provides a REST HTTP gateway supporting all HDFS File System operations (read and write). And it is inteoperable with the **webhdfs** REST HTTP API.
+
+HttpFS can be used to transfer data between clusters running different versions of Hadoop (overcoming RPC versioning issues), for example using Hadoop DistCP.
+
+HttpFS can be used to access data in HDFS on a cluster behind of a firewall (the HttpFS server acts as a gateway and is the only system that is allowed to cross the firewall into the cluster).
+
+HttpFS can be used to access data in HDFS using HTTP utilities (such as curl and wget) and HTTP libraries Perl from other languages than Java.
+
+The **webhdfs** client FileSytem implementation can be used to access HttpFS using the Hadoop filesystem command (`hadoop fs`) line tool as well as from Java aplications using the Hadoop FileSystem Java API.
+
+HttpFS has built-in security supporting Hadoop pseudo authentication and HTTP SPNEGO Kerberos and other pluggable authentication mechanims. It also provides Hadoop proxy user support.
+
+How Does HttpFS Works?
+----------------------
+
+HttpFS is a separate service from Hadoop NameNode.
+
+HttpFS itself is Java web-application and it runs using a preconfigured Tomcat bundled with HttpFS binary distribution.
+
+HttpFS HTTP web-service API calls are HTTP REST calls that map to a HDFS file system operation. For example, using the `curl` Unix command:
+
+* `$ curl http://httpfs-host:14000/webhdfs/v1/user/foo/README.txt` returns the contents of the HDFS `/user/foo/README.txt` file.
+
+* `$ curl http://httpfs-host:14000/webhdfs/v1/user/foo?op=list` returns the contents of the HDFS `/user/foo` directory in JSON format.
+
+* `$ curl -X POST http://httpfs-host:14000/webhdfs/v1/user/foo/bar?op=mkdirs` creates the HDFS `/user/foo.bar` directory.
+
+User and Developer Documentation
+--------------------------------
+
+* [HttpFS Server Setup](./ServerSetup.html)
+
+* [Using HTTP Tools](./UsingHttpTools.html)
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21e9e91c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f8b0c37..6dc7a0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -141,6 +141,8 @@ Trunk (Unreleased)
HDFS-7668. Convert site documentation from apt to markdown (Masatake
Iwasaki via aw)
+ HDFS-7460. Rewrite httpfs to use new shell framework (John Smith via aw)
+
OPTIMIZATIONS
BUG FIXES