You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ma...@apache.org on 2011/09/22 22:16:59 UTC
svn commit: r1174368 - in /hadoop/common/branches/branch-0.20-security-205:
CHANGES.txt src/packages/hadoop-validate-setup.sh
Author: mattf
Date: Thu Sep 22 20:16:59 2011
New Revision: 1174368
URL: http://svn.apache.org/viewvc?rev=1174368&view=rev
Log:
HADOOP-7655. provide a small validation script that smoke tests the installed cluster. Contributed by Arpit Gupta.
Added:
hadoop/common/branches/branch-0.20-security-205/src/packages/hadoop-validate-setup.sh
Modified:
hadoop/common/branches/branch-0.20-security-205/CHANGES.txt
Modified: hadoop/common/branches/branch-0.20-security-205/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-205/CHANGES.txt?rev=1174368&r1=1174367&r2=1174368&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-205/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security-205/CHANGES.txt Thu Sep 22 20:16:59 2011
@@ -177,6 +177,9 @@ Release 0.20.205.0 - 2011.09.12
IMPROVEMENTS
+ HADOOP-7655. provide a small validation script that smoke tests the installed
+ cluster. (Arpit Gupta via mattf)
+
MAPREDUCE-2187. Reporter sends progress during sort/merge. (Anupam Seth via
acmurthy)
Added: hadoop/common/branches/branch-0.20-security-205/src/packages/hadoop-validate-setup.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-205/src/packages/hadoop-validate-setup.sh?rev=1174368&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-205/src/packages/hadoop-validate-setup.sh (added)
+++ hadoop/common/branches/branch-0.20-security-205/src/packages/hadoop-validate-setup.sh Thu Sep 22 20:16:59 2011
@@ -0,0 +1,181 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+
+###############################################################################
+# Run the following jobs to validate a hadoop cluster
+## teragen
+## terasort
+## teravalidate
+# If they all pass 0 will be returned and 1 otherwise
+# The test will work for both secure and unsecure deploys. If the kerberos-realm
+# is passed we will assume that the deploy is secure and proceed with a kinit before
+# running the validation jobs.
+################################################################################
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/../libexec/hadoop-config.sh
+
+usage() {
+ echo "
+usage: $0 <parameters>
+
+ Optional parameters:
+ -h Display this message
+ --user=hdfs
+ --user_keytab=/home/hdfs/hdfs.keytab
+ --kerberos-realm=KERBEROS.EXAMPLE.COM Set Kerberos realm
+ "
+ exit 1
+}
+
+OPTS=$(getopt \
+ -n $0 \
+ -o '' \
+ -l 'user:' \
+ -l 'user-keytab:' \
+ -l 'kerberos-realm:' \
+ -o 'h' \
+ -- "$@")
+
+if [ $? != 0 ] ; then
+ usage
+fi
+
+eval set -- "${OPTS}"
+while true ; do
+ case "$1" in
+ --user)
+ TEST_USER=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --user-keytab)
+ USER_KEYTAB_FILE=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --kerberos-realm)
+ KERBEROS_REALM=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --)
+ shift ; break
+ ;;
+ *)
+ echo "Unknown option: $1"
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+#set the hadoop command and the path to the hadoop examples jar
+HADOOP_CMD="${HADOOP_PREFIX}/bin/hadoop --config $HADOOP_CONF_DIR"
+
+#find the hadoop examples jar
+HADOOP_EXAMPLES_JAR=''
+
+#find under HADOOP_PREFIX (tar ball install)
+HADOOP_EXAMPLES_JAR=`find ${HADOOP_PREFIX} -name 'hadoop-examples-*.jar' | head -n1`
+
+#if its not found look under /usr/share/hadoop (rpm/deb installs)
+if [ "$HADOOP_EXAMPLES_JAR" == '' ]
+then
+ HADOOP_EXAMPLES_JAR=`find /usr/share/hadoop -name 'hadoop-examples-*.jar' | head -n1`
+fi
+
+#if it is still empty then dont run the tests
+if [ "$HADOOP_EXAMPLES_JAR" == '' ]
+then
+ echo "Did not find hadoop-examples-*.jar under '${HADOOP_PREFIX} or '/usr/share/hadoop'"
+ exit 1
+fi
+
+# do a kinit if secure
+if [ "${KERBEROS_REALM}" != "" ]; then
+ # Determine kerberos location base on Linux distro.
+ if [ -e /etc/lsb-release ]; then
+ KERBEROS_BIN=/usr/bin
+ else
+ KERBEROS_BIN=/usr/kerberos/bin
+ fi
+ kinit_cmd="su -c '${KERBEROS_BIN}/kinit -kt ${USER_KEYTAB_FILE} ${TEST_USER}' ${TEST_USER}"
+ echo $kinit_cmd
+ eval $kinit_cmd
+ if [ $? -ne 0 ]
+ then
+ echo "kinit command did not run successfully."
+ exit 1
+ fi
+fi
+
+
+#dir where to store the data on hdfs. The data is relative of the users home dir on hdfs.
+PARENT_DIR="validate_deploy_`date +%s`"
+TERA_GEN_OUTPUT_DIR="${PARENT_DIR}/tera_gen_data"
+TERA_SORT_OUTPUT_DIR="${PARENT_DIR}/tera_sort_data"
+TERA_VALIDATE_OUTPUT_DIR="${PARENT_DIR}/tera_validate_data"
+#tera gen cmd
+TERA_GEN_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR teragen 10000 $TERA_GEN_OUTPUT_DIR' $TEST_USER"
+
+#tera sort cmd
+TERA_SORT_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR terasort $TERA_GEN_OUTPUT_DIR $TERA_SORT_OUTPUT_DIR' $TEST_USER"
+
+#tera validate cmd
+TERA_VALIDATE_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR teravalidate $TERA_SORT_OUTPUT_DIR $TERA_VALIDATE_OUTPUT_DIR' $TEST_USER"
+
+echo "Starting teragen...."
+
+#run tera gen
+echo $TERA_GEN_CMD
+eval $TERA_GEN_CMD
+if [ $? -ne 0 ]; then
+ echo "tera gen failed."
+ exit 1
+fi
+
+echo "Teragen passed starting terasort...."
+
+
+#run tera sort
+echo $TERA_SORT_CMD
+eval $TERA_SORT_CMD
+if [ $? -ne 0 ]; then
+ echo "tera sort failed."
+ exit 1
+fi
+
+echo "Terasort passed starting teravalidate...."
+
+#run tera validate
+echo $TERA_VALIDATE_CMD
+eval $TERA_VALIDATE_CMD
+if [ $? -ne 0 ]; then
+ echo "tera validate failed."
+ exit 1
+fi
+
+echo "teragen, terasort, teravalidate passed."
+echo "Cleaning the data created by tests: $PARENT_DIR"
+
+CLEANUP_CMD="su -c '$HADOOP_CMD dfs -rmr -skipTrash $PARENT_DIR' $TEST_USER"
+echo $CLEANUP_CMD
+eval $CLEANUP_CMD
+
+exit 0