You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2017/02/27 20:31:09 UTC
[43/50] [abbrv] bigtop git commit: ODPI-200. Added hive support for
run_itest.sh and made some enhancements.
ODPI-200. Added hive support for run_itest.sh and made some enhancements.
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/eab23cfe
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/eab23cfe
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/eab23cfe
Branch: refs/heads/BIGTOP-1406
Commit: eab23cfe3df37d24b04a245173190de344b02d8b
Parents: 94efd55
Author: Raj Desai <rd...@us.ibm.com>
Authored: Thu Nov 10 15:09:33 2016 -0800
Committer: Roman Shaposhnik <rv...@apache.org>
Committed: Fri Feb 24 12:03:52 2017 -0800
----------------------------------------------------------------------
bigtop-tests/build.gradle | 8 +-
bigtop-tests/smoke-tests/run_itest.sh | 182 ------------------
run_itest.sh | 293 +++++++++++++++++++++++++++++
3 files changed, 296 insertions(+), 187 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab23cfe/bigtop-tests/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-tests/build.gradle b/bigtop-tests/build.gradle
index f49546e..1940998 100644
--- a/bigtop-tests/build.gradle
+++ b/bigtop-tests/build.gradle
@@ -32,12 +32,13 @@ distributions {
include 'gradlew'
include '*.gradle'
include 'gradle/**'
+ include 'run_itest.sh'
}
from ("$rootDir/buildSrc") {
into 'buildSrc'
}
from ("$rootDir/bigtop-packages/src/common/bigtop-utils") {
- into 'bin'
+ into 'bigtop-packages/src/common/bigtop-utils'
include '*'
}
from ('smoke-tests') {
@@ -47,11 +48,9 @@ distributions {
include 'hdfs/**/*'
include 'mapreduce/**/*'
include 'yarn/**/*'
+ include 'hive/**/*'
include 'logger-test-config/**'
}
- from ('smoke-tests') { // Put the driver script to the top-level
- include 'run_itest.sh'
- }
from ('spec-tests') {
into 'bigtop-tests/spec-tests'
include '**/*'
@@ -67,4 +66,3 @@ distributions {
distZip.dependsOn clean
distTar.dependsOn clean
-
http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab23cfe/bigtop-tests/smoke-tests/run_itest.sh
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/run_itest.sh b/bigtop-tests/smoke-tests/run_itest.sh
deleted file mode 100755
index d93bd5a..0000000
--- a/bigtop-tests/smoke-tests/run_itest.sh
+++ /dev/null
@@ -1,182 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## Borrowed heavily from
-## https://github.com/jctanner/odp-scripts/blob/master/run_itest.sh
-## Kudos to https://github.com/jctanner
-
-# https://github.com/apache/bigtop/tree/master/bigtop-test-framework
-
-# "ITEST" is an integration testing framework written for and by the
-# apache bigtop project. It consists of typical java tools/libraries
-# such as junit, gradle and maven.
-
-# This script is a helper to run itest on any hadoop system without
-# requiring intimate knowledge of bigtop. If running for the first
-# time, simply execute ./run_itest.sh without any arguments. If you
-# want more information, use these additional parameters:
-#
-# --info - turns on the log4j output
-# --debug - turns up the log4j output to maximum
-# --traceback - shows tracebacks from tests
-
-set_java_home() {
-
- #####################################################################
- # Use bigtop's bigtop-detect-javahome if JAVA_HOME is not already set
- #####################################################################
-
- if [ -z "$JAVA_HOME" ]; then
- source bin/bigtop-detect-javahome
- fi
-}
-
-set_hadoop_vars() {
-
- #####################################################################
- # Set the HADOOP_MAPRED_HOME and HADOOP_CONF vars
- #####################################################################
-
- # ITEST wants the MR dir with the examples jar ...
- # java.lang.AssertionError: Can't find hadoop-examples.jar file
-
- if ( [ -z "$HADOOP_HOME" ] && [ -d /usr/lib/hadoop ] ); then
- export HADOOP_HOME=/usr/lib/hadoop
- fi
- if ( [ -z "$HADOOP_CONF_DIR" ] && [ -d /etc/hadoop/conf ] ); then
- export HADOOP_CONF_DIR=/etc/hadoop/conf
- fi
- if ( [ -z "$HADOOP_MAPRED_HOME" ] && [ -d /usr/lib/hadoop-mapreduce-client ] ); then
- export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce-client
- elif ( [ -z "$HADOOP_MAPRED_HOME" ] && [ -d /usr/lib/hadoop-mapreduce ] ); then
- export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce
- fi
-}
-
-
-print_cluster_info() {
-
- # ODPI-87
-
- echo "######################################################"
- echo "# CLUSTER INFORMATION #"
- echo "######################################################"
-
- which facter >/dev/null 2>&1
- RC=$?
- if [[ $RC == 0 ]]; then
- echo "# OS: $(facter lsbdistdescription)"
- echo "# ARCH: $(facter architecture)"
- echo "# KERNEL: $(facter kernelrelease)"
- echo "# MEMORY: $(facter memorysize)"
- else
- echo "# OS: $(cat /etc/issue | tr '\n' ' ')"
- echo "# ARCH: $(uname -i)"
- echo "# KERNEL: $(uname -a)"
- echo "# MEMORY: $(head -n1 /proc/meminfo)"
- fi
-
- YARN_NODES=$(yarn node -list 2>/dev/null | egrep ^Total | sed 's/Total Nodes:/TOTAL YARN NODES: /g')
- echo "# $YARN_NODES"
-
- HADOOP_VERSION=$(hadoop version 2>/dev/null | head -n1)
- echo "# HADOOP_VERSION: $HADOOP_VERSION"
- echo "# HADOOP_CONF_DIR: $HADOOP_CONF_DIR"
- echo "# HADOOP_MAPRED_HOME: $HADOOP_MAPRED_HOME"
-
- echo "# BASH_VERSION: $BASH_VERSION"
- echo "# SH_VERSION: $(/bin/sh -c 'echo $BASH_VERSION')"
-
- echo "# JAVA_HOME: $JAVA_HOME"
- JAVA_VERSION=$(java -version 2>&1 | head -n 1 | awk -F '"' '{print $2}')
- echo "# JAVA_VERSION: $JAVA_VERSION"
-}
-
-print_tests() {
- echo "######################################################"
- echo "# RESULTS #"
- echo "######################################################"
-
- pushd `pwd`
- for TEST in $(echo $ITESTS | tr ',' '\n'); do
- TESTDIR=bigtop-tests/smoke-tests/$TEST/build
-
- if [ -d $TESTDIR ]; then
- cd $TESTDIR
-
- for FILE in $(find -L reports/tests/classes -type f -name "*.html"); do
- echo "## $TESTDIR/$FILE"
- if [ $(which links) ]; then
- links $FILE -dump
- else
- echo "PLEASE INSTALL LINKS: sudo yum -y install links"
- fi
- echo ""
- done
- fi
- done
-
- popd
- for TEST in $SPEC_TESTS; do
- TESTDIR=bigtop-tests/spec-tests/$TEST/build
-
- if [ -d $TESTDIR ]; then
- cd $TESTDIR
-
- for FILE in $(find -L reports/tests/classes -type f -name "*.html"); do
- echo "## $TESTDIR/$FILE"
- if [ $(which links) ]; then
- links $FILE -dump
- else
- echo "PLEASE INSTALL LINKS: sudo yum -y install links"
- fi
- echo ""
- done
- fi
- done
-}
-
-# SET JAVA_HOME
-set_java_home
-
-# SET HADOOP SERVICE HOMES
-set_hadoop_vars
-
-# ODPI-87
-print_cluster_info
-
-echo "######################################################"
-echo "# STARTING ITEST #"
-echo "######################################################"
-echo "# Use --debug/--info/--stacktrace for more details"
-
-# SET THE DEFAULT TESTS
-if [ -z "$ITESTS" ]; then
- export ITESTS="hcfs,hdfs,yarn,mapreduce"
-fi
-SPEC_TESTS="runtime"
-for s in `echo $ITESTS | sed -e 's#,# #g'`; do
- ALL_SMOKE_TASKS="$ALL_SMOKE_TASKS bigtop-tests:smoke-tests:$s:test"
-done
-for s in $SPEC_TESTS; do
- ALL_SPEC_TASKS="$ALL_SPEC_TASKS bigtop-tests:spec-tests:$s:test"
-done
-
-# CALL THE GRADLE WRAPPER TO RUN THE FRAMEWORK
-./gradlew -q --continue clean -Psmoke.tests $ALL_SMOKE_TASKS -Pspec.tests $ALL_SPEC_TASKS $@
-
-# SHOW RESULTS (HTML)
-print_tests
http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab23cfe/run_itest.sh
----------------------------------------------------------------------
diff --git a/run_itest.sh b/run_itest.sh
new file mode 100644
index 0000000..e6184e2
--- /dev/null
+++ b/run_itest.sh
@@ -0,0 +1,293 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## Borrowed heavily from
+## https://github.com/jctanner/odp-scripts/blob/master/run_itest.sh
+## Kudos to https://github.com/jctanner
+
+# https://github.com/apache/bigtop/tree/master/bigtop-test-framework
+
+# "ITEST" is an integration testing framework written for and by the
+# apache bigtop project. It consists of typical java tools/libraries
+# such as junit, gradle and maven.
+
+# This script is a helper to run itest on any hadoop system without
+# requiring intimate knowledge of bigtop. If running for the first
+# time, simply execute ./run_itest.sh without any arguments. If you
+# want more information, use these additional parameters:
+#
+# --info - turns on the log4j output
+# --debug - turns up the log4j output to maximum
+# --traceback - shows tracebacks from tests
+
+usage ()
+{
+ echo "Usage : $0 [-options]"
+ echo
+ echo " -j, --hivejdbc hive jdbc url - default: e.g. jdbc:hive2://localhost:10000"
+ echo " -m, --hivemeta hive metastore url - default: thrift://localhost:9083"
+ echo " -l, --hivelocation location of hdfs for hive to write to - default: /user/<current user>"
+ echo " -u, --hiveuser hive user - default: current user"
+ echo " -p, --hivepassword hive user password - default: current user"
+ echo " -t, --hivethrift optional: true/false to test thrift, defaults to true"
+ echo " -c, --hivecatalog optional: true/false to test HCatalog, default to true"
+ echo " -C, --hiveconf hive conf dir - default: /etc/hive/conf"
+ echo " -F, --hadoopconf hadoop user - default: /etc/hadoop/conf"
+ echo " -i, --info optional: info/debug"
+ echo " -h, --help display this help and exit"
+ echo
+ exit
+}
+
+while [ "$1" != "" ]; do
+case $1 in
+ -j | --hivejdbc ) shift
+ HIVE_JDBC_URL=$1
+ ;;
+ -m | --hivemeta ) shift
+ HIVE_METASTORE_URL=$1
+ ;;
+ -l | --hivelocation ) shift
+ HIVE_HDFS_LOCATION=$1
+ ;;
+ -u | --hiveuser ) shift
+ HIVE_USER=$1
+ ;;
+ -p | --hivepassword ) shift
+ HIVE_PASSWORD=$1
+ ;;
+ -t | --hivethrift ) shift
+ TEST_THRIFT=$1
+ ;;
+ -c | --hivecatalog ) shift
+ TEST_HCATALOG=$1
+ ;;
+ -C | --hiveconf ) shift
+ HIVE_CONF_DIR=$1
+ ;;
+ -F | --hadoopconf ) shift
+ HADOOP_CONF_DIR=$1
+ ;;
+ -i | --info ) shift
+ LOGGING=$1
+ ;;
+ -h | --help )
+ usage # Call your function
+ exit 0
+ ;;
+ esac
+ shift
+done
+
+if [ ! -f /etc/hive/conf/hive-site.xml ]; then
+ echo "Could not find hive site configuration file, please specify hive params!"
+ usage
+fi
+
+if [ -z "$HIVE_JDBC_URL" ]; then
+ HIVE_PORT=`sed -n '/hive.server2.thrift.port/{n;p}' /etc/hive/conf/hive-site.xml | sed -n 's:.*<value>\(.*\)</value>.*:\1:p'`
+ netstat -nltp | grep $HIVE_PORT > /dev/null 2>&1
+ if [ $? -eq 0 ]; then
+ HIVE_JDBC_URL=jdbc:hive2://localhost:$HIVE_PORT
+ else
+ echo "Could not find hive server 2 service, please specify --hivejdbc argument."
+ usage
+ fi
+fi
+if [ -z "$HIVE_METASTORE_URL" ]; then
+ HIVE_METASTORE_URL=`sed -n '/hive.metastore.uris/{n;p}' /etc/hive/conf/hive-site.xml | sed -n 's:.*<value>\(.*\)</value>.*:\1:p'`
+fi
+if [ -z "$HIVE_HDFS_LOCATION" ]; then
+ HIVE_HDFS_LOCATION=/tmp/`id -u -n`
+fi
+if [ -z "$HIVE_USER" ]; then
+ HIVE_USER=`id -u -n`
+fi
+if [ -z "$HIVE_PASSWORD" ]; then
+ HIVE_PASSWORD=`id -u -n`
+fi
+if [ -z "$HIVE_CONF_DIR" ]; then
+ export HIVE_CONF_DIR=/etc/hive/conf
+fi
+if [ -z "$HADOOP_CONF_DIR" ]; then
+ export HADOOP_CONF_DIR=/etc/hadoop/conf
+fi
+if [ -z "$TEST_THRIFT" ]; then
+ TEST_THRIFT=true
+fi
+if [ -z "$TEST_HCATALOG" ]; then
+ TEST_HCATALOG=true
+fi
+if [ "$LOGGING" == "info" ]; then
+ LOGGING="--info --stacktrace"
+elif [ "$LOGGING" == "debug" ]; then
+ LOGGING="--debug --stacktrace"
+else
+ LOGGING=""
+fi
+
+export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+set_java_home() {
+
+ #####################################################################
+ # Use bigtop's bigtop-detect-javahome if JAVA_HOME is not already set
+ #####################################################################
+
+ if [ -z "$JAVA_HOME" ]; then
+ source $DIR/bigtop-packages/src/common/bigtop-utils/bigtop-detect-javahome
+ fi
+}
+
+set_hadoop_vars() {
+
+ #####################################################################
+ # Set the HADOOP_MAPRED_HOME and HADOOP_CONF vars
+ #####################################################################
+
+ # ITEST wants the MR dir with the examples jar ...
+ # java.lang.AssertionError: Can't find hadoop-examples.jar file
+
+ if ( [ -z "$HADOOP_HOME" ] && [ -d /usr/lib/hadoop ] ); then
+ export HADOOP_HOME=/usr/lib/hadoop
+ fi
+ if ( [ -z "$HADOOP_CONF_DIR" ] && [ -d /etc/hadoop/conf ] ); then
+ export HADOOP_CONF_DIR=/etc/hadoop/conf
+ fi
+ if ( [ -z "$HADOOP_MAPRED_HOME" ] && [ -d /usr/lib/hadoop-mapreduce-client ] ); then
+ export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce-client
+ elif ( [ -z "$HADOOP_MAPRED_HOME" ] && [ -d /usr/lib/hadoop-mapreduce ] ); then
+ export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce
+ fi
+ if ( [ -z "$HIVE_HOME" ] && [ -d /usr/lib/hive ] ); then
+ export HIVE_HOME=/usr/lib/hive
+ fi
+}
+
+
+print_cluster_info() {
+
+ # ODPI-87
+
+ echo "######################################################"
+ echo "# CLUSTER INFORMATION #"
+ echo "######################################################"
+
+ which facter >/dev/null 2>&1
+ RC=$?
+ if [[ $RC == 0 ]]; then
+ echo "# OS: $(facter lsbdistdescription)"
+ echo "# ARCH: $(facter architecture)"
+ echo "# KERNEL: $(facter kernelrelease)"
+ echo "# MEMORY: $(facter memorysize)"
+ else
+ echo "# OS: $(cat /etc/issue | tr '\n' ' ')"
+ echo "# ARCH: $(uname -i)"
+ echo "# KERNEL: $(uname -a)"
+ echo "# MEMORY: $(head -n1 /proc/meminfo)"
+ fi
+
+ YARN_NODES=$(yarn node -list 2>/dev/null | egrep ^Total | sed 's/Total Nodes:/TOTAL YARN NODES: /g')
+ echo "# $YARN_NODES"
+
+ HADOOP_VERSION=$(hadoop version 2>/dev/null | head -n1)
+ echo "# HADOOP_VERSION: $HADOOP_VERSION"
+ echo "# HADOOP_CONF_DIR: $HADOOP_CONF_DIR"
+ echo "# HADOOP_MAPRED_HOME: $HADOOP_MAPRED_HOME"
+
+ echo "# BASH_VERSION: $BASH_VERSION"
+ echo "# SH_VERSION: $(/bin/sh -c 'echo $BASH_VERSION')"
+
+ echo "# JAVA_HOME: $JAVA_HOME"
+ JAVA_VERSION=$(java -version 2>&1 | head -n 1 | awk -F '"' '{print $2}')
+ echo "# JAVA_VERSION: $JAVA_VERSION"
+}
+
+print_tests() {
+ echo "######################################################"
+ echo "# RESULTS #"
+ echo "######################################################"
+
+ pushd `pwd`
+ for TEST in $(echo $ITESTS | tr ',' '\n'); do
+ TESTDIR=$DIR/bigtop-tests/smoke-tests/$TEST/build
+
+ if [ -d $TESTDIR ]; then
+ cd $TESTDIR
+
+ for FILE in $(find -L reports/tests/classes -type f -name "*.html"); do
+ echo "## $TESTDIR/$FILE"
+ if [ $(which links) ]; then
+ links $FILE -dump
+ else
+ echo "PLEASE INSTALL LINKS: sudo yum -y install links"
+ fi
+ echo ""
+ done
+ fi
+ done
+
+ popd
+ for TEST in $SPEC_TESTS; do
+ TESTDIR=$DIR/bigtop-tests/spec-tests/$TEST/build
+
+ if [ -d $TESTDIR ]; then
+ cd $TESTDIR
+
+ for FILE in $(find -L reports/tests/classes -type f -name "*.html"); do
+ echo "## $TESTDIR/$FILE"
+ if [ $(which links) ]; then
+ links $FILE -dump
+ else
+ echo "PLEASE INSTALL LINKS: sudo yum -y install links"
+ fi
+ echo ""
+ done
+ fi
+ done
+}
+
+# SET JAVA_HOME
+set_java_home
+
+# SET HADOOP SERVICE HOMES
+set_hadoop_vars
+
+# ODPI-87
+print_cluster_info
+
+echo "######################################################"
+echo "# STARTING ITEST #"
+echo "######################################################"
+echo "# Use --debug/--info for more details"
+
+# SET THE DEFAULT TESTS
+if [ -z "$ITESTS" ]; then
+ export ITESTS="hcfs,hdfs,yarn,mapreduce,hive"
+fi
+SPEC_TESTS="runtime"
+for s in `echo $ITESTS | sed -e 's#,# #g'`; do
+ ALL_SMOKE_TASKS="$ALL_SMOKE_TASKS bigtop-tests:smoke-tests:$s:test"
+done
+for s in $SPEC_TESTS; do
+ ALL_SPEC_TASKS="$ALL_SPEC_TASKS bigtop-tests:spec-tests:$s:test"
+done
+
+# CALL THE GRADLE WRAPPER TO RUN THE FRAMEWORK
+$DIR/gradlew -q --continue clean -Psmoke.tests $ALL_SMOKE_TASKS -Pspec.tests $ALL_SPEC_TASKS -Dodpi.test.hive.jdbc.url=$HIVE_JDBC_URL -Dodpi.test.hive.metastore.url=$HIVE_METASTORE_URL -Dodpi.test.hive.location=$HIVE_HDFS_LOCATION -Dodpi.test.hive.jdbc.user=$HIVE_USER -Dodpi.test.hive.jdbc.password=$HIVE_PASSWORD -Dodpi.test.hive.conf.dir=$HIVE_CONF_DIR -Dodpi.test.hadoop.conf.dir=$HADOOP_CONF_DIR -Dodpi.test.hive.thrift.test=$TEST_THRIFT -Dodpi.test.hive.hcatalog.test=$TEST_HCATALOG $LOGGING
+
+# SHOW RESULTS (HTML)
+print_tests