You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by bl...@apache.org on 2012/11/26 18:13:26 UTC
[1/5] git commit: SQOOP-719 Improve addtowar.sh script file (Jarek
Jarcec Cecho)
Updated Branches:
refs/heads/sqoop2 a348d5bb6 -> 0c94dbade
SQOOP-719 Improve addtowar.sh script file
(Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/0c94dbad
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/0c94dbad
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/0c94dbad
Branch: refs/heads/sqoop2
Commit: 0c94dbade89343d4a5d2f2bdda53e986caffdf97
Parents: 0e7451f
Author: Bilung Lee <bl...@apache.org>
Authored: Mon Nov 26 09:02:50 2012 -0800
Committer: Bilung Lee <bl...@apache.org>
Committed: Mon Nov 26 09:02:50 2012 -0800
----------------------------------------------------------------------
dist/src/main/bin/addtowar.sh | 89 +++++++++++++++++++++++++++++++-----
1 files changed, 77 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/0c94dbad/dist/src/main/bin/addtowar.sh
----------------------------------------------------------------------
diff --git a/dist/src/main/bin/addtowar.sh b/dist/src/main/bin/addtowar.sh
index fb65d35..77750e2 100644
--- a/dist/src/main/bin/addtowar.sh
+++ b/dist/src/main/bin/addtowar.sh
@@ -71,7 +71,7 @@ function checkFileDoesNotExist() {
# Finds a file under a directory any depth, file returns in variable RET
function findFile() {
- RET=`find -H ${1} -name ${2} | grep -e "[0-9.a${hadoopJarsSuffix}].jar"`
+ RET=`find -H ${1//:/$' '} -name ${2} | grep -e "[0-9.a${hadoopJarsSuffix}].jar"`
RET=`echo ${RET} | sed "s/ .*//"`
if [ "${RET}" = "" ]; then
echo
@@ -92,20 +92,63 @@ function checkOption() {
fi
}
+# Try to guess installed Hadoop version
+function guessHadoopVersion() {
+ # Runn "hadoop version" command
+ guessedVersion=`hadoop version | grep "Hadoop"`
+ if [[ $? -ne 0 ]]; then
+ echo
+ echo "Hadoop binary is not in path, can't guess Hadoop version"
+ echo
+ printUsage
+ exit -1
+ fi
+
+ # Get the version number from line in format "Hadoop 2.0.2-alpha"
+ hadoopVersion=`echo $guessedVersion | sed -re "s/^Hadoop[ ]*//"`
+}
+
+# Try to guess common hadoop paths
+function guessHadoopHome() {
+ hadoopPossiblePaths="/usr/lib/hadoop /usr/lib/hadoop-mapreduce/ /usr/lib/hadoop-yarn/ /usr/lib/hadoop-hdfs"
+
+ # Check existence of each possible path and build up final structure
+ for path in $hadoopPossiblePaths; do
+ if [ -e ${path} ]; then
+ if [ -z "$hadoopHome"] ; then
+ hadoopHome="$path"
+ else
+ hadoopHome="$hadoopHome:$path"
+ fi
+ fi
+ done
+
+ if [ -z "$hadoopHome"] ; then
+ echo
+ echo "Non of expected directories with Hadoop exists"
+ echo
+ printUsage
+ exit -1
+ fi
+}
+
# Get the list of hadoop jars that will be injected based on the hadoop version
-# TODO(jarcec): Add configuration specific to Hadoop 1.x
function getHadoopJars() {
version=$1
- if [[ "${version}" =~ 1.* ]]; then
- #List is separated by ":"
+ # List is separated by ":"
+ if [[ "${version}" =~ cdh4mr1 ]]; then
+ suffix="-[0-9.]*"
+ hadoopJars="hadoop-mapreduce-client-core${suffix}.jar:hadoop-mapreduce-client-common${suffix}.jar:hadoop-mapreduce-client-jobclient${suffix}.jar:hadoop-mapreduce-client-app${suffix}.jar:hadoop-yarn-common${suffix}.jar:hadoop-yarn-api${suffix}.jar:hadoop-hdfs${suffix}.jar:hadoop-common${suffix}.jar:hadoop-auth${suffix}.jar:guava*.jar:protobuf-*.jar:jackson-core-asl-*.jar:jackson-mapper-asl-*.jar:commons-configuration-*.jar:commons-cli-*.jar:commons-logging-*.jar:slf4j-api-*.jar:slf4j-log4j*.jar:avro-*.jar:hadoop-2.0.0-mr1-*-core.jar"
+ elif [[ "${version}" =~ cdh3 ]]; then
+ hadoopJars="hadoop-core*.jar:jackson-core-asl-*.jar:jackson-mapper-asl-*.jar:commons-logging-*.jar:slf4j-api-*.jar:slf4j-log4j*.jar:guava*.jar"
+ elif [[ "${version}" =~ ^1.* ]]; then
hadoopJars="hadoop-core*.jar:jackson-core-asl-*.jar:jackson-mapper-asl-*.jar:commons-configuration-*.jar:commons-logging-*.jar:slf4j-api-*.jar:slf4j-log4j*.jar"
- elif [[ "${version}" =~ 2.* ]]; then
+ elif [[ "${version}" =~ ^2.* ]]; then
suffix="-[0-9.]*"
- # List is separated by ":"
hadoopJars="hadoop-mapreduce-client-core${suffix}.jar:hadoop-mapreduce-client-common${suffix}.jar:hadoop-mapreduce-client-jobclient${suffix}.jar:hadoop-mapreduce-client-app${suffix}.jar:hadoop-yarn-common${suffix}.jar:hadoop-yarn-api${suffix}.jar:hadoop-hdfs${suffix}.jar:hadoop-common${suffix}.jar:hadoop-auth${suffix}.jar:guava*.jar:protobuf-*.jar:jackson-core-asl-*.jar:jackson-mapper-asl-*.jar:commons-configuration-*.jar:commons-cli-*.jar:commons-logging-*.jar:slf4j-api-*.jar:slf4j-log4j*.jar:avro-*.jar"
else
echo
- echo "Exiting: Unsupported Hadoop version '${hadoopVer}', supported versions: 1.x, 2.x"
+ echo "Exiting: Unsupported Hadoop version '${version}', supported versions: 1.x, 2.x, cdh3, cdh4mr1"
echo
cleanUp
exit -1;
@@ -114,9 +157,11 @@ function getHadoopJars() {
function printUsage() {
echo " Usage : addtowar.sh <OPTIONS>"
- echo " Options: -hadoop HADOOP_VERSION HADOOP_PATH"
- echo " [-jars JARS_PATH] (multiple JAR path separated by ':')"
- echo " [-war SQOOP_WAR]"
+ echo " Options: -hadoop-auto Try to guess hadoop version and path"
+ echo " -hadoop-version HADOOP_VERSION Specify used version"
+ echo " -hadoop-path HADOOP_PATHS Where to find hadoop jars (multiple paths with Hadoop jars separated by ':')"
+ echo " -jars JARS_PATH Special jars that should be added (multiple JAR paths separated by ':')"
+ echo " -war SQOOP_WAR Target Sqoop war file where all jars should be ingested"
echo
}
@@ -140,7 +185,7 @@ warPath="`dirname $0`/../server/webapps/sqoop.war"
# Parse command line arguments
while [ $# -gt 0 ]
do
- if [ "$1" = "-hadoop" ]; then
+ if [ "$1" = "-hadoop-version" ]; then
shift
if [ $# -eq 0 ]; then
echo
@@ -150,6 +195,8 @@ do
exit -1
fi
hadoopVersion=$1
+ addHadoop=true
+ elif [ "$1" = "-hadoop-path" ]; then
shift
if [ $# -eq 0 ]; then
echo
@@ -160,6 +207,10 @@ do
fi
hadoopHome=$1
addHadoop=true
+ elif [ "$1" = "-hadoop-auto" ]; then
+ guessHadoopVersion
+ guessHadoopHome
+ addHadoop=true
elif [ "$1" = "-jars" ]; then
shift
if [ $# -eq 0 ]; then
@@ -201,7 +252,10 @@ checkOption "-war" ${warPath}
checkFileExists ${warPath}
if [ "${addHadoop}" = "true" ]; then
- checkFileExists ${hadoopHome}
+ for hadoopPath in ${hadoopHome//:/$'\n'}
+ do
+ checkFileExists ${hadoopPath}
+ done
getHadoopJars ${hadoopVersion}
fi
@@ -218,6 +272,11 @@ checkExec "Unzipping Sqoop WAR"
components=""
+# Let's print useful information
+echo "Hadoop version: $hadoopVersion"
+echo "Hadoop path: $hadoopHome"
+echo "Extra jars: $jarsPath"
+
# Adding hadoop binaries to WAR file
if [ "${addHadoop}" = "true" ]; then
components="Hadoop JARs";
@@ -231,6 +290,7 @@ if [ "${addHadoop}" = "true" ]; then
exit -1
fi
## adding hadoop
+ echo ""
echo "Injecting following Hadoop JARs"
echo
for jar in ${hadoopJars//:/$'\n'}
@@ -250,6 +310,8 @@ if [ "${addJars}" = "true" ]; then
fi
components="${components}JARs"
+ echo ""
+ echo "Injecting following additional JARs"
for jarPath in ${jarsPath//:/$'\n'}
do
found=`ls ${tmpWarDir}/WEB-INF/lib/${jarPath} 2> /dev/null | wc -l`
@@ -261,6 +323,9 @@ if [ "${addJars}" = "true" ]; then
cleanUp
exit -1
fi
+ echo ""
+ echo "$jarPath"
+ echo ""
cp ${jarPath} ${tmpWarDir}/WEB-INF/lib/
checkExec "Copying jar ${jarPath} to staging"
done