You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mrql.apache.org by fe...@apache.org on 2013/10/29 15:18:39 UTC
git commit: MRQL-21: Modify shell scripts to work on Mac OS
Updated Branches:
refs/heads/master bc396b564 -> 287533984
MRQL-21: Modify shell scripts to work on Mac OS
Project: http://git-wip-us.apache.org/repos/asf/incubator-mrql/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mrql/commit/28753398
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mrql/tree/28753398
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mrql/diff/28753398
Branch: refs/heads/master
Commit: 28753398445ed9590c99a51084873e9f69a7c04b
Parents: bc396b5
Author: fegaras <fe...@cse.uta.edu>
Authored: Tue Oct 29 09:17:48 2013 -0500
Committer: fegaras <fe...@cse.uta.edu>
Committed: Tue Oct 29 09:17:48 2013 -0500
----------------------------------------------------------------------
bin/mrql | 17 ++++++++---------
bin/mrql.bsp | 20 ++++++++++----------
bin/mrql.spark | 15 ++++++++-------
conf/mrql-env.sh | 11 +++++++----
4 files changed, 33 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/28753398/bin/mrql
----------------------------------------------------------------------
diff --git a/bin/mrql b/bin/mrql
index 90f8453..7ea280f 100755
--- a/bin/mrql
+++ b/bin/mrql
@@ -24,23 +24,22 @@
#
#--------------------------------------------------------------------------------
-MRQL_HOME=$(dirname $(readlink -f $0))/..
+MRQL_HOME="$(cd `dirname $0`/..; pwd -P)"
-. $MRQL_HOME/conf/mrql-env.sh
-
-GEN_JAR=$(readlink -f $MRQL_HOME/lib/mrql-gen-*.jar)
-CORE_JAR=$(readlink -f $MRQL_HOME/lib/mrql-core-*.jar)
-MR_JAR=$(readlink -f $MRQL_HOME/lib/mrql-mr-*.jar)
+. "$MRQL_HOME/conf/mrql-env.sh"
+GEN_JAR=`ls "$MRQL_HOME"/lib/mrql-gen-*.jar`
+CORE_JAR=`ls "$MRQL_HOME"/lib/mrql-core-*.jar`
+MR_JAR=`ls "$MRQL_HOME"/lib/mrql-mr-*.jar`
export JAVA_HOME MAPRED_JOB_TRACKER FS_DEFAULT_NAME
if [ "$1" == "-local" ] || [ "$1" == "-dist" ]; then
- LIBJARS=$CUP_JAR,$JLINE_JAR,$GEN_JAR,$CORE_JAR,$MR_JAR
- export HADOOP_CLASSPATH=$CUP_JAR:$JLINE_JAR:$GEN_JAR:$CORE_JAR:$MR_JAR
+ LIBJARS="$CUP_JAR,$JLINE_JAR,$GEN_JAR,$CORE_JAR,$MR_JAR"
+ export HADOOP_CLASSPATH="$CUP_JAR:$JLINE_JAR:$GEN_JAR:$CORE_JAR:$MR_JAR"
$HADOOP_HOME/bin/hadoop jar $MRQL_HOME/lib/mrql-mr-*.jar org.apache.mrql.Main -libjars $LIBJARS $*
else
- CLASSPATH=$CUP_JAR:$JLINE_JAR:$GEN_JAR:$CORE_JAR:$MR_JAR:$HADOOP_JARS
+ CLASSPATH="$CUP_JAR:$JLINE_JAR:$GEN_JAR:$CORE_JAR:$MR_JAR:$HADOOP_JARS"
$JAVA_HOME/bin/java -classpath $CLASSPATH org.apache.mrql.Main $*
fi
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/28753398/bin/mrql.bsp
----------------------------------------------------------------------
diff --git a/bin/mrql.bsp b/bin/mrql.bsp
index 2adb018..c0fd179 100755
--- a/bin/mrql.bsp
+++ b/bin/mrql.bsp
@@ -24,24 +24,24 @@
#
#--------------------------------------------------------------------------------
-MRQL_HOME=$(dirname $(readlink -f $0))/..
+MRQL_HOME="$(cd `dirname $0`/..; pwd -P)"
-. $MRQL_HOME/conf/mrql-env.sh
-
-GEN_JAR=$(readlink -f $MRQL_HOME/lib/mrql-gen-*.jar)
-CORE_JAR=$(readlink -f $MRQL_HOME/lib/mrql-core-*.jar)
-BSP_JAR=$(readlink -f $MRQL_HOME/lib/mrql-bsp-*.jar)
+. "$MRQL_HOME/conf/mrql-env.sh"
+GEN_JAR=`ls "$MRQL_HOME"/lib/mrql-gen-*.jar`
+CORE_JAR=`ls "$MRQL_HOME"/lib/mrql-core-*.jar`
+BSP_JAR=`ls "$MRQL_HOME"/lib/mrql-bsp-*.jar`
export JAVA_HOME FS_DEFAULT_NAME BSP_MASTER_ADDRESS HAMA_ZOOKEEPER_QUORUM
+
if [ "$1" == "-local" ]; then
- LIBJARS=$CUP_JAR,$JLINE_JAR,$GEN_JAR,$CORE_JAR,$BSP_JAR
- export HAMA_CLASSPATH=$CUP_JAR:$JLINE_JAR:$CORE_JAR:$GEN_JAR:$BSP_JAR
+ LIBJARS="$CUP_JAR,$JLINE_JAR,$GEN_JAR,$CORE_JAR,$BSP_JAR"
+ export HAMA_CLASSPATH="$CUP_JAR:$JLINE_JAR:$CORE_JAR:$GEN_JAR:$BSP_JAR"
$HAMA_HOME/bin/hama jar $BSP_JAR org.apache.mrql.Main -libjars $LIBJARS -bsp $*
else if [ "$1" == "-dist" ]; then
# Hama distributed mode -libjars has a bug; create a single jar instead
- mkdir -p $MRQL_HOME/tmp/classes
+ mkdir -p "$MRQL_HOME/tmp/classes"
pushd $MRQL_HOME/tmp/classes >/dev/null
$JAVA_HOME/bin/jar xf $CUP_JAR
$JAVA_HOME/bin/jar xf $JLINE_JAR
@@ -53,7 +53,7 @@ else if [ "$1" == "-dist" ]; then
popd >/dev/null
$HAMA_HOME/bin/hama jar $MRQL_HOME/tmp/mrql-bsp.jar org.apache.mrql.Main -bsp $*
else
- HAMA_CLASSPATH=$CUP_JAR:$JLINE_JAR:$GEN_JAR:$CORE_JAR:$BSP_JAR:$HAMA_JAR:$HADOOP_JARS
+ HAMA_CLASSPATH="$CUP_JAR:$JLINE_JAR:$GEN_JAR:$CORE_JAR:$BSP_JAR:$HAMA_JAR:$HADOOP_JARS"
$JAVA_HOME/bin/java -classpath $HAMA_CLASSPATH org.apache.mrql.Main -bsp $*
fi
fi
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/28753398/bin/mrql.spark
----------------------------------------------------------------------
diff --git a/bin/mrql.spark b/bin/mrql.spark
index 2f9643c..529fe2f 100755
--- a/bin/mrql.spark
+++ b/bin/mrql.spark
@@ -24,17 +24,18 @@
#
#--------------------------------------------------------------------------------
-MRQL_HOME=$(dirname $(readlink -f $0))/..
+MRQL_HOME="$(cd `dirname $0`/..; pwd -P)"
-. $MRQL_HOME/conf/mrql-env.sh
+. "$MRQL_HOME/conf/mrql-env.sh"
-GEN_JAR=$(readlink -f $MRQL_HOME/lib/mrql-gen-*.jar)
-CORE_JAR=$(readlink -f $MRQL_HOME/lib/mrql-core-*.jar)
-SPARK_JAR=$(readlink -f $MRQL_HOME/lib/mrql-spark-*.jar)
+GEN_JAR=`ls "$MRQL_HOME"/lib/mrql-gen-*.jar`
+CORE_JAR=`ls "$MRQL_HOME"/lib/mrql-core-*.jar`
+SPARK_JAR=`ls "$MRQL_HOME"/lib/mrql-spark-*.jar`
-export SPARK_HOME FS_DEFAULT_NAME SPARK_MASTER
+export SPARK_HOME FS_DEFAULT_NAME SPARK_MASTER SPARK_MEM
+
+SPARK_CLASSPATH="$CUP_JAR:$JLINE_JAR:$GEN_JAR:$CORE_JAR:$SPARK_JAR:$SPARK_JARS:$HADOOP_JARS"
-SPARK_CLASSPATH=$CUP_JAR:$JLINE_JAR:$GEN_JAR:$CORE_JAR:$SPARK_JAR:$SPARK_JARS:$HADOOP_JARS
$JAVA_HOME/bin/java -classpath $SPARK_CLASSPATH org.apache.mrql.Main -spark $*
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/28753398/conf/mrql-env.sh
----------------------------------------------------------------------
diff --git a/conf/mrql-env.sh b/conf/mrql-env.sh
index 4c2af45..66d9da5 100644
--- a/conf/mrql-env.sh
+++ b/conf/mrql-env.sh
@@ -25,15 +25,15 @@
#--------------------------------------------------------------------------------
-# Required: The java installation directory
+# Required: The Java installation directory
JAVA_HOME=/root/jdk
# Required: The CUP parser library
-# You may download it from http://www2.cs.tum.edu/projects/cup/
+# You can download it from http://www2.cs.tum.edu/projects/cup/
CUP_JAR=${HOME}/.m2/repository/net/sf/squirrel-sql/thirdparty/non-maven/java-cup/11a/java-cup-11a.jar
# Required: The JLine library
-# You may download from http://jline.sourceforge.net
+# You can download from http://jline.sourceforge.net
JLINE_JAR=${HOME}/.m2/repository/jline/jline/1.0/jline-1.0.jar
@@ -61,7 +61,7 @@ SPARK_HOME=${HOME}/spark-0.8.0-incubating-bin-hadoop1
# URI of the Spark master node
SPARK_MASTER=spark://crete:7077
# Spark memory per node
-export SPARK_MEM="1g"
+SPARK_MEM="1g"
# Claspaths
@@ -71,3 +71,6 @@ HADOOP_JARS=${HADOOP_HOME}/hadoop-core-${HADOOP_VERSION}.jar:${HADOOP_HOME}/lib/
HAMA_JAR=${HAMA_HOME}/hama-core-${HAMA_VERSION}.jar
SPARK_JARS=${SPARK_HOME}/assembly/target/scala-2.9.3/*
+
+# for hadoop yarn (build using eg, mvn -Pyarn -Dhadoop.version=2.2.0 install)
+#HADOOP_JARS=${HADOOP_HOME}/share/hadoop/common/hadoop-common-${HADOOP_VERSION}.jar:${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-mapreduce-client-core-${HADOOP_VERSION}.jar:${HADOOP_HOME}/share/hadoop/hdfs/hadoop-hdfs-${HADOOP_VERSION}.jar:${HADOOP_HOME}/share/hadoop/common/lib/hadoop-annotations-${HADOOP_VERSION}.jar:${HADOOP_HOME}/share/hadoop/common/lib/commons-cli-1.2.jar