[incubator-mxnet] branch master updated: [scala-package][spark] fix example script (#7411)

[li...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

liuyizhi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git


The following commit(s) were added to refs/heads/master by this push:
     new d2dbffe  [scala-package][spark] fix example script (#7411)
d2dbffe is described below

commit d2dbffe194f4eed728c883927ce639919cb2078f
Author: Nan Zhu <Co...@users.noreply.github.com>
AuthorDate: Fri Aug 18 02:27:31 2017 -0700

    [scala-package][spark] fix example script (#7411)
    
    * temp
    
    * temp
    
    * fix example script
    
    * update indent
---
 .gitignore                                   |  2 +
 scala-package/spark/bin/run-mnist-example.sh | 59 +++++++++++++++++-----------
 scala-package/spark/pom.xml                  |  2 +-
 3 files changed, 40 insertions(+), 23 deletions(-)

diff --git a/.gitignore b/.gitignore
index 82d2e56..4e4ff78 100644
--- a/.gitignore
+++ b/.gitignore
@@ -146,3 +146,5 @@ bld
 target
 
 bin/im2rec
+
+model/
\ No newline at end of file
diff --git a/scala-package/spark/bin/run-mnist-example.sh b/scala-package/spark/bin/run-mnist-example.sh
index cae1938..57b8a18 100755
--- a/scala-package/spark/bin/run-mnist-example.sh
+++ b/scala-package/spark/bin/run-mnist-example.sh
@@ -18,47 +18,62 @@
 # under the License.
 
 CURR_DIR=$(cd `dirname $0`; pwd)
-MODULE_DIR=$(cd $CURR_DIR/../; pwd)
-ROOT_DIR=$(cd $CURR_DIR/../../; pwd)
+SPARK_MODULE_DIR=$(cd $CURR_DIR/../; pwd)
+SCALA_PKG_DIR=$(cd $CURR_DIR/../../; pwd)
 
+OS=""
 
-LIB_DIR=${MODULE_DIR}/target/classes/lib
-JAR=${MODULE_DIR}/target/mxnet-spark_2.10-0.1.2-SNAPSHOT.jar
+if [ "$(uname)" == "Darwin" ]; then
+	# Do something under Mac OS X platform
+  OS='osx-x86_64-cpu'
+elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
+  OS='linux-x86_64-cpu'
+fi
 
-LIBS=${ROOT_DIR}/assembly/linux-x86_64-cpu/target/mxnet-full_2.10-linux-x86_64-cpu-0.1.2-SNAPSHOT.jar
-LIBS="${LIBS},${LIB_DIR}/args4j-2.0.29.jar,${LIB_DIR}/scala-library-2.10.4.jar,${JAR}"
+LIB_DIR=${SPARK_MODULE_DIR}/target/classes/lib
+SPARK_JAR=`find ${SPARK_MODULE_DIR}/target -name "*.jar" -type f -exec ls "{}" + | grep -v -E '(javadoc|sources)'`
+SCALA_JAR=`find ${SCALA_PKG_DIR}/assembly/$OS/target -maxdepth 1 -name "*.jar" -type f -exec ls "{}" + | grep -v -E '(javadoc|sources)'`
 
-SPARK_OPTS+=" --name mxnet"
+SPARK_OPTS+=" --name mxnet-spark-mnist"
 SPARK_OPTS+=" --driver-memory 1g"
 SPARK_OPTS+=" --executor-memory 1g"
 SPARK_OPTS+=" --num-executors 2"
 SPARK_OPTS+=" --executor-cores 1"
-SPARK_OPTS+=" --jars ${LIBS}"
+SPARK_OPTS+=" --jars ${SCALA_JAR}"
 
-# You can download these two files as training & validation set.
-# They were converted from the MNIST dataset,
-# in which each sample was simply flatterned to an array of floats.
-# https://s3-us-west-2.amazonaws.com/mxnet.liuyz/data/mnist/train.txt
-# https://s3-us-west-2.amazonaws.com/mxnet.liuyz/data/mnist/val.txt
+# Download training and test set
+if [ ! -f ./train.txt ]; then
+  wget https://s3-us-west-2.amazonaws.com/mxnet.liuyz/data/mnist/train.txt
+fi
+
+if [ ! -f ./val.txt ]; then
+  wget https://s3-us-west-2.amazonaws.com/mxnet.liuyz/data/mnist/val.txt
+fi
 
 # running opts
-RUN_OPTS+=" --input ${INPUT_TRAIN}"
-RUN_OPTS+=" --input-val ${INPUT_VAL}"
-RUN_OPTS+=" --output ${OUTPUT}"
+RUN_OPTS+=" --input train.txt"
+RUN_OPTS+=" --input-val val.txt"
+RUN_OPTS+=" --output ./"
 # These jars are required by the KVStores at runtime.
 # They will be uploaded and distributed to each node automatically.
-RUN_OPTS+=" --jars ${LIBS}"
+RUN_OPTS+=" --jars $SCALA_JAR,$SPARK_JAR"
 RUN_OPTS+=" --num-server 1"
 RUN_OPTS+=" --num-worker 2"
-RUN_OPTS+=" --java /usr/local/jdk1.8.0_60/bin/java"
+RUN_OPTS+=" --java $JAVA_HOME/bin/java"
 RUN_OPTS+=" --model mlp"
 RUN_OPTS+=" --cpus 0,1"
 RUN_OPTS+=" --num-epoch 5"
 
-${SPARK_HOME}/bin/spark-submit --master spark://localhost:7077 \
-  --conf spark.dynamicAllocation.enabled=false \
-  --conf spark.speculation=false \
+# check if SPARK_HOME is set
+if [ -z "$SPARK_HOME" ]; then
+  echo "SPARK_HOME is unset";
+  exit 1
+fi
+
+HOST=`hostname`
+
+$SPARK_HOME/bin/spark-submit --master spark://$HOST:7077 \
   --class ml.dmlc.mxnet.spark.example.ClassificationExample \
   ${SPARK_OPTS} \
-  ${JAR} \
+  ${SPARK_JAR} \
   ${RUN_OPTS}
diff --git a/scala-package/spark/pom.xml b/scala-package/spark/pom.xml
index 18170b9..c59662f 100644
--- a/scala-package/spark/pom.xml
+++ b/scala-package/spark/pom.xml
@@ -14,7 +14,7 @@
   <name>MXNet Scala Package - Spark ML</name>
 
   <properties>
-    <spark.version>1.6.1</spark.version>
+    <spark.version>1.6.3</spark.version>
   </properties>
 
   <dependencies>

-- 
To stop receiving notification emails like this one, please contact
['"commits@mxnet.apache.org" <co...@mxnet.apache.org>'].