You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by la...@apache.org on 2019/01/10 02:45:07 UTC
[incubator-mxnet] branch master updated: Add new Maven build for
Scala package (#13819)
This is an automated email from the ASF dual-hosted git repository.
lanking pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 5fe2172 Add new Maven build for Scala package (#13819)
5fe2172 is described below
commit 5fe2172d256d51bbedd0b8b972ffa479ddb5cf35
Author: Lanking <la...@live.com>
AuthorDate: Wed Jan 9 21:44:50 2019 -0500
Add new Maven build for Scala package (#13819)
* clean up build
* fix minor issue and add mkldnn
* fix mx_dist problem
* fix clojure build
* fix skip test
---
ci/docker/runtime_functions.sh | 18 +++++++------
ci/jenkins/Jenkins_steps.groovy | 30 +++++++++++++++-------
ci/jenkins/Jenkinsfile_unix_cpu | 1 +
contrib/clojure-package/README.md | 4 +--
.../CNNClassifierExampleSuite.scala | 3 +--
.../ImageClassifierExampleSuite.scala | 1 -
.../mxnetexamples/multitask/MultiTaskSuite.scala | 3 +--
.../mxnetexamples/profiler/ProfilerSuite.scala | 3 +--
scala-package/spark/README.md | 3 ++-
tests/jenkins/run_test.sh | 4 +--
tests/jenkins/run_test_amzn_linux_gpu.sh | 4 +--
tests/jenkins/run_test_ubuntu.sh | 4 +--
12 files changed, 45 insertions(+), 33 deletions(-)
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 94803ce..a6bb106 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -847,23 +847,24 @@ unittest_ubuntu_python3_quantization_gpu() {
unittest_ubuntu_cpu_scala() {
set -ex
scala_prepare
- make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
- make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
+ cd scala-package
+ mvn -B integration-test
}
unittest_centos7_cpu_scala() {
set -ex
cd /work/mxnet
scala_prepare
- make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
- make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
+ cd scala-package
+ mvn -B integration-test
}
unittest_ubuntu_cpu_clojure() {
set -ex
scala_prepare
- make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
- make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
+ cd scala-package
+ mvn -B install
+ cd ..
./contrib/clojure-package/ci-test.sh
}
@@ -1008,8 +1009,9 @@ integrationtest_ubuntu_cpu_dist_kvstore() {
integrationtest_ubuntu_gpu_scala() {
set -ex
scala_prepare
- make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1 CI=1
- make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
+ cd scala-package
+ export SCALA_TEST_ON_GPU=1
+ mvn -B integration-test -DskipTests=false
}
integrationtest_ubuntu_gpu_dist_kvstore() {
diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index 57454b9..33d76aa 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -28,8 +28,6 @@ mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdpart
// Python wheels
mx_pip = 'build/*.whl'
-// for scala build, need to pass extra libs when run with dist_kvstore
-mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a'
// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
@@ -100,7 +98,7 @@ def compile_unix_cpu_openblas() {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false)
- utils.pack_lib('cpu', mx_dist_lib, true)
+ utils.pack_lib('cpu', mx_lib, true)
}
}
}
@@ -128,7 +126,7 @@ def compile_unix_mkl_cpu() {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkl', false)
- utils.pack_lib('cpu_mkl', mx_dist_lib, true)
+ utils.pack_lib('cpu_mkl', mx_mkldnn_lib, true)
}
}
}
@@ -254,7 +252,7 @@ def compile_centos7_cpu() {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
utils.docker_run('centos7_cpu', 'build_centos7_cpu', false)
- utils.pack_lib('centos7_cpu', mx_dist_lib, true)
+ utils.pack_lib('centos7_cpu', mx_lib, true)
}
}
}
@@ -825,7 +823,21 @@ def test_unix_scala_cpu() {
node(NODE_LINUX_CPU) {
ws('workspace/ut-scala-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
- utils.unpack_and_init('cpu', mx_dist_lib, true)
+ utils.unpack_and_init('cpu', mx_lib, true)
+ utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_scala', false)
+ utils.publish_test_coverage()
+ }
+ }
+ }
+ }]
+}
+
+def test_unix_scala_mkldnn_cpu(){
+ return ['Scala: MKLDNN-CPU': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/ut-scala-mkldnn-cpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.unpack_and_init('mkldnn_cpu', mx_mkldnn_lib, true)
utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_scala', false)
utils.publish_test_coverage()
}
@@ -839,7 +851,7 @@ def test_unix_scala_gpu() {
node(NODE_LINUX_GPU) {
ws('workspace/ut-scala-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
- utils.unpack_and_init('gpu', mx_dist_lib, true)
+ utils.unpack_and_init('gpu', mx_lib, true)
utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_scala', true)
utils.publish_test_coverage()
}
@@ -853,7 +865,7 @@ def test_unix_clojure_cpu() {
node(NODE_LINUX_CPU) {
ws('workspace/ut-clojure-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
- utils.unpack_and_init('cpu', mx_dist_lib, true)
+ utils.unpack_and_init('cpu', mx_lib, true)
utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_clojure', false)
utils.publish_test_coverage()
}
@@ -1029,7 +1041,7 @@ def test_centos7_scala_cpu() {
node(NODE_LINUX_CPU) {
ws('workspace/ut-scala-centos7-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
- utils.unpack_and_init('centos7_cpu', mx_dist_lib, true)
+ utils.unpack_and_init('centos7_cpu', mx_lib, true)
utils.docker_run('centos7_cpu', 'unittest_centos7_cpu_scala', false)
utils.publish_test_coverage()
}
diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu
index 8f41fa6..1cbb6a7 100644
--- a/ci/jenkins/Jenkinsfile_unix_cpu
+++ b/ci/jenkins/Jenkinsfile_unix_cpu
@@ -50,6 +50,7 @@ core_logic: {
custom_steps.test_unix_python3_mkldnn_cpu(),
custom_steps.test_unix_python3_mkldnn_mkl_cpu(),
custom_steps.test_unix_scala_cpu(),
+ custom_steps.test_unix_scala_mkldnn_cpu(),
custom_steps.test_unix_clojure_cpu(),
custom_steps.test_unix_r_cpu(),
custom_steps.test_unix_julia_cpu(),
diff --git a/contrib/clojure-package/README.md b/contrib/clojure-package/README.md
index 7930bef..ba6160a 100644
--- a/contrib/clojure-package/README.md
+++ b/contrib/clojure-package/README.md
@@ -193,8 +193,8 @@ The outcome of this step will be a shared library `lib/libmxnet.so` that is used
- Build and install the Scala package in your local Maven directory using the following commands:
```bash
- make scalapkg
- make scalainstall
+ cd scala-package
+ mvn install
```
#### Building the Clojure jar
diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala
index 44025c0..a7e36df 100644
--- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala
+++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala
@@ -29,8 +29,7 @@ import org.slf4j.LoggerFactory
import scala.sys.process.Process
/**
- * Integration test for imageClassifier example.
- * This will run as a part of "make scalatest"
+ * Integration test for CNN example.
*/
class CNNClassifierExampleSuite extends FunSuite with BeforeAndAfterAll {
private val logger = LoggerFactory.getLogger(classOf[CNNClassifierExampleSuite])
diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala
index 27d9bb4..c5308ac 100644
--- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala
+++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala
@@ -27,7 +27,6 @@ import sys.process.Process
/**
* Integration test for imageClassifier example.
- * This will run as a part of "make scalatest"
*/
class ImageClassifierExampleSuite extends FunSuite with BeforeAndAfterAll {
private val logger = LoggerFactory.getLogger(classOf[ImageClassifierExampleSuite])
diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/multitask/MultiTaskSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/multitask/MultiTaskSuite.scala
index 983978d..65902c7 100644
--- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/multitask/MultiTaskSuite.scala
+++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/multitask/MultiTaskSuite.scala
@@ -25,8 +25,7 @@ import org.scalatest.FunSuite
/**
- * Integration test for imageClassifier example.
- * This will run as a part of "make scalatest"
+ * Integration test for Multi-task example.
*/
class MultiTaskSuite extends FunSuite {
test("Multitask Test") {
diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/profiler/ProfilerSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/profiler/ProfilerSuite.scala
index 859a8c0..67638b8 100644
--- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/profiler/ProfilerSuite.scala
+++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/profiler/ProfilerSuite.scala
@@ -25,8 +25,7 @@ import org.apache.mxnet.Profiler
import org.apache.mxnet.Context
/**
- * Integration test for imageClassifier example.
- * This will run as a part of "make scalatest"
+ * Integration test for profiler example.
*/
class ProfilerSuite extends FunSuite with BeforeAndAfterAll {
private val logger = LoggerFactory.getLogger(classOf[ProfilerSuite])
diff --git a/scala-package/spark/README.md b/scala-package/spark/README.md
index 0610664..503c279 100644
--- a/scala-package/spark/README.md
+++ b/scala-package/spark/README.md
@@ -16,7 +16,8 @@ Checkout the [Installation Guide](http://mxnet.io/get_started/setup.html) contai
Compile the Scala Package by
```bash
-make scalapkg
+cd scala-package
+mvn package
```
This will automatically build the `spark` submodule. Now you can submit Spark job with these built jars.
diff --git a/tests/jenkins/run_test.sh b/tests/jenkins/run_test.sh
index bc69ca1..5ef8c1a 100755
--- a/tests/jenkins/run_test.sh
+++ b/tests/jenkins/run_test.sh
@@ -56,8 +56,8 @@ nosetests3 --verbose tests/python/train || exit -1
echo "BUILD scala_test"
export PATH=$PATH:/opt/apache-maven/bin
-make scalapkg || exit -1
-make scalatest || exit -1
+cd scala-package
+mvn install || exit -1
# echo "BUILD julia_test"
# export MXNET_HOME="${PWD}"
diff --git a/tests/jenkins/run_test_amzn_linux_gpu.sh b/tests/jenkins/run_test_amzn_linux_gpu.sh
index ecfb521..57d9c78 100755
--- a/tests/jenkins/run_test_amzn_linux_gpu.sh
+++ b/tests/jenkins/run_test_amzn_linux_gpu.sh
@@ -65,5 +65,5 @@ nosetests3 --verbose tests/python/train
#julia -e 'try Pkg.clone("MXNet"); catch end; Pkg.checkout("MXNet"); Pkg.build("MXNet"); Pkg.test("MXNet")' || exit -1
echo "BUILD scala_test"
-make scalapkg
-make scalatest
+cd scala-package
+mvn integration-test
diff --git a/tests/jenkins/run_test_ubuntu.sh b/tests/jenkins/run_test_ubuntu.sh
index cdddd28..28e0033 100755
--- a/tests/jenkins/run_test_ubuntu.sh
+++ b/tests/jenkins/run_test_ubuntu.sh
@@ -68,6 +68,6 @@ nosetests3 --verbose tests/python/train || exit 1
echo "BUILD scala_test"
export PATH=$PATH:/opt/apache-maven/bin
-make scalapkg || exit 1
-make scalatest || exit 1
+cd scala-package
+mvn integration-test || exit 1