You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/07/12 05:19:10 UTC
[GitHub] nswamy closed pull request #11596: [MXNET-531] Integration Test for
Scala
nswamy closed pull request #11596: [MXNET-531] Integration Test for Scala
URL: https://github.com/apache/incubator-mxnet/pull/11596
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/Jenkinsfile b/Jenkinsfile
index 4ee1d982fed..c78a88e3b76 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -712,18 +712,6 @@ try {
}
}
},
- 'Scala: GPU': {
- node('mxnetlinux-gpu') {
- ws('workspace/ut-scala-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- unpack_lib('gpu', mx_dist_lib)
- docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_scala', true)
- publish_test_coverage()
- }
- }
- }
- },
'Clojure: CPU': {
node('mxnetlinux-cpu') {
ws('workspace/ut-clojure-cpu') {
@@ -979,6 +967,18 @@ try {
}
}
}
+ },
+ 'Scala: GPU': {
+ node('mxnetlinux-gpu') {
+ ws('workspace/ut-scala-gpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ unpack_lib('gpu', mx_dist_lib)
+ docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_scala', true)
+ publish_test_coverage()
+ }
+ }
+ }
}
// Disable until fixed https://github.com/apache/incubator-mxnet/issues/11441
// 'dist-kvstore tests GPU': {
diff --git a/Makefile b/Makefile
index 2cf1ed1ece2..5816637f989 100644
--- a/Makefile
+++ b/Makefile
@@ -593,9 +593,15 @@ scalapkg:
-Dcurrent_libdir="$(ROOTDIR)/lib" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
-scalatest:
+scalaunittest:
(cd $(ROOTDIR)/scala-package; \
- mvn verify -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \
+ mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \
+ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
+ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS))
+
+scalaintegrationtest:
+ (cd $(ROOTDIR)/scala-package; \
+ mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS))
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 4f0b1464742..3c196164d57 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -638,13 +638,7 @@ unittest_ubuntu_python3_quantization_gpu() {
unittest_ubuntu_cpu_scala() {
set -ex
make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1
- make scalatest USE_BLAS=openblas USE_DIST_KVSTORE=1
-}
-
-unittest_ubuntu_gpu_scala() {
- set -ex
- make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1
- make scalatest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1
+ make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1
}
unittest_ubuntu_cpu_clojure() {
@@ -724,6 +718,12 @@ integrationtest_ubuntu_gpu_cpp_package() {
cpp-package/tests/ci_test.sh
}
+integrationtest_ubuntu_gpu_scala() {
+ set -ex
+ make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1
+ make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1
+}
+
integrationtest_ubuntu_gpu_dist_kvstore() {
set -ex
export PYTHONPATH=./python/
diff --git a/scala-package/README.md b/scala-package/README.md
index 785bb39bd83..79caaf30915 100644
--- a/scala-package/README.md
+++ b/scala-package/README.md
@@ -55,13 +55,14 @@ make scalapkg
(Optional) run unit/integration tests by
```bash
-make scalatest
+make scalaunittest
+make scalaintegrationtest
```
Or run a subset of unit tests by, e.g.,
```bash
-make SCALA_TEST_ARGS=-Dsuites=org.apache.mxnet.NDArraySuite scalatest
+make SCALA_TEST_ARGS=-Dsuites=org.apache.mxnet.NDArraySuite scalaunittest
```
If everything goes well, you will find jars for `assembly`, `core` and `example` modules.
diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml
index 3b1b051f60b..631b6bd3810 100644
--- a/scala-package/core/pom.xml
+++ b/scala-package/core/pom.xml
@@ -14,6 +14,18 @@
<name>MXNet Scala Package - Core</name>
<profiles>
+ <profile>
+ <id>unittest</id>
+ <properties>
+ <skiptest>false</skiptest>
+ </properties>
+ </profile>
+ <profile>
+ <id>integrationtest</id>
+ <properties>
+ <skiptest>true</skiptest>
+ </properties>
+ </profile>
<profile>
<id>osx-x86_64-cpu</id>
<properties>
@@ -55,6 +67,7 @@
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<configuration>
+ <skipTests>${skiptest}</skipTests>
<argLine>
-Djava.library.path=${project.parent.basedir}/native/${platform}/target \
-Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties
@@ -74,6 +87,7 @@
<artifactId>scala-maven-plugin</artifactId>
<version>3.3.2</version>
<configuration>
+ <skip>${skiptest}</skip>
</configuration>
<executions>
<execution>
diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml
index 4c7a0ed2e12..0d751323a93 100644
--- a/scala-package/examples/pom.xml
+++ b/scala-package/examples/pom.xml
@@ -14,6 +14,18 @@
<name>MXNet Scala Package - Examples</name>
<profiles>
+ <profile>
+ <id>unittest</id>
+ <properties>
+ <skiptest>true</skiptest>
+ </properties>
+ </profile>
+ <profile>
+ <id>integrationtest</id>
+ <properties>
+ <skiptest>false</skiptest>
+ </properties>
+ </profile>
<profile>
<id>osx-x86_64-cpu</id>
<properties>
@@ -129,6 +141,7 @@
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<configuration>
+ <skipTests>${skiptest}</skipTests>
<argLine>
-Djava.library.path=${project.parent.basedir}/native/${platform}/target \
-Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties
diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/MNISTExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/MNISTExampleSuite.scala
index 0e3b7ec8f34..3e91b5b0245 100644
--- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/MNISTExampleSuite.scala
+++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/imclassification/MNISTExampleSuite.scala
@@ -35,11 +35,7 @@ class MNISTExampleSuite extends FunSuite with BeforeAndAfterAll {
private val logger = LoggerFactory.getLogger(classOf[MNISTExampleSuite])
test("Example CI: Test MNIST Training") {
- // This test is CPU only
- if (System.getenv().containsKey("SCALA_TEST_ON_GPU") &&
- System.getenv("SCALA_TEST_ON_GPU").toInt == 1) {
- logger.info("CPU test only, skipped...")
- } else {
+
logger.info("Downloading mnist model")
val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci"
val tempDirPath = System.getProperty("java.io.tmpdir")
@@ -60,7 +56,5 @@ class MNISTExampleSuite extends FunSuite with BeforeAndAfterAll {
Process("rm -rf " + modelDirPath) !
assert(output >= 0.95f)
- }
-
}
}
diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml
index 208d19ee9ce..8ff08c3de2d 100644
--- a/scala-package/infer/pom.xml
+++ b/scala-package/infer/pom.xml
@@ -14,6 +14,18 @@
<name>MXNet Scala Package - Inference</name>
<profiles>
+ <profile>
+ <id>unittest</id>
+ <properties>
+ <skiptest>false</skiptest>
+ </properties>
+ </profile>
+ <profile>
+ <id>integrationtest</id>
+ <properties>
+ <skiptest>true</skiptest>
+ </properties>
+ </profile>
<profile>
<id>osx-x86_64-cpu</id>
<properties>
@@ -55,6 +67,7 @@
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<configuration>
+ <skipTests>${skiptest}</skipTests>
<argLine>
-Djava.library.path=${project.parent.basedir}/native/${platform}/target \
-Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties
diff --git a/scala-package/macros/pom.xml b/scala-package/macros/pom.xml
index 9436af6f5c3..15f13f3940e 100644
--- a/scala-package/macros/pom.xml
+++ b/scala-package/macros/pom.xml
@@ -14,6 +14,18 @@
<name>MXNet Scala Package - Macros</name>
<profiles>
+ <profile>
+ <id>unittest</id>
+ <properties>
+ <skiptest>false</skiptest>
+ </properties>
+ </profile>
+ <profile>
+ <id>integrationtest</id>
+ <properties>
+ <skiptest>true</skiptest>
+ </properties>
+ </profile>
<profile>
<id>osx-x86_64-cpu</id>
<properties>
@@ -103,6 +115,7 @@
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<configuration>
+ <skipTests>${skiptest}</skipTests>
<environmentVariables>
<MXNET_BASEDIR>${project.parent.basedir}/init-native</MXNET_BASEDIR>
</environmentVariables>
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services