You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2019/01/16 02:12:25 UTC
[incubator-mxnet] Diff for: [GitHub] lanking520 merged pull request #13767:
[MXNET-1191] Maven Jenkins Pipeline with Static Binary and GPU
diff --git a/.gitignore b/.gitignore
index 7eb8e7d6e77..fc7992675e2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -146,6 +146,7 @@ tools/pip_package/mxnet
# temporary path for building dependencies when building wheel
./deps/
+./staticdeps/
bld
./tmp/*
*.jar
diff --git a/ci/docker/Dockerfile.publish.ubuntu1604_cpu b/ci/docker/Dockerfile.publish.ubuntu1604_cpu
new file mode 100644
index 00000000000..df284be2b37
--- /dev/null
+++ b/ci/docker/Dockerfile.publish.ubuntu1604_cpu
@@ -0,0 +1,42 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+
+FROM ubuntu:16.04
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_base.sh /work/
+RUN /work/ubuntu_base.sh
+
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
+
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY runtime_functions.sh /work/
+
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/Dockerfile.publish.ubuntu1604_gpu b/ci/docker/Dockerfile.publish.ubuntu1604_gpu
new file mode 100644
index 00000000000..2a1f8594e75
--- /dev/null
+++ b/ci/docker/Dockerfile.publish.ubuntu1604_gpu
@@ -0,0 +1,42 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+
+FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu16.04
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_base.sh /work/
+RUN /work/ubuntu_base.sh
+
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
+
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY runtime_functions.sh /work/
+
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/install/ubuntu_base.sh b/ci/docker/install/ubuntu_base.sh
index b34c0b3e18f..f36e53279f5 100755
--- a/ci/docker/install/ubuntu_base.sh
+++ b/ci/docker/install/ubuntu_base.sh
@@ -33,4 +33,8 @@ apt-get install -y \
software-properties-common \
sudo \
unzip \
+ expect \
+ gnupg \
+ gnupg2 \
+ gnupg-agent \
wget
diff --git a/ci/docker/install/ubuntu_scala.sh b/ci/docker/install/ubuntu_scala.sh
index 5bade47463e..8b23d614b49 100755
--- a/ci/docker/install/ubuntu_scala.sh
+++ b/ci/docker/install/ubuntu_scala.sh
@@ -36,9 +36,6 @@ apt-get install -y \
openjdk-8-jdk \
openjdk-8-jre \
software-properties-common \
- gnupg \
- gnupg2 \
- gnupg-agent \
scala
# Ubuntu 14.04
diff --git a/ci/publish/Jenkinsfile b/ci/publish/Jenkinsfile
index 9a360c6b5be..c14ab2502cd 100644
--- a/ci/publish/Jenkinsfile
+++ b/ci/publish/Jenkinsfile
@@ -21,7 +21,7 @@
// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
//mxnet libraries
-mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/**, scala-package/local-snapshot/**'
+mx_scala_pub = 'lib/**, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/**/target/**, scala-package/*/target/repo/**'
// timeout in minutes
max_time = 120
@@ -34,8 +34,9 @@ node('restricted-utility') {
utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')
// CPU and GPU. OSX nodes are not currently supported by Jenkins
-def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU]
+def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU_P3]
def scalaOSMap = ['cpu': 'linux-x86_64-cpu', 'gpu': 'linux-x86_64-gpu']
+def scalaVariantMap = ['cpu': 'cpu', 'gpu': 'cu92']
def wrapStep(nodeToRun, workspaceName, step) {
return {
@@ -50,13 +51,13 @@ def wrapStep(nodeToRun, workspaceName, step) {
}
def toBuild = [:]
-def labels = ['cpu'] // , 'gpu']
+def labels = ['cpu', 'gpu']
for (x in labels) {
def label = x // Required due to language
- toBuild["Scala Build ${label}"] = wrapStep(nodeMap[label], "build-scala-${label}") {
- withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) {
+ toBuild["Scala Build ${label}"] = wrapStep(nodeMap['cpu'], "build-scala-${label}") {
+ withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}", "mxnet_variant=${scalaVariantMap[label]}"]) {
utils.init_git()
- utils.docker_run("ubuntu_${label}", 'publish_scala_build', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE')
+ utils.docker_run("publish.ubuntu1404_cpu", 'publish_scala_build', false, '500m', 'MAVEN_PUBLISH_OS_TYPE mxnet_variant')
utils.pack_lib("scala_${label}", mx_scala_pub, false)
}
}
@@ -69,8 +70,10 @@ for (x in labels) {
for (y in systems) {
def system = y // Required due to language
toTest["Scala Test ${system} ${label}"] = wrapStep(nodeMap[label], "test-scala-${system}-${label}") {
- utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
- utils.docker_run("publish.test.${system}_${label}", 'publish_scala_test', label == 'gpu')
+ withEnv(["mxnet_variant=${scalaVariantMap[label]}"]) {
+ utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
+ utils.docker_run("publish.test.${system}_${label}", 'publish_scala_test', label == 'gpu', '500m', 'mxnet_variant')
+ }
}
}
}
@@ -78,10 +81,10 @@ for (x in labels) {
def toDeploy = [:]
for (x in labels) {
def label = x // Required due to language
- toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap[label], "deploy-scala-${label}") {
- withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) {
+ toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap['cpu'], "deploy-scala-${label}") {
+ withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}", "mxnet_variant=${scalaVariantMap[label]}"]) {
utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
- utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION')
+ utils.docker_run("publish.ubuntu1604_cpu", 'publish_scala_deploy', false, '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION mxnet_variant')
}
}
}
@@ -101,7 +104,7 @@ core_logic: {
,
failure_handler: {
if (currentBuild.result == "FAILURE") {
- // emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}'
+ emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}'
}
}
)
diff --git a/ci/publish/scala/build.sh b/ci/publish/scala/build.sh
index 17f969afe14..11386ce2114 100755
--- a/ci/publish/scala/build.sh
+++ b/ci/publish/scala/build.sh
@@ -22,7 +22,9 @@ set -ex
# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu
# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu
-bash scala-package/dev/compile-mxnet-backend.sh $MAVEN_PUBLISH_OS_TYPE ./
+source tools/staticbuild/build.sh $mxnet_variant maven
+
+set -ex
# Compile tests for discovery later
cd scala-package
diff --git a/ci/publish/scala/deploy.sh b/ci/publish/scala/deploy.sh
index 4eb33907eeb..00c67214742 100755
--- a/ci/publish/scala/deploy.sh
+++ b/ci/publish/scala/deploy.sh
@@ -18,12 +18,10 @@
set -ex
-# Setup Environment Variables
-# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu
-# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu
-
-# Run python to configure keys
-python3 ci/publish/scala/buildkey.py
+# On Jenkins, run python script to configure keys
+if [[ $BUILD_ID ]]; then
+ python3 ci/publish/scala/buildkey.py
+fi
# Updating cache
mkdir -p ~/.gnupg
@@ -37,5 +35,7 @@ cd scala-package
mvn -B deploy -Pnightly
-# Clear all password .xml files, exp files, and gpg key files
-rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp
+# On Jenkins, clear all password .xml files, exp files, and gpg key files
+if [[ $BUILD_ID ]]; then
+ rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp
+fi
diff --git a/ci/publish/scala/fullDeploy.sh b/ci/publish/scala/fullDeploy.sh
old mode 100644
new mode 100755
diff --git a/ci/publish/scala/test.sh b/ci/publish/scala/test.sh
index 5cef35ca3c2..32e36299a6f 100755
--- a/ci/publish/scala/test.sh
+++ b/ci/publish/scala/test.sh
@@ -24,5 +24,10 @@ fi
# Test
cd scala-package/packageTest
-# make testlocal CI=1
-make testsnapshot UNIT=1 CI=1
+
+if [[ $mxnet_variant == cu* ]]; then
+ export SCALA_TEST_ON_GPU=1
+ make testlocal USE_CUDA=1 CI=1
+else
+ make testlocal CI=1
+fi
diff --git a/make/maven/maven_linux_cu92.mk b/make/maven/maven_linux_cu92.mk
new file mode 100644
index 00000000000..0ed18a71b97
--- /dev/null
+++ b/make/maven/maven_linux_cu92.mk
@@ -0,0 +1,185 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------------------------
+# Template configuration for compiling mxnet for making maven package
+#-------------------------------------------------------------------------------
+
+#---------------------
+# choice of compiler
+#--------------------
+
+export CC = gcc
+export CXX = g++
+export NVCC = nvcc
+
+# whether compile with options for MXNet developer
+DEV = 0
+
+# whether compile with debug
+DEBUG = 0
+
+# whether compiler with profiler
+USE_PROFILER = 1
+
+# whether to turn on signal handler (e.g. segfault logger)
+USE_SIGNAL_HANDLER = 0
+
+# the additional link flags you want to add
+ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections
+
+# the additional compile flags you want to add
+ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections
+
+#---------------------------------------------
+# matrix computation libraries for CPU/GPU
+#---------------------------------------------
+
+# choose the version of blas you want to use
+# can be: mkl, blas, atlas, openblas
+# in default use atlas for linux while apple for osx
+USE_BLAS=openblas
+
+# whether use opencv during compilation
+# you can disable it, however, you will not able to use
+# imbin iterator
+USE_OPENCV = 1
+
+# whether use CUDA during compile
+USE_CUDA = 1
+
+# add the path to CUDA library to link and compile flag
+# if you have already add them to environment variable, leave it as NONE
+# USE_CUDA_PATH = /usr/local/cuda
+USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.2
+
+# whether to use CuDNN library
+USE_CUDNN = 1
+
+# whether to use NCCL library
+USE_NCCL = 1
+
+# CUDA architecture setting: going with all of them.
+# For CUDA < 6.0, comment the *_50 lines for compatibility.
+# CUDA_ARCH :=
+
+# whether use cuda runtime compiling for writing kernels in native language (i.e. Python)
+USE_NVRTC = 1
+
+# use openmp for parallelization
+USE_OPENMP = 1
+USE_OPERATOR_TUNING = 1
+USE_LIBJPEG_TURBO = 1
+
+# whether use MKL-DNN library
+USE_MKLDNN = 0
+
+
+# MKL ML Library for Intel CPU/Xeon Phi
+# Please refer to MKL_README.md for details
+
+# MKL ML Library folder, need to be root for /usr/local
+# Change to User Home directory for standard user
+# For USE_BLAS!=mkl only
+MKLML_ROOT=/usr/local
+
+# whether use MKL2017 library
+USE_MKL2017 = 0
+
+# whether use MKL2017 experimental feature for high performance
+# Prerequisite USE_MKL2017=1
+USE_MKL2017_EXPERIMENTAL = 0
+
+# whether use NNPACK library
+USE_NNPACK = 0
+
+# whether use lapack during compilation
+# only effective when compiled with blas versions openblas/apple/atlas/mkl
+USE_LAPACK = 1
+
+# path to lapack library in case of a non-standard installation
+USE_LAPACK_PATH = $(DEPS_PATH)/lib
+
+# add path to intel library, you may need it for MKL, if you did not add the path
+# to environment variable
+USE_INTEL_PATH = NONE
+
+# If use MKL, choose static link automatically to allow python wrapper
+ifeq ($(USE_BLAS), mkl)
+USE_STATIC_MKL = 1
+else
+USE_STATIC_MKL = NONE
+endif
+
+#----------------------------
+# Settings for power and arm arch
+#----------------------------
+ARCH := $(shell uname -a)
+ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64))
+ USE_SSE=0
+else
+ USE_SSE=1
+endif
+
+#----------------------------
+# distributed computing
+#----------------------------
+
+# whether or not to enable multi-machine supporting
+USE_DIST_KVSTORE = 1
+
+# whether or not allow to read and write HDFS directly. If yes, then hadoop is
+# required
+USE_HDFS = 0
+
+# path to libjvm.so. required if USE_HDFS=1
+LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server
+
+# whether or not allow to read and write AWS S3 directly. If yes, then
+# libcurl4-openssl-dev is required, it can be installed on Ubuntu by
+# sudo apt-get install -y libcurl4-openssl-dev
+USE_S3 = 1
+
+#----------------------------
+# additional operators
+#----------------------------
+
+# path to folders containing projects specific operators that you don't want to put in src/operators
+EXTRA_OPERATORS =
+
+
+#----------------------------
+# plugins
+#----------------------------
+
+# whether to use caffe integration. This requires installing caffe.
+# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH
+# CAFFE_PATH = $(HOME)/caffe
+# MXNET_PLUGINS += plugin/caffe/caffe.mk
+
+# whether to use torch integration. This requires installing torch.
+# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH
+# TORCH_PATH = $(HOME)/torch
+# MXNET_PLUGINS += plugin/torch/torch.mk
+
+# WARPCTC_PATH = $(HOME)/warp-ctc
+# MXNET_PLUGINS += plugin/warpctc/warpctc.mk
+
+# whether to use sframe integration. This requires build sframe
+# git@github.com:dato-code/SFrame.git
+# SFRAME_PATH = $(HOME)/SFrame
+# MXNET_PLUGINS += plugin/sframe/plugin.mk
diff --git a/scala-package/deploy/pom.xml b/scala-package/deploy/pom.xml
index 542bf6c1688..4e9da8908be 100644
--- a/scala-package/deploy/pom.xml
+++ b/scala-package/deploy/pom.xml
@@ -23,7 +23,6 @@
<ARTIFACT_ID>mxnet-full_2.11-${platform}-${flavor}</ARTIFACT_ID>
<revision>1.5.0-SNAPSHOT</revision>
<repositoryId>apache.snapshots.https</repositoryId>
- <repo_url>file://${project.build.directory}/repo</repo_url>
<deploy_asc_types/>
<deploy_asc_classifers/>
<deploy_asc_files/>
@@ -118,6 +117,7 @@
</goals>
<configuration>
<description>${project.description}</description>
+ <skip>false</skip>
<repositoryId>${repositoryId}</repositoryId>
<url>${repo_url}</url>
<groupId>${project.groupId}</groupId>
diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala
index 14fb7b85e9b..2ccd38fc4f9 100644
--- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala
+++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala
@@ -49,14 +49,19 @@ class ExampleRNNSuite extends FunSuite with BeforeAndAfterAll {
System.getenv("SCALA_TEST_ON_GPU").toInt == 1) {
ctx = Context.gpu()
}
- LstmBucketing.runTraining(tempDirPath + "/RNN/sherlockholmes.train.txt",
- tempDirPath + "/RNN/sherlockholmes.valid.txt", Array(ctx), 1)
+ if (!System.getenv().containsKey("CI")) {
+ LstmBucketing.runTraining(tempDirPath + "/RNN/sherlockholmes.train.txt",
+ tempDirPath + "/RNN/sherlockholmes.valid.txt", Array(ctx), 1)
+ } else {
+ logger.info("Skipping test on CI...")
+ }
}
test("Example CI: Test TrainCharRNN") {
val tempDirPath = System.getProperty("java.io.tmpdir")
if (System.getenv().containsKey("SCALA_TEST_ON_GPU") &&
- System.getenv("SCALA_TEST_ON_GPU").toInt == 1) {
+ System.getenv("SCALA_TEST_ON_GPU").toInt == 1 &&
+ !System.getenv().containsKey("CI")) {
val ctx = Context.gpu()
TrainCharRnn.runTrainCharRnn(tempDirPath + "/RNN/obama.txt",
tempDirPath, ctx, 1)
diff --git a/scala-package/init/pom.xml b/scala-package/init/pom.xml
index a0bb6be384b..c514177db17 100644
--- a/scala-package/init/pom.xml
+++ b/scala-package/init/pom.xml
@@ -60,6 +60,13 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
</plugins>
</build>
diff --git a/scala-package/packageTest/README.md b/scala-package/packageTest/README.md
index 3f1eeb842f0..e9980f35375 100644
--- a/scala-package/packageTest/README.md
+++ b/scala-package/packageTest/README.md
@@ -7,7 +7,7 @@ This is an project created to run the test suite on a fully packaged mxnet jar.
To setup the packageTest, you must first build your tests. To build the tests, follow these steps from the mxnet main directory:
1. Build MXNet and the scala package from source following the directions [here](https://mxnet.incubator.apache.org/install/scala_setup.html#source)
-2. Build the tests by running `make scalatestcompile`.
+2. Build the tests by running `mvn test-compile`.
3. Follow setup instructions below for your testing goal
## Running
@@ -18,13 +18,13 @@ There are three different modes of operation for testing based on the location o
If you have a jar file, you can install it to your maven cache repository(`~/.m2/repository`). This might be useful if you acquire the .jar file from elsewhere. To install, it is easiest to use `mvn install:install-file -Dfile=<path-to-file> -DpomFile=<path-to-pomfile>`. If the pom file is not available, you can also run `mvn install:install-file -Dfile=<path-to-file> -DgroupId=<group-id> -DartifactId=<artifact-id> -Dversion=<version> -Dpackaging=<packaging>`. With the full mxnet jar, this might look like `mvn install:install-file -Dfile=<path-to-file> -DgroupId=org.apache.mxnet -DartifactId=mxnet-full_2.11-linux-x86_64-cpu -Dversion=1.3.0 -Dpackaging=jar`.
-You can also run `make scalainstall` to install from a local build.
+You can also run `mvn install` to install from a local build.
After installing, run `make testinstall` in the package test directory to run the tests. Note that unless you also install an additional mxnetexamples jar, you can only run the unit tests.
### Test Local Deployment
-To test the jars that would be produced by a deployment, you can run `make scaladeploylocal` from the main mxnet directory. This produces a local snapshot located at `scala-package/local-snapshot`. To test this local snapshot, run `make testlocal`.
+To test the jars that would be produced by a deployment, you can run `mvn deploy` from the main mxnet directory. This produces a local snapshot located at `scala-package/deploy/target/repo`. To test this local snapshot, run `make testlocal`. It also installs the component packages needed for testing the examples in `scala-package/*/target/repo`.
### Remote Repository Snapshot
@@ -36,11 +36,13 @@ Test the snapshot repo using `make testsnapshot` or a different repo using `make
You are able to run unit tests, integration tests, or both using this utility. To run the unit tests, add the flag `UNIT=1` to make (e.g. `make testsnapshot UNIT=1`). Use `INTEGRATION=1` for integration tests. The default behavior is to run both the unit and integration tests. However, the integration tests require that the mxnet examples be installed in addition to the full mxnet package (see test mode instructions above).
+For running on GPU, add the flag `USE_CUDA=1`.
+
An additional option, you can specify the mxnet version with `MXNET_VERSION=1.3.1-SNAPSHOT`.
## Cleaning Up
-You can clean temporary files and target artifacts by running `make scalaclean`.
+You can clean temporary files and target artifacts by running `make clean`.
## Troubleshooting
diff --git a/scala-package/packageTest/examples/pom.xml b/scala-package/packageTest/examples/pom.xml
index e11be657e22..070b78dda99 100644
--- a/scala-package/packageTest/examples/pom.xml
+++ b/scala-package/packageTest/examples/pom.xml
@@ -21,6 +21,53 @@
<skipTests>false</skipTests>
</properties>
</profile>
+ <profile>
+ <id>fromLocal</id>
+ <repositories>
+ <repository>
+ <id>parent</id>
+ <url>file://${basedir}/../../target/repo</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <id>init</id>
+ <url>file://${basedir}/../../init/target/repo</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <id>macros</id>
+ <url>file://${basedir}/../../macros/target/repo</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <id>core</id>
+ <url>file://${basedir}/../../core/target/repo</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <id>infer</id>
+ <url>file://${basedir}/../../infer/target/repo</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <id>examples</id>
+ <url>file://${basedir}/../../examples/target/repo</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+ </profile>
</profiles>
<build>
@@ -39,8 +86,8 @@
<dependencies>
<dependency>
<groupId>org.apache.mxnet</groupId>
- <artifactId>mxnet-examples_${mxnet.scalaprofile}</artifactId>
- <version>${mxnet.version}</version>
+ <artifactId>mxnet-examples</artifactId>
+ <version>INTERNAL</version>
<scope>test</scope>
</dependency>
</dependencies>
diff --git a/scala-package/packageTest/pom.xml b/scala-package/packageTest/pom.xml
index f7d9e3b180b..f6a16dd77c9 100644
--- a/scala-package/packageTest/pom.xml
+++ b/scala-package/packageTest/pom.xml
@@ -41,7 +41,7 @@
<id>fromLocal</id>
<repositories>
<repository>
- <id>local-snapshot</id>
+ <id>full</id>
<url>file://${basedir}/../deploy/target/repo</url>
<snapshots>
<enabled>true</enabled>
diff --git a/scala-package/pom.xml b/scala-package/pom.xml
index d39075f2c99..07baeab9b63 100644
--- a/scala-package/pom.xml
+++ b/scala-package/pom.xml
@@ -43,6 +43,7 @@
<dollar>$</dollar>
<MXNET_DIR>${project.basedir}/..</MXNET_DIR>
<skipJavaTests>true</skipJavaTests>
+ <repo_url>file://${project.build.directory}/repo</repo_url>
</properties>
<packaging>pom</packaging>
@@ -158,6 +159,34 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>staging</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>nightly</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
<build>
@@ -337,7 +366,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
- <skip>true</skip>
+ <skip>false</skip>
+ <altDeploymentRepository>deployrepo::default::${repo_url}</altDeploymentRepository>
</configuration>
</plugin>
</plugins>
diff --git a/tools/dependencies/cityhash.sh b/tools/dependencies/cityhash.sh
index 81cc9cbaad3..6bc663e906f 100755
--- a/tools/dependencies/cityhash.sh
+++ b/tools/dependencies/cityhash.sh
@@ -18,15 +18,17 @@
# under the License.
# This script builds the static library of cityhash that can be used as dependency of mxnet.
+set -ex
CITYHASH_VERSION=1.1.1
if [[ ! -f $DEPS_PATH/lib/libcityhash.a ]]; then
# Download and build cityhash
>&2 echo "Building cityhash..."
git clone https://github.com/google/cityhash $DEPS_PATH/cityhash-$CITYHASH_VERSION
+ pushd .
cd $DEPS_PATH/cityhash-$CITYHASH_VERSION
git reset --hard 8af9b8c2b889d80c22d6bc26ba0df1afb79a30db
./configure -prefix=$DEPS_PATH --enable-sse4.2
- make CXXFLAGS="-g -O3 -msse4.2"
- make install
- cd -
+ $MAKE CXXFLAGS="-g -O3 -msse4.2"
+ $MAKE install
+ popd
fi
diff --git a/tools/dependencies/curl.sh b/tools/dependencies/curl.sh
index 9633edb7853..bb947715f2a 100755
--- a/tools/dependencies/curl.sh
+++ b/tools/dependencies/curl.sh
@@ -18,12 +18,14 @@
# under the License.
# This script builds the static library of libcurl that can be used as dependency of mxnet.
+set -ex
LIBCURL_VERSION=7.61.0
if [[ ! -f $DEPS_PATH/lib/libcurl.a ]]; then
# download and build libcurl
>&2 echo "Building libcurl..."
curl -s -L https://curl.haxx.se/download/curl-$LIBCURL_VERSION.zip -o $DEPS_PATH/libcurl.zip
unzip -q $DEPS_PATH/libcurl.zip -d $DEPS_PATH
+ pushd .
cd $DEPS_PATH/curl-$LIBCURL_VERSION
if [[ $PLATFORM == 'linux' ]]; then
CONFIG_FLAG=""
@@ -58,7 +60,7 @@ if [[ ! -f $DEPS_PATH/lib/libcurl.a ]]; then
--disable-gopher \
--disable-manual \
--prefix=$DEPS_PATH
- make
- make install
- cd -
+ $MAKE
+ $MAKE install
+ popd
fi
diff --git a/tools/dependencies/eigen.sh b/tools/dependencies/eigen.sh
index ac2f75a03a5..a0cd8fcc95e 100755
--- a/tools/dependencies/eigen.sh
+++ b/tools/dependencies/eigen.sh
@@ -18,6 +18,7 @@
# under the License.
# This script imports the headers from eigen3 that can be used to in opencv.
+set -ex
EIGEN_VERSION=3.3.4
if [[ ! -d $DEPS_PATH/include/eigen3 ]]; then
# download eigen
@@ -25,10 +26,11 @@ if [[ ! -d $DEPS_PATH/include/eigen3 ]]; then
curl -s -L https://github.com/eigenteam/eigen-git-mirror/archive/$EIGEN_VERSION.zip -o $DEPS_PATH/eigen.zip
unzip -q $DEPS_PATH/eigen.zip -d $DEPS_PATH
mkdir -p $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build
+ pushd .
cd $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build
cmake \
-D CMAKE_BUILD_TYPE=RELEASE \
-D CMAKE_INSTALL_PREFIX=$DEPS_PATH ..
- make install
- cd -
+ $MAKE install
+ popd
fi
diff --git a/tools/dependencies/libpng.sh b/tools/dependencies/libpng.sh
index d1523c65447..3faa9f027dc 100755
--- a/tools/dependencies/libpng.sh
+++ b/tools/dependencies/libpng.sh
@@ -18,6 +18,7 @@
# under the License.
# This script builds the static library of libpng that can be used as dependency of mxnet/opencv.
+set -ex
PNG_VERSION=1.6.34
if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then
# download and build libpng
@@ -25,6 +26,7 @@ if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then
curl -s -L https://github.com/glennrp/libpng/archive/v$PNG_VERSION.zip -o $DEPS_PATH/libpng.zip
unzip -q $DEPS_PATH/libpng.zip -d $DEPS_PATH
mkdir -p $DEPS_PATH/libpng-$PNG_VERSION/build
+ pushd .
cd $DEPS_PATH/libpng-$PNG_VERSION/build
cmake \
-D PNG_SHARED=OFF \
@@ -32,9 +34,9 @@ if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then
-D CMAKE_BUILD_TYPE=RELEASE \
-D CMAKE_INSTALL_PREFIX=$DEPS_PATH \
-D CMAKE_C_FLAGS=-fPIC ..
- make
- make install
+ $MAKE
+ $MAKE install
mkdir -p $DEPS_PATH/include/libpng
ln -s $DEPS_PATH/include/png.h $DEPS_PATH/include/libpng/png.h
- cd -
+ popd
fi
diff --git a/tools/dependencies/libtiff.sh b/tools/dependencies/libtiff.sh
index 14dcb2d7bde..2a402bbcac2 100755
--- a/tools/dependencies/libtiff.sh
+++ b/tools/dependencies/libtiff.sh
@@ -18,15 +18,17 @@
# under the License.
# This script builds the static library of libtiff that can be used as dependency of mxnet/opencv.
+set -ex
TIFF_VERSION="4-0-9"
if [[ ! -f $DEPS_PATH/lib/libtiff.a ]]; then
# download and build libtiff
>&2 echo "Building libtiff..."
curl -s -L https://gitlab.com/libtiff/libtiff/-/archive/Release-v$TIFF_VERSION/libtiff-Release-v$TIFF_VERSION.zip -o $DEPS_PATH/libtiff.zip
unzip -q $DEPS_PATH/libtiff.zip -d $DEPS_PATH
+ pushd .
cd $DEPS_PATH/libtiff-Release-v$TIFF_VERSION
./configure --quiet --disable-shared --disable-jpeg --disable-zlib --disable-jbig --disable-lzma --prefix=$DEPS_PATH
- make
- make install
- cd -
+ $MAKE
+ $MAKE install
+ popd
fi
diff --git a/tools/dependencies/libturbojpeg.sh b/tools/dependencies/libturbojpeg.sh
index 4991906f887..ac813ebec1c 100755
--- a/tools/dependencies/libturbojpeg.sh
+++ b/tools/dependencies/libturbojpeg.sh
@@ -19,6 +19,7 @@
# This script builds the static library of libturbojpeg that can be used as dependency of
# mxnet/opencv.
+set -ex
TURBO_JPEG_VERSION=1.5.90
if [[ $PLATFORM == 'darwin' ]]; then
JPEG_NASM_OPTION="-D CMAKE_ASM_NASM_COMPILER=/usr/local/bin/nasm"
@@ -30,6 +31,7 @@ if [[ ! -f $DEPS_PATH/lib/libjpeg.a ]] || [[ ! -f $DEPS_PATH/lib/libturbojpeg.a
curl -s -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/$TURBO_JPEG_VERSION.zip -o $DEPS_PATH/libjpeg.zip
unzip -q $DEPS_PATH/libjpeg.zip -d $DEPS_PATH
mkdir -p $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build
+ pushd .
cd $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build
cmake \
-G"Unix Makefiles" \
@@ -41,7 +43,7 @@ if [[ ! -f $DEPS_PATH/lib/libjpeg.a ]] || [[ ! -f $DEPS_PATH/lib/libturbojpeg.a
-D WITH_JPEG8=TRUE \
$JPEG_NASM_OPTION \
-D ENABLE_SHARED=FALSE ..
- make
- make install
- cd -
+ $MAKE
+ $MAKE install
+ popd
fi
diff --git a/tools/dependencies/libz.sh b/tools/dependencies/libz.sh
index 927f1de82e7..c5f9953c8d1 100755
--- a/tools/dependencies/libz.sh
+++ b/tools/dependencies/libz.sh
@@ -18,6 +18,7 @@
# under the License.
# This script builds the static library of libz that can be used as dependency of mxnet.
+set -ex
ZLIB_VERSION=1.2.6
if [[ ! -f $DEPS_PATH/lib/libz.a ]]; then
# Download and build zlib
@@ -25,12 +26,13 @@ if [[ ! -f $DEPS_PATH/lib/libz.a ]]; then
curl -s -L https://github.com/LuaDist/zlib/archive/$ZLIB_VERSION.zip -o $DEPS_PATH/zlib.zip
unzip -q $DEPS_PATH/zlib.zip -d $DEPS_PATH
mkdir -p $DEPS_PATH/zlib-$ZLIB_VERSION/build
+ pushd .
cd $DEPS_PATH/zlib-$ZLIB_VERSION/build
cmake \
-D CMAKE_BUILD_TYPE=RELEASE \
-D CMAKE_INSTALL_PREFIX=$DEPS_PATH \
-D BUILD_SHARED_LIBS=OFF ..
- make
- make install
- cd -
+ $MAKE
+ $MAKE install
+ popd
fi
diff --git a/tools/dependencies/lz4.sh b/tools/dependencies/lz4.sh
index a4269bf29bb..478c9925e96 100755
--- a/tools/dependencies/lz4.sh
+++ b/tools/dependencies/lz4.sh
@@ -18,14 +18,16 @@
# under the License.
# This script builds the static library of lz4 that can be used as dependency of mxnet.
+set -ex
LZ4_VERSION=r130
if [[ ! -f $DEPS_PATH/lib/liblz4.a ]]; then
# Download and build lz4
>&2 echo "Building lz4..."
curl -s -L https://github.com/lz4/lz4/archive/$LZ4_VERSION.zip -o $DEPS_PATH/lz4.zip
unzip -q $DEPS_PATH/lz4.zip -d $DEPS_PATH
+ pushd .
cd $DEPS_PATH/lz4-$LZ4_VERSION
- make
- make PREFIX=$DEPS_PATH install
- cd -
+ $MAKE
+ $MAKE PREFIX=$DEPS_PATH install
+ popd
fi
diff --git a/tools/dependencies/make_shared_dependencies.sh b/tools/dependencies/make_shared_dependencies.sh
index d678fddcc02..16508586169 100755
--- a/tools/dependencies/make_shared_dependencies.sh
+++ b/tools/dependencies/make_shared_dependencies.sh
@@ -38,3 +38,5 @@ source $DIR/protobuf.sh
source $DIR/cityhash.sh
source $DIR/zmq.sh
source $DIR/lz4.sh
+
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(dirname $(find $DEPS_PATH -type f -name 'libprotoc*' | grep protobuf | head -n 1)):$DEPS_PATH/lib
diff --git a/tools/dependencies/openblas.sh b/tools/dependencies/openblas.sh
index 9463e3325e0..27f473919a5 100755
--- a/tools/dependencies/openblas.sh
+++ b/tools/dependencies/openblas.sh
@@ -18,6 +18,8 @@
# under the License.
# This script builds the static library of openblas that can be used as dependency of mxnet.
+set +e # This script throws an error but otherwise works
+set -x
OPENBLAS_VERSION=0.3.3
if [[ ! -e $DEPS_PATH/lib/libopenblas.a ]]; then
# download and build openblas
@@ -25,11 +27,12 @@ if [[ ! -e $DEPS_PATH/lib/libopenblas.a ]]; then
curl -s -L https://github.com/xianyi/OpenBLAS/archive/v$OPENBLAS_VERSION.zip -o $DEPS_PATH/openblas.zip
unzip -q $DEPS_PATH/openblas.zip -d $DEPS_PATH
+ pushd .
cd $DEPS_PATH/OpenBLAS-$OPENBLAS_VERSION
- make DYNAMIC_ARCH=1 NO_SHARED=1 USE_OPENMP=1
- make PREFIX=$DEPS_PATH install
- cd -
+ $MAKE DYNAMIC_ARCH=1 NO_SHARED=1 USE_OPENMP=1
+ $MAKE PREFIX=$DEPS_PATH install
+ popd
ln -s libopenblas.a $DEPS_PATH/lib/libcblas.a
ln -s libopenblas.a $DEPS_PATH/lib/liblapack.a
fi
diff --git a/tools/dependencies/opencv.sh b/tools/dependencies/opencv.sh
index 99d0ecb71c3..11c9c2155f9 100755
--- a/tools/dependencies/opencv.sh
+++ b/tools/dependencies/opencv.sh
@@ -19,6 +19,7 @@
# This script builds the static library of opencv that can be used as dependency of mxnet.
# It expects openblas, libjpeg, libpng, libtiff, eigen, etc., to be in $DEPS_PATH.
+set -ex
OPENCV_VERSION=3.4.2
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
if [[ $PLATFORM == 'linux' ]]; then
@@ -41,6 +42,7 @@ if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f $DEPS_PATH/lib/libopenc
curl -s -L https://github.com/opencv/opencv/archive/$OPENCV_VERSION.zip -o $DEPS_PATH/opencv.zip
unzip -q $DEPS_PATH/opencv.zip -d $DEPS_PATH
mkdir -p $DEPS_PATH/opencv-$OPENCV_VERSION/build
+ pushd .
cd $DEPS_PATH/opencv-$OPENCV_VERSION/build
cmake \
-D OPENCV_ENABLE_NONFREE=OFF \
@@ -184,9 +186,9 @@ if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f $DEPS_PATH/lib/libopenc
if [[ $PLATFORM == 'linux' ]]; then
cp $DIR/patch/opencv_lapack.h ./
fi
- make
- make install
- cd -
+ $MAKE
+ $MAKE install
+ popd
# @szha: compatibility header
cat $DEPS_PATH/include/opencv2/imgcodecs/imgcodecs_c.h >> $DEPS_PATH/include/opencv2/imgcodecs.hpp
fi
diff --git a/tools/dependencies/openssl.sh b/tools/dependencies/openssl.sh
index b7e4317d4a8..93284db3e39 100755
--- a/tools/dependencies/openssl.sh
+++ b/tools/dependencies/openssl.sh
@@ -18,6 +18,7 @@
# under the License.
# This script builds the static library of openssl that can be used as dependency of mxnet.
+set -ex
OPENSSL_VERSION=1.0.2l
if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a ]]; then
# download and build openssl
@@ -25,6 +26,7 @@ if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a ]];
OPENSSL_VERSION=$(echo $OPENSSL_VERSION | sed 's/\./_/g')
curl -s -L https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.zip -o $DEPS_PATH/openssl.zip
unzip -q $DEPS_PATH/openssl.zip -d $DEPS_PATH
+ pushd .
cd $DEPS_PATH/openssl-OpenSSL_$OPENSSL_VERSION
if [[ $PLATFORM == 'linux' ]]; then
TARGET=linux-x86_64
@@ -32,7 +34,7 @@ if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a ]];
TARGET=darwin64-x86_64-cc
fi
./Configure no-shared no-zlib --prefix=$DEPS_PATH --openssldir=$DEPS_PATH/ssl $TARGET
- make
- make install
- cd -
+ $MAKE
+ $MAKE install
+ popd
fi
diff --git a/tools/dependencies/protobuf.sh b/tools/dependencies/protobuf.sh
index 1564701042a..76ce1de8e82 100755
--- a/tools/dependencies/protobuf.sh
+++ b/tools/dependencies/protobuf.sh
@@ -18,6 +18,7 @@
# under the License.
# This script builds the static library of protobuf along with protoc, that can be used as dependency of mxnet.
+set -ex
PROTOBUF_VERSION=3.5.1
if [[ $PLATFORM == 'darwin' ]]; then
DY_EXT="dylib"
@@ -32,12 +33,12 @@ if [[ ! -e $LIBPROTOBUF ]] || [[ ! -e $LIBPROTOC ]]; then
>&2 echo "Building protobuf..."
curl -s -L https://github.com/google/protobuf/archive/v$PROTOBUF_VERSION.zip -o $DEPS_PATH/protobuf.zip
unzip -q $DEPS_PATH/protobuf.zip -d $DEPS_PATH
+ pushd .
cd $DEPS_PATH/protobuf-$PROTOBUF_VERSION
./autogen.sh
./configure -prefix=$DEPS_PATH
- make
- make install
- cd -
+ $MAKE
+ $MAKE install
+ popd
fi
-export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(dirname $(find $DEPS_PATH -type f -name 'libprotoc*' | grep protobuf | head -n 1)):$DEPS_PATH/lib
diff --git a/tools/dependencies/zmq.sh b/tools/dependencies/zmq.sh
index 55e17798c2d..0042d6bcd07 100755
--- a/tools/dependencies/zmq.sh
+++ b/tools/dependencies/zmq.sh
@@ -18,6 +18,7 @@
# under the License.
# This script builds the static library of zeroMQ that can be used as dependency of mxnet.
+set -ex
ZEROMQ_VERSION=4.2.2
if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then
# Download and build zmq
@@ -25,14 +26,14 @@ if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then
curl -s -L https://github.com/zeromq/libzmq/archive/v$ZEROMQ_VERSION.zip -o $DEPS_PATH/zeromq.zip
unzip -q $DEPS_PATH/zeromq.zip -d $DEPS_PATH
mkdir -p $DEPS_PATH/libzmq-$ZEROMQ_VERSION/build
+ pushd .
cd $DEPS_PATH/libzmq-$ZEROMQ_VERSION/build
cmake \
-D CMAKE_BUILD_TYPE=RELEASE \
-D CMAKE_INSTALL_PREFIX=$DEPS_PATH \
-D WITH_LIBSODIUM=OFF \
-D BUILD_SHARED_LIBS=OFF ..
- make
- make install
- cp $DEPS_PATH/lib/x86_64-linux-gnu/libzmq.a $DEPS_PATH/lib/libzmq.a
- cd -
+ $MAKE
+ $MAKE install
+ popd
fi
diff --git a/tools/setup_gpu_build_tools.sh b/tools/setup_gpu_build_tools.sh
index 44b44c57411..167d4c6a6e1 100755
--- a/tools/setup_gpu_build_tools.sh
+++ b/tools/setup_gpu_build_tools.sh
@@ -246,6 +246,8 @@ if [[ ! -d $DEPS_PATH/usr/local/cuda-${CUDA_MAJOR_VERSION} ]]; then
rm package.deb
done
+ mkdir -p ${prefix}/include
+ mkdir -p ${prefix}/lib
cp ${prefix}/usr/include/x86_64-linux-gnu/cudnn_v${LIBCUDNN_MAJOR}.h ${prefix}/include/cudnn.h
ln -s libcudnn_static_v${LIBCUDNN_MAJOR}.a ${prefix}/usr/lib/x86_64-linux-gnu/libcudnn.a
cp ${prefix}/usr/local/cuda-${CUDA_MAJOR_VERSION}/lib64/*.a ${prefix}/lib/
diff --git a/tools/staticbuild/build.sh b/tools/staticbuild/build.sh
new file mode 100755
index 00000000000..4f8a78d7049
--- /dev/null
+++ b/tools/staticbuild/build.sh
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if [ $# -lt 1 ]; then
+ >&2 echo "Usage: build.sh <VARIANT> <TARGET>"
+fi
+
+export CURDIR=$PWD
+export DEPS_PATH=$PWD/staticdeps
+export VARIANT=$(echo $1 | tr '[:upper:]' '[:lower:]')
+export STATIC_BUILD_TARGET=$(echo $2 | tr '[:upper:]' '[:lower:]')
+export PLATFORM=$(uname | tr '[:upper:]' '[:lower:]')
+
+if [[ $VARIANT == darwin* ]]; then
+ export VARIANT="darwin"
+fi
+
+NUM_PROC=1
+if [[ ! -z $(command -v nproc) ]]; then
+ NUM_PROC=$(nproc)
+elif [[ ! -z $(command -v sysctl) ]]; then
+ NUM_PROC=$(sysctl -n hw.ncpu)
+else
+ >&2 echo "Can't discover number of cores."
+fi
+export NUM_PROC
+>&2 echo "Using $NUM_PROC parallel jobs in building."
+
+if [[ $DEBUG -eq 1 ]]; then
+ export ADD_MAKE_FLAG="-j $NUM_PROC"
+else
+ export ADD_MAKE_FLAG="--quiet -j $NUM_PROC"
+fi
+export MAKE="make $ADD_MAKE_FLAG"
+
+export CC="gcc -fPIC"
+export CXX="g++ -fPIC"
+export FC="gfortran"
+export PKG_CONFIG_PATH=$DEPS_PATH/lib/pkgconfig:$DEPS_PATH/lib64/pkgconfig:$DEPS_PATH/lib/x86_64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH
+export CPATH=$DEPS_PATH/include:$CPATH
+
+if [[ $PLATFORM == 'linux' && $VARIANT == cu* ]]; then
+ source tools/setup_gpu_build_tools.sh $VARIANT $DEPS_PATH
+fi
+
+mkdir -p $DEPS_PATH
+
+# Build Dependencies
+source tools/dependencies/make_shared_dependencies.sh
+
+# Build mxnet
+source tools/staticbuild/build_lib.sh
diff --git a/tools/build/build_lib.sh b/tools/staticbuild/build_lib.sh
similarity index 77%
rename from tools/build/build_lib.sh
rename to tools/staticbuild/build_lib.sh
index 032fcb92045..dc6bd4e8b76 100755
--- a/tools/build/build_lib.sh
+++ b/tools/staticbuild/build_lib.sh
@@ -18,22 +18,20 @@
# under the License.
# This script builds the libraries of mxnet.
-make_config=config/pip_${PLATFORM}_${VARIANT}.mk
+make_config=make/${STATIC_BUILD_TARGET}/${STATIC_BUILD_TARGET}_${PLATFORM}_${VARIANT}.mk
if [[ ! -f $make_config ]]; then
>&2 echo "Couldn't find make config $make_config for the current settings."
exit 1
fi
-git clone --recursive https://github.com/apache/incubator-mxnet mxnet-build
-
>&2 echo "Now building mxnet modules..."
-cp $make_config mxnet-build/config.mk
+cp $make_config config.mk
-cd mxnet-build
+git submodule update --init --recursive || true
-make DEPS_PATH=$DEPS_PATH DMLCCORE
-make DEPS_PATH=$DEPS_PATH $PWD/3rdparty/tvm/nnvm/lib/libnnvm.a
-make DEPS_PATH=$DEPS_PATH PSLITE
+$MAKE DEPS_PATH=$DEPS_PATH DMLCCORE
+$MAKE DEPS_PATH=$DEPS_PATH $PWD/3rdparty/tvm/nnvm/lib/libnnvm.a
+$MAKE DEPS_PATH=$DEPS_PATH PSLITE
if [[ $VARIANT == *mkl ]]; then
MKLDNN_LICENSE='license.txt'
@@ -46,7 +44,11 @@ if [[ $VARIANT == *mkl ]]; then
MKLML_LIBFILE='libmklml.dylib'
MKLDNN_LIBFILE='libmkldnn.0.dylib'
fi
- make DEPS_PATH=$DEPS_PATH mkldnn
+ $MAKE DEPS_PATH=$DEPS_PATH mkldnn
+ cp 3rdparty/mkldnn/build/install/lib/$IOMP_LIBFILE lib
+ cp 3rdparty/mkldnn/build/install/lib/$MKLML_LIBFILE lib
+ cp 3rdparty/mkldnn/build/install/lib/$MKLDNN_LIBFILE lib
+ cp 3rdparty/mkldnn/build/install/$MKLDNN_LICENSE lib
cp 3rdparty/mkldnn/LICENSE ./MKLML_LICENSE
fi
@@ -58,7 +60,7 @@ if [[ $VARIANT == *mkl ]]; then
fi
>&2 echo "Now building mxnet..."
-make DEPS_PATH=$DEPS_PATH || exit 1;
+$MAKE DEPS_PATH=$DEPS_PATH
if [[ $PLATFORM == 'linux' ]]; then
cp -L /usr/lib/gcc/x86_64-linux-gnu/4.8/libgfortran.so lib/libgfortran.so.3
@@ -77,4 +79,4 @@ else
>&2 echo "Not available"
fi
-cd ../
+ln -s staticdeps/ deps
diff --git a/tools/build/build_wheel.sh b/tools/staticbuild/build_wheel.sh
similarity index 100%
rename from tools/build/build_wheel.sh
rename to tools/staticbuild/build_wheel.sh
With regards,
Apache Git Services