You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by ma...@apache.org on 2019/01/18 00:25:29 UTC
[incubator-mxnet] branch master updated: Static build instruction
for MXNet in general (#13914)
This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 34a2dc8 Static build instruction for MXNet in general (#13914)
34a2dc8 is described below
commit 34a2dc85c052c5ea2ac17cf4fdb6cb97cd1efea8
Author: Lanking <la...@live.com>
AuthorDate: Thu Jan 17 19:25:06 2019 -0500
Static build instruction for MXNet in general (#13914)
* update scripts and tutorial
* add the static test for scala package
* kill publish test
* fix build issue
* address comments
---
ci/docker/runtime_functions.sh | 10 +++++++
ci/jenkins/Jenkins_steps.groovy | 13 +++++++++
ci/jenkins/Jenkinsfile_unix_cpu | 1 +
ci/publish/Jenkinsfile | 4 +--
ci/publish/README.md | 33 ++++++++++++++++++++++
.../{maven_darwin_cpu.mk => maven_darwin_mkl.mk} | 4 +--
...{maven_linux_cu90.mk => maven_linux_cu90mkl.mk} | 4 +--
...{maven_linux_cu92.mk => maven_linux_cu92mkl.mk} | 4 +--
.../{maven_linux_cpu.mk => maven_linux_mkl.mk} | 4 +--
tools/dependencies/README.md | 11 ++++++++
tools/staticbuild/README.md | 32 +++++++++++++++++++++
tools/staticbuild/build_lib.sh | 7 -----
12 files changed, 110 insertions(+), 17 deletions(-)
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 8c079c2..7b50ae3 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -1285,6 +1285,16 @@ deploy_jl_docs() {
# ...
}
+build_scala_static_mkl() {
+ set -ex
+ pushd .
+ scala_prepare
+ export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu
+ export mxnet_variant=mkl
+ ./ci/publish/scala/build.sh
+ popd
+}
+
publish_scala_build() {
set -ex
pushd .
diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index 7e84539..8b0e83e 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -521,6 +521,19 @@ def compile_windows_gpu_mkldnn() {
}]
}
+def test_static_scala_cpu() {
+ return ['Static build CPU 14.04 Scala' : {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/ut-publish-scala-cpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.init_git()
+ utils.docker_run("publish.ubuntu1404_cpu", 'build_scala_static_mkl', false)
+ }
+ }
+ }
+ }]
+}
+
def test_unix_python2_cpu() {
return ['Python2: CPU': {
node(NODE_LINUX_CPU) {
diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu
index ea3c061..2fe4576 100644
--- a/ci/jenkins/Jenkinsfile_unix_cpu
+++ b/ci/jenkins/Jenkinsfile_unix_cpu
@@ -57,6 +57,7 @@ core_logic: {
custom_steps.test_unix_julia10_cpu(),
custom_steps.test_unix_onnx_cpu(),
custom_steps.test_unix_cpp_cpu(),
+ custom_steps.test_static_scala_cpu(),
/* Disabled due to master build failure:
* http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/
* https://github.com/apache/incubator-mxnet/issues/11801
diff --git a/ci/publish/Jenkinsfile b/ci/publish/Jenkinsfile
index c14ab25..2b91f4f 100644
--- a/ci/publish/Jenkinsfile
+++ b/ci/publish/Jenkinsfile
@@ -36,7 +36,7 @@ utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-m
// CPU and GPU. OSX nodes are not currently supported by Jenkins
def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU_P3]
def scalaOSMap = ['cpu': 'linux-x86_64-cpu', 'gpu': 'linux-x86_64-gpu']
-def scalaVariantMap = ['cpu': 'cpu', 'gpu': 'cu92']
+def scalaVariantMap = ['cpu': 'mkl', 'gpu': 'cu92mkl']
def wrapStep(nodeToRun, workspaceName, step) {
return {
@@ -84,7 +84,7 @@ for (x in labels) {
toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap['cpu'], "deploy-scala-${label}") {
withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}", "mxnet_variant=${scalaVariantMap[label]}"]) {
utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
- utils.docker_run("publish.ubuntu1604_cpu", 'publish_scala_deploy', false, '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION mxnet_variant')
+ utils.docker_run("publish.ubuntu1604_${label}", 'publish_scala_deploy', false, '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION mxnet_variant')
}
}
}
diff --git a/ci/publish/README.md b/ci/publish/README.md
new file mode 100644
index 0000000..2344cdf
--- /dev/null
+++ b/ci/publish/README.md
@@ -0,0 +1,33 @@
+# MXNet Publish Settings
+
+This folder contains the configuration of restricted node on Jenkins for the publish. It also contains a folder called `scala` that contains everything required for scala publish. In this `README`, we would bring a brief walkthrough of the Jenkins configuration as well as the usages of the scala deployment files.
+
+## Jenkins
+Currently, Jenkins contains three build stages, namely `Build Packages`, `Test Packages` and `Deploy Packages`. During the `build package` stages, all dependencies will be built and a Scala package would be created. In the second stage, the package created from the previous stage would move to this stage to specifically run the tests. In the final stage, the packages passed the test would be deployed by the instances.
+
+The job is scheduled to be triggered every 24 hours on a [restricted instance](http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/restricted-publish-artifacts).
+
+Currently, we are supporting tests in the following systems:
+
+- Ubuntu 16.04
+- Ubuntu 18.04
+- Cent OS 7
+
+All packages are currently built in `Ubuntu 14.04`. All Dockerfile used for publishing are available in `ci/docker/` with prefix `Dockerfile.publish`.
+
+Apart from that, the script used to create the environment and publish are available under `ci/docker/install`:
+
+- `ubuntu_publish.sh` install all required dependencies for Ubuntu 14.04 for publishing
+- `ubuntu_base.sh` install minimum dependencies required to run the published packages
+
+## Scala publish
+Currently Scala publish on Linux is fully supported on jenkins. The `scala/` folder contains all files needed to do the publish. Here is a breif instroduction of the files:
+
+- `build.sh` Main executable files to build the backend as well as scala package
+- `buildkey.py` Main file used to extract password from the system and configure the maven
+- `deploy.sh` Script to deploy the package
+- `fullDeploy.sh` Used by CI to make full publish
+- `test.sh` Make Scala test on CI
+
+## Python
+We plans to support Python build on Jenkins soon
\ No newline at end of file
diff --git a/make/maven/maven_darwin_cpu.mk b/make/maven/maven_darwin_mkl.mk
similarity index 99%
rename from make/maven/maven_darwin_cpu.mk
rename to make/maven/maven_darwin_mkl.mk
index b8bd97f..f5b77ae 100644
--- a/make/maven/maven_darwin_cpu.mk
+++ b/make/maven/maven_darwin_mkl.mk
@@ -37,7 +37,7 @@ DEBUG = 0
USE_PROFILER = 1
# whether to turn on signal handler (e.g. segfault logger)
-USE_SIGNAL_HANDLER = 0
+USE_SIGNAL_HANDLER = 1
# the additional link flags you want to add
ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -lz -framework CoreFoundation -framework Security -Wl,-exported_symbols_list,$(CURDIR)/make/config/libmxnet.sym,-rpath,'$${ORIGIN}',-dead_strip
@@ -87,7 +87,7 @@ USE_LIBJPEG_TURBO = 1
# Please refer to MKL_README.md for details
# whether use MKL-DNN library
-USE_MKLDNN = 0
+USE_MKLDNN = 1
# MKL ML Library folder, need to be root for /usr/local
# Change to User Home directory for standard user
diff --git a/make/maven/maven_linux_cu90.mk b/make/maven/maven_linux_cu90mkl.mk
similarity index 99%
rename from make/maven/maven_linux_cu90.mk
rename to make/maven/maven_linux_cu90mkl.mk
index 097d252..661f444 100644
--- a/make/maven/maven_linux_cu90.mk
+++ b/make/maven/maven_linux_cu90mkl.mk
@@ -37,7 +37,7 @@ DEBUG = 0
USE_PROFILER = 1
# whether to turn on signal handler (e.g. segfault logger)
-USE_SIGNAL_HANDLER = 0
+USE_SIGNAL_HANDLER = 1
# the additional link flags you want to add
ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections
@@ -86,7 +86,7 @@ USE_OPERATOR_TUNING = 1
USE_LIBJPEG_TURBO = 1
# whether use MKL-DNN library
-USE_MKLDNN = 0
+USE_MKLDNN = 1
# MKL ML Library for Intel CPU/Xeon Phi
diff --git a/make/maven/maven_linux_cu92.mk b/make/maven/maven_linux_cu92mkl.mk
similarity index 99%
rename from make/maven/maven_linux_cu92.mk
rename to make/maven/maven_linux_cu92mkl.mk
index 0ed18a7..ecd2527 100644
--- a/make/maven/maven_linux_cu92.mk
+++ b/make/maven/maven_linux_cu92mkl.mk
@@ -37,7 +37,7 @@ DEBUG = 0
USE_PROFILER = 1
# whether to turn on signal handler (e.g. segfault logger)
-USE_SIGNAL_HANDLER = 0
+USE_SIGNAL_HANDLER = 1
# the additional link flags you want to add
ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections
@@ -86,7 +86,7 @@ USE_OPERATOR_TUNING = 1
USE_LIBJPEG_TURBO = 1
# whether use MKL-DNN library
-USE_MKLDNN = 0
+USE_MKLDNN = 1
# MKL ML Library for Intel CPU/Xeon Phi
diff --git a/make/maven/maven_linux_cpu.mk b/make/maven/maven_linux_mkl.mk
similarity index 99%
rename from make/maven/maven_linux_cpu.mk
rename to make/maven/maven_linux_mkl.mk
index 07177fb..6cb9f32 100644
--- a/make/maven/maven_linux_cpu.mk
+++ b/make/maven/maven_linux_mkl.mk
@@ -37,7 +37,7 @@ DEBUG = 0
USE_PROFILER = 1
# whether to turn on signal handler (e.g. segfault logger)
-USE_SIGNAL_HANDLER = 0
+USE_SIGNAL_HANDLER = 1
# the additional link flags you want to add
ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -lgfortran -ldl -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections
@@ -83,7 +83,7 @@ USE_OPERATOR_TUNING = 1
USE_LIBJPEG_TURBO = 1
# whether use MKL-DNN library
-USE_MKLDNN = 0
+USE_MKLDNN = 1
# MKL ML Library for Intel CPU/Xeon Phi
diff --git a/tools/dependencies/README.md b/tools/dependencies/README.md
index cfe3d6c..c30e85d 100644
--- a/tools/dependencies/README.md
+++ b/tools/dependencies/README.md
@@ -12,3 +12,14 @@ The scripts use the following environment variables for setting behavior:
`PLATFORM`: name of the OS in lower case. Supported options are 'linux' and 'darwin'.
It also expects the following build tools in path: make, cmake, tar, unzip, autoconf, nasm
+
+# FAQ
+
+## Build failure regarding to gcc, g++, gfortran
+Currently, we only support gcc-4.8 build. It's your own choice to use a higher version of gcc. Please make sure your gcc, g++ and gfortran always have the same version in order to eliminate build failure.
+
+## idn2 not found
+This issue appeared in the OSX build with XCode version 8.0 above (reproduced on 9.2). Please add the following build flag in `curl.sh` if your XCode version is more than 8.0:
+```
+--without-libidn2
+```
\ No newline at end of file
diff --git a/tools/staticbuild/README.md b/tools/staticbuild/README.md
new file mode 100644
index 0000000..7cfeb78
--- /dev/null
+++ b/tools/staticbuild/README.md
@@ -0,0 +1,32 @@
+# MXNet Static build
+
+This folder contains the core script used to build the static library. This README would bring you the information and usages of the script in here. Please be aware, all of the scripts are designed to be run under the root folder.
+
+## `build.sh`
+This script is a wrapper around `build_lib.sh` aimed to simplify the usage of it. It would automatically identify the system version, number of cores and all environment variable settings. Here are the examples you can run this script:
+
+```
+tools/staticbuild/build.sh cu92 maven
+```
+This would build the mxnet package based on CUDA9.2 and Maven (Scala) build setttings.
+```
+tools/staticbuild/build.sh mkl pip
+```
+This would build the mxnet package based on MKLDNN and and pypi configuration settings.
+
+As the result, users would have a complete static dependencies in `/staticdeps` in the root folder as well as a static-linked `libmxnet.so` file lives in `lib`. You can build your language binding by using the `libmxnet.so`.
+
+## `build_lib.sh`
+This script would clone the most up-to-date master and build the MXNet backend with static library. In order to run that, you should have prepare the the following environment variable:
+
+- `DEPS_PATH` Path to your static dependencies
+- `STATIC_BUILD_TARGET` Either `pip` or `maven` as your publish platform
+- `PLATFORM` linux, darwin
+- `VARIANT` cpu, cu*, cu*mkl, mkl
+
+It is not recommended to run this file alone since there are a bunch of variables need to be set.
+
+After running this script, you would have everything you need ready in the `/lib` folder.
+
+## `build_wheel.sh`
+This script is used to build the python package as well as running a sanity test
\ No newline at end of file
diff --git a/tools/staticbuild/build_lib.sh b/tools/staticbuild/build_lib.sh
index dc6bd4e..b080573 100755
--- a/tools/staticbuild/build_lib.sh
+++ b/tools/staticbuild/build_lib.sh
@@ -52,13 +52,6 @@ if [[ $VARIANT == *mkl ]]; then
cp 3rdparty/mkldnn/LICENSE ./MKLML_LICENSE
fi
-if [[ $VARIANT == *mkl ]]; then
- >&2 echo "Copying MKL license."
- rm lib/libmkldnn.{so,dylib}
- rm lib/libmkldnn.0.*.dylib
- rm lib/libmkldnn.so.0.*
-fi
-
>&2 echo "Now building mxnet..."
$MAKE DEPS_PATH=$DEPS_PATH