You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by za...@apache.org on 2019/05/09 18:24:18 UTC
[incubator-mxnet] branch v1.4.x updated: [1.4.x] Adds additional
build envs (#14920)
This is an automated email from the ASF dual-hosted git repository.
zachgk pushed a commit to branch v1.4.x
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/v1.4.x by this push:
new b9ea0fa [1.4.x] Adds additional build envs (#14920)
b9ea0fa is described below
commit b9ea0fa11a0549b14854d35665f79f77bb55704d
Author: perdasilva <pe...@gmail.com>
AuthorDate: Thu May 9 20:23:35 2019 +0200
[1.4.x] Adds additional build envs (#14920)
* Adds CUDA build environment for version 8.0, 9.0, 9.2 and 10.0
* Renames ubuntu_gpu environment to ubuntu_gpu_cu91 and updates references
* Adds static build image and updates ci/build.py
---
Jenkinsfile | 36 +++++------
ci/build.py | 33 ++++++----
...buntu_gpu => Dockerfile.build.ubuntu_gpu_cu100} | 7 ++-
...ubuntu_gpu => Dockerfile.build.ubuntu_gpu_cu80} | 2 +-
...ubuntu_gpu => Dockerfile.build.ubuntu_gpu_cu90} | 7 ++-
...ubuntu_gpu => Dockerfile.build.ubuntu_gpu_cu91} | 0
...ubuntu_gpu => Dockerfile.build.ubuntu_gpu_cu92} | 7 ++-
ci/docker/Dockerfile.publish.ubuntu1404_cpu | 36 +++++++++++
ci/docker/install/ubuntu_cudnn.sh | 54 +++++++++++++++++
ci/docker/install/ubuntu_publish.sh | 70 ++++++++++++++++++++++
ci/jenkins/Jenkins_steps.groovy | 34 +++++------
dev_menu.py | 4 +-
12 files changed, 238 insertions(+), 52 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 03ad7d8..17265a6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -360,7 +360,7 @@ core_logic: {
ws('workspace/build-cmake-mkldnn-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
- utils.docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake_mkldnn', false)
+ utils.docker_run('ubuntu_gpu_cu91', 'build_ubuntu_gpu_cmake_mkldnn', false)
utils.pack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib, true)
}
}
@@ -371,7 +371,7 @@ core_logic: {
ws('workspace/build-cmake-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
- utils.docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake', false)
+ utils.docker_run('ubuntu_gpu_cu91', 'build_ubuntu_gpu_cmake', false)
utils.pack_lib('cmake_gpu', mx_cmake_lib, true)
}
}
@@ -542,7 +542,7 @@ core_logic: {
ws('workspace/ut-python2-gpu') {
try {
utils.unpack_and_init('gpu', mx_lib, true)
- python2_gpu_ut('ubuntu_gpu')
+ python2_gpu_ut('ubuntu_gpu_cu91')
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_gpu.xml')
@@ -555,7 +555,7 @@ core_logic: {
ws('workspace/ut-python3-gpu') {
try {
utils.unpack_and_init('gpu', mx_lib, true)
- python3_gpu_ut('ubuntu_gpu')
+ python3_gpu_ut('ubuntu_gpu_cu91')
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_gpu.xml')
@@ -569,7 +569,7 @@ core_logic: {
timeout(time: max_time, unit: 'MINUTES') {
try {
utils.unpack_and_init('gpu', mx_lib, true)
- utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_python2_quantization_gpu', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'unittest_ubuntu_python2_quantization_gpu', true)
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_quantization_gpu.xml', 'nosetests_python2_quantize_gpu.xml')
@@ -584,7 +584,7 @@ core_logic: {
timeout(time: max_time, unit: 'MINUTES') {
try {
utils.unpack_and_init('gpu', mx_lib, true)
- utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_python3_quantization_gpu', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'unittest_ubuntu_python3_quantization_gpu', true)
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_quantization_gpu.xml', 'nosetests_python3_quantize_gpu.xml')
@@ -613,7 +613,7 @@ core_logic: {
ws('workspace/ut-python2-mkldnn-gpu') {
try {
utils.unpack_and_init('mkldnn_gpu', mx_mkldnn_lib, true)
- python2_gpu_ut('ubuntu_gpu')
+ python2_gpu_ut('ubuntu_gpu_cu91')
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_mkldnn_gpu.xml')
@@ -640,7 +640,7 @@ core_logic: {
ws('workspace/ut-python3-mkldnn-gpu') {
try {
utils.unpack_and_init('mkldnn_gpu', mx_mkldnn_lib, true)
- python3_gpu_ut('ubuntu_gpu')
+ python3_gpu_ut('ubuntu_gpu_cu91')
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu.xml')
@@ -653,7 +653,7 @@ core_logic: {
ws('workspace/ut-python3-mkldnn-gpu-nocudnn') {
try {
utils.unpack_and_init('mkldnn_gpu_nocudnn', mx_mkldnn_lib, true)
- python3_gpu_ut_nocudnn('ubuntu_gpu')
+ python3_gpu_ut_nocudnn('ubuntu_gpu_cu91')
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu_nocudnn.xml')
@@ -756,7 +756,7 @@ core_logic: {
ws('workspace/ut-perl-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_lib, true)
- utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_cpugpu_perl', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'unittest_ubuntu_cpugpu_perl', true)
utils.publish_test_coverage()
}
}
@@ -767,7 +767,7 @@ core_logic: {
ws('workspace/ut-cpp-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('cmake_gpu', mx_cmake_lib, true)
- utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'unittest_ubuntu_gpu_cpp', true)
utils.publish_test_coverage()
}
}
@@ -778,7 +778,7 @@ core_logic: {
ws('workspace/ut-cpp-mkldnn-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib, true)
- utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'unittest_ubuntu_gpu_cpp', true)
utils.publish_test_coverage()
}
}
@@ -800,7 +800,7 @@ core_logic: {
ws('workspace/ut-r-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_lib, true)
- utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_R', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'unittest_ubuntu_gpu_R', true)
utils.publish_test_coverage()
}
}
@@ -911,7 +911,7 @@ core_logic: {
ws('workspace/it-python-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_lib, true)
- utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_python', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'integrationtest_ubuntu_gpu_python', true)
utils.publish_test_coverage()
}
}
@@ -922,7 +922,7 @@ core_logic: {
ws('workspace/it-cpp-package') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_lib_cpp_examples, true)
- utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_cpp_package', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'integrationtest_ubuntu_gpu_cpp_package', true)
utils.publish_test_coverage()
}
}
@@ -935,7 +935,7 @@ core_logic: {
// timeout(time: max_time, unit: 'MINUTES') {
// utils.init_git()
// utils.unpack_lib('gpu', mx_lib)
- // utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_caffe', true)
+ // utils.docker_run('ubuntu_gpu_cu91', 'integrationtest_ubuntu_gpu_caffe', true)
// utils.publish_test_coverage()
// }
// }
@@ -946,7 +946,7 @@ core_logic: {
ws('workspace/it-dist-kvstore') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_lib, true)
- utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_dist_kvstore', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'integrationtest_ubuntu_gpu_dist_kvstore', true)
utils.publish_test_coverage()
}
}
@@ -972,7 +972,7 @@ core_logic: {
ws('workspace/ut-scala-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_dist_lib, true)
- utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_scala', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'integrationtest_ubuntu_gpu_scala', true)
utils.publish_test_coverage()
}
}
diff --git a/ci/build.py b/ci/build.py
index acf277d..1c7a4f8 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -92,22 +92,24 @@ def get_dockerfiles_path():
def get_platforms(path: str = get_dockerfiles_path()) -> List[str]:
"""Get a list of architectures given our dockerfiles"""
- dockerfiles = glob.glob(os.path.join(path, "Dockerfile.build.*"))
+ dockerfiles = glob.glob(os.path.join(path, "Dockerfile.*"))
dockerfiles = list(filter(lambda x: x[-1] != '~', dockerfiles))
- files = list(map(lambda x: re.sub(r"Dockerfile.build.(.*)", r"\1", x), dockerfiles))
+ files = list(map(lambda x: re.sub(r"Dockerfile.(.*)", r"\1", x), dockerfiles))
platforms = list(map(lambda x: os.path.split(x)[1], sorted(files)))
return platforms
def get_docker_tag(platform: str, registry: str) -> str:
""":return: docker tag to be used for the container"""
+ platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform)
if not registry:
registry = "mxnet_local"
- return "{0}/build.{1}".format(registry, platform)
+ return "{0}/{1}".format(registry, platform)
def get_dockerfile(platform: str, path=get_dockerfiles_path()) -> str:
- return os.path.join(path, "Dockerfile.build.{0}".format(platform))
+ platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform)
+ return os.path.join(path, "Dockerfile.{0}".format(platform))
def get_docker_binary(use_nvidia_docker: bool) -> str:
@@ -200,7 +202,7 @@ def default_ccache_dir() -> str:
ccache_dir = "/tmp/_mxnet_ccache"
os.makedirs(ccache_dir, exist_ok=True)
return ccache_dir
- return os.path.join(tempfile.gettempdir(), "ci_ccache")
+ return os.path.join(os.path.expanduser("~"), ".ccache")
def trim_container_id(cid):
@@ -215,20 +217,21 @@ def container_run(platform: str,
local_ccache_dir: str,
command: List[str],
cleanup: Cleanup,
+ environment: Dict[str, str],
dry_run: bool = False) -> int:
"""Run command in a container"""
container_wait_s = 600
#
# Environment setup
#
- environment = {
+ environment.update({
'CCACHE_MAXSIZE': '500G',
'CCACHE_TEMPDIR': '/tmp/ccache', # temp dir should be local and not shared
'CCACHE_DIR': '/work/ccache', # this path is inside the container as /work/ccache is
# mounted
'CCACHE_LOGFILE': '/tmp/ccache.log', # a container-scoped log, useful for ccache
# verification.
- }
+ })
# These variables are passed to the container to the process tree killer can find runaway
# process inside the container
# https://wiki.jenkins.io/display/JENKINS/ProcessTreeKiller
@@ -446,6 +449,10 @@ def main() -> int:
parser.add_argument("--no-cache", action="store_true",
help="passes --no-cache to docker build")
+ parser.add_argument("-e", "--environment", nargs="*", default=[],
+ help="Environment variables for the docker container. "
+ "Specify with a list containing either names or name=value")
+
parser.add_argument("command",
help="command to run in the container",
nargs='*', action='append', type=str)
@@ -474,6 +481,9 @@ def main() -> int:
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
+ environment = dict([(e.split('=')[:2] if '=' in e else (e, os.environ[e]))
+ for e in args.environment])
+
if args.list:
print(list_platforms())
elif args.platform:
@@ -493,13 +503,13 @@ def main() -> int:
ret = container_run(
platform=platform, nvidia_runtime=args.nvidiadocker,
shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
- local_ccache_dir=args.ccache_dir, cleanup=cleanup)
+ local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment)
elif args.print_docker_run:
command = []
ret = container_run(
platform=platform, nvidia_runtime=args.nvidiadocker,
shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
- local_ccache_dir=args.ccache_dir, dry_run=True, cleanup=cleanup)
+ local_ccache_dir=args.ccache_dir, dry_run=True, cleanup=cleanup, environment=environment)
else:
# With no commands, execute a build function for the target platform
command = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)]
@@ -507,7 +517,7 @@ def main() -> int:
ret = container_run(
platform=platform, nvidia_runtime=args.nvidiadocker,
shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
- local_ccache_dir=args.ccache_dir, cleanup=cleanup)
+ local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment)
if ret != 0:
logging.critical("Execution of %s failed with status: %d", command, ret)
@@ -515,6 +525,7 @@ def main() -> int:
elif args.all:
platforms = get_platforms()
+ platforms = [platform for platform in platforms if 'build.' in platform]
logging.info("Building for all architectures: %s", platforms)
logging.info("Artifacts will be produced in the build/ directory.")
for platform in platforms:
@@ -535,7 +546,7 @@ def main() -> int:
container_run(
platform=platform, nvidia_runtime=args.nvidiadocker,
shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
- local_ccache_dir=args.ccache_dir, cleanup=cleanup)
+ local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment)
shutil.move(buildir(), plat_buildir)
logging.info("Built files left in: %s", plat_buildir)
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu_cu100
similarity index 94%
copy from ci/docker/Dockerfile.build.ubuntu_gpu
copy to ci/docker/Dockerfile.build.ubuntu_gpu_cu100
index 8fcbcbb..63fd597 100644
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_cu100
@@ -18,7 +18,9 @@
#
# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
-FROM nvidia/cuda:9.1-cudnn7-devel
+FROM nvidia/cuda:10.0-devel-ubuntu16.04
+
+ENV CUDNN_VERSION=7.3.1.20
WORKDIR /work/deps
@@ -72,6 +74,9 @@ ARG GROUP_ID=0
COPY install/ubuntu_adduser.sh /work/
RUN /work/ubuntu_adduser.sh
+COPY install/ubuntu_cudnn.sh /work/
+RUN /work/ubuntu_cudnn.sh
+
COPY runtime_functions.sh /work/
WORKDIR /work/mxnet
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu_cu80
similarity index 97%
copy from ci/docker/Dockerfile.build.ubuntu_gpu
copy to ci/docker/Dockerfile.build.ubuntu_gpu_cu80
index 8fcbcbb..9c7a808 100644
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_cu80
@@ -18,7 +18,7 @@
#
# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
-FROM nvidia/cuda:9.1-cudnn7-devel
+FROM nvidia/cuda:8.0-cudnn7-devel-ubuntu16.04
WORKDIR /work/deps
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu_cu90
similarity index 94%
copy from ci/docker/Dockerfile.build.ubuntu_gpu
copy to ci/docker/Dockerfile.build.ubuntu_gpu_cu90
index 8fcbcbb..f1e6570 100644
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_cu90
@@ -18,7 +18,9 @@
#
# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
-FROM nvidia/cuda:9.1-cudnn7-devel
+FROM nvidia/cuda:9.0-devel-ubuntu16.04
+
+ENV CUDNN_VERSION=7.3.1.20
WORKDIR /work/deps
@@ -72,6 +74,9 @@ ARG GROUP_ID=0
COPY install/ubuntu_adduser.sh /work/
RUN /work/ubuntu_adduser.sh
+COPY install/ubuntu_cudnn.sh /work/
+RUN /work/ubuntu_cudnn.sh
+
COPY runtime_functions.sh /work/
WORKDIR /work/mxnet
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu_cu91
similarity index 100%
copy from ci/docker/Dockerfile.build.ubuntu_gpu
copy to ci/docker/Dockerfile.build.ubuntu_gpu_cu91
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu_cu92
similarity index 94%
rename from ci/docker/Dockerfile.build.ubuntu_gpu
rename to ci/docker/Dockerfile.build.ubuntu_gpu_cu92
index 8fcbcbb..81b337e 100644
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_cu92
@@ -18,7 +18,9 @@
#
# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
-FROM nvidia/cuda:9.1-cudnn7-devel
+FROM nvidia/cuda:9.2-devel-ubuntu16.04
+
+ENV CUDNN_VERSION=7.3.1.20
WORKDIR /work/deps
@@ -72,6 +74,9 @@ ARG GROUP_ID=0
COPY install/ubuntu_adduser.sh /work/
RUN /work/ubuntu_adduser.sh
+COPY install/ubuntu_cudnn.sh /work/
+RUN /work/ubuntu_cudnn.sh
+
COPY runtime_functions.sh /work/
WORKDIR /work/mxnet
diff --git a/ci/docker/Dockerfile.publish.ubuntu1404_cpu b/ci/docker/Dockerfile.publish.ubuntu1404_cpu
new file mode 100644
index 0000000..04ce94f
--- /dev/null
+++ b/ci/docker/Dockerfile.publish.ubuntu1404_cpu
@@ -0,0 +1,36 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build and run MXNet on Ubuntu 14.04 for CPU
+
+FROM ubuntu:14.04
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_publish.sh /work/
+RUN /work/ubuntu_publish.sh
+
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY runtime_functions.sh /work/
+
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/install/ubuntu_cudnn.sh b/ci/docker/install/ubuntu_cudnn.sh
new file mode 100755
index 0000000..3f64907
--- /dev/null
+++ b/ci/docker/install/ubuntu_cudnn.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+# Assumes base image is from nvidia/cuda
+
+set -ex
+
+if [ -z ${CUDNN_VERSION} ]; then
+ echo "Error: CUDNN_VERSION environment variable undefiend"
+ exit 1
+fi
+
+apt-get update || true
+
+case ${CUDA_VERSION} in
+ 10\.0*)
+ export libcudnn7_version="${CUDNN_VERSION}-1+cuda10.0"
+ export libcudnn7_dev_version="${CUDNN_VERSION}-1+cuda10.0"
+ ;;
+ 9\.0*)
+ export libcudnn7_version="${CUDNN_VERSION}-1+cuda9.0"
+ export libcudnn7_dev_version="${CUDNN_VERSION}-1+cuda9.0"
+ ;;
+ 9\.2*)
+ export libcudnn7_version="${CUDNN_VERSION}-1+cuda9.2"
+ export libcudnn7_dev_version="${CUDNN_VERSION}-1+cuda9.2"
+ ;;
+ *)
+ echo "Unsupported CUDA version ${CUDA_VERSION}"
+ exit 1
+ ;;
+esac
+
+apt-get install -y libcudnn7=${libcudnn7_version} libcudnn7-dev=${libcudnn7_dev_version}
+
diff --git a/ci/docker/install/ubuntu_publish.sh b/ci/docker/install/ubuntu_publish.sh
new file mode 100755
index 0000000..2d8b019
--- /dev/null
+++ b/ci/docker/install/ubuntu_publish.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Build on Ubuntu 14.04 LTS for LINUX CPU/GPU
+set -ex
+
+apt-get update
+apt-get install -y software-properties-common
+add-apt-repository ppa:ubuntu-toolchain-r/test -y
+add-apt-repository ppa:openjdk-r/ppa -y # Java lib
+apt-get update
+apt-get install -y git \
+ cmake3 \
+ libcurl4-openssl-dev \
+ unzip \
+ gcc-4.8 \
+ g++-4.8 \
+ gfortran \
+ gfortran-4.8 \
+ binutils \
+ nasm \
+ libtool \
+ curl \
+ wget \
+ sudo \
+ gnupg \
+ gnupg2 \
+ gnupg-agent \
+ pandoc \
+ python3-pip \
+ automake \
+ pkg-config \
+ openjdk-8-jdk
+
+curl -o apache-maven-3.3.9-bin.tar.gz http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz \
+ || curl -o apache-maven-3.3.9-bin.tar.gz https://search.maven.org/remotecontent?filepath=org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.tar.gz
+
+tar xzf apache-maven-3.3.9-bin.tar.gz
+mkdir /usr/local/maven
+mv apache-maven-3.3.9/ /usr/local/maven/
+update-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1
+update-ca-certificates -f
+
+apt-get install -y python python3
+
+# the version of the pip shipped with ubuntu may be too lower, install a recent version here
+wget -nv https://bootstrap.pypa.io/get-pip.py
+python3 get-pip.py
+python2 get-pip.py
+
+apt-get remove -y python3-urllib3
+
+pip2 install nose cpplint==1.3.0 'numpy<=1.15.2,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
+pip3 install nose cpplint==1.3.0 pylint==2.3.1 'numpy<=1.15.2,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index 73c6029..00615c2 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -183,7 +183,7 @@ def compile_unix_cmake_mkldnn_gpu() {
ws('workspace/build-cmake-mkldnn-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
- utils.docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake_mkldnn', false)
+ utils.docker_run('ubuntu_gpu_cu91', 'build_ubuntu_gpu_cmake_mkldnn', false)
utils.pack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib, true)
}
}
@@ -197,7 +197,7 @@ def compile_unix_cmake_gpu() {
ws('workspace/build-cmake-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
- utils.docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake', false)
+ utils.docker_run('ubuntu_gpu_cu91', 'build_ubuntu_gpu_cmake', false)
utils.pack_lib('cmake_gpu', mx_cmake_lib, true)
}
}
@@ -513,7 +513,7 @@ def test_unix_python2_gpu() {
ws('workspace/ut-python2-gpu') {
try {
utils.unpack_and_init('gpu', mx_lib, true)
- python2_gpu_ut('ubuntu_gpu')
+ python2_gpu_ut('ubuntu_gpu_cu91')
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_gpu.xml')
@@ -530,7 +530,7 @@ def test_unix_python2_quantize_gpu() {
timeout(time: max_time, unit: 'MINUTES') {
try {
utils.unpack_and_init('gpu', mx_lib, true)
- utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_python2_quantization_gpu', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'unittest_ubuntu_python2_quantization_gpu', true)
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_quantization_gpu.xml', 'nosetests_python2_quantize_gpu.xml')
@@ -547,7 +547,7 @@ def test_unix_python2_mkldnn_gpu() {
ws('workspace/ut-python2-mkldnn-gpu') {
try {
utils.unpack_and_init('mkldnn_gpu', mx_mkldnn_lib, true)
- python2_gpu_ut('ubuntu_gpu')
+ python2_gpu_ut('ubuntu_gpu_cu91')
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_mkldnn_gpu.xml')
@@ -580,7 +580,7 @@ def test_unix_python3_gpu() {
ws('workspace/ut-python3-gpu') {
try {
utils.unpack_and_init('gpu', mx_lib, true)
- python3_gpu_ut('ubuntu_gpu')
+ python3_gpu_ut('ubuntu_gpu_cu91')
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_gpu.xml')
@@ -597,7 +597,7 @@ def test_unix_python3_quantize_gpu() {
timeout(time: max_time, unit: 'MINUTES') {
try {
utils.unpack_and_init('gpu', mx_lib, true)
- utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_python3_quantization_gpu', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'unittest_ubuntu_python3_quantization_gpu', true)
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_quantization_gpu.xml', 'nosetests_python3_quantize_gpu.xml')
@@ -665,7 +665,7 @@ def test_unix_python3_mkldnn_gpu() {
ws('workspace/ut-python3-mkldnn-gpu') {
try {
utils.unpack_and_init('mkldnn_gpu', mx_mkldnn_lib, true)
- python3_gpu_ut('ubuntu_gpu')
+ python3_gpu_ut('ubuntu_gpu_cu91')
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu.xml')
@@ -681,7 +681,7 @@ def test_unix_python3_mkldnn_nocudnn_gpu() {
ws('workspace/ut-python3-mkldnn-gpu-nocudnn') {
try {
utils.unpack_and_init('mkldnn_gpu_nocudnn', mx_mkldnn_lib, true)
- python3_gpu_ut_nocudnn('ubuntu_gpu')
+ python3_gpu_ut_nocudnn('ubuntu_gpu_cu91')
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu_nocudnn.xml')
@@ -715,7 +715,7 @@ def test_unix_python3_integration_gpu() {
ws('workspace/it-python-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_lib, true)
- utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_python', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'integrationtest_ubuntu_gpu_python', true)
utils.publish_test_coverage()
}
}
@@ -730,7 +730,7 @@ def test_unix_caffe_gpu() {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
utils.unpack_lib('gpu', mx_lib)
- utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_caffe', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'integrationtest_ubuntu_gpu_caffe', true)
utils.publish_test_coverage()
}
}
@@ -744,7 +744,7 @@ def test_unix_cpp_package_gpu() {
ws('workspace/it-cpp-package') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_lib_cpp_examples, true)
- utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_cpp_package', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'integrationtest_ubuntu_gpu_cpp_package', true)
utils.publish_test_coverage()
}
}
@@ -772,7 +772,7 @@ def test_unix_scala_gpu() {
ws('workspace/ut-scala-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_dist_lib, true)
- utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_scala', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'integrationtest_ubuntu_gpu_scala', true)
utils.publish_test_coverage()
}
}
@@ -814,7 +814,7 @@ def test_unix_cpp_gpu() {
ws('workspace/ut-cpp-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('cmake_gpu', mx_cmake_lib, true)
- utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'unittest_ubuntu_gpu_cpp', true)
utils.publish_test_coverage()
}
}
@@ -828,7 +828,7 @@ def test_unix_cpp_mkldnn_gpu() {
ws('workspace/ut-cpp-mkldnn-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib, true)
- utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'unittest_ubuntu_gpu_cpp', true)
utils.publish_test_coverage()
}
}
@@ -842,7 +842,7 @@ def test_unix_r_gpu() {
ws('workspace/ut-perl-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_lib, true)
- utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_cpugpu_perl', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'unittest_ubuntu_cpugpu_perl', true)
utils.publish_test_coverage()
}
}
@@ -897,7 +897,7 @@ def test_unix_distributed_kvstore_gpu() {
ws('workspace/it-dist-kvstore') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_lib, true)
- utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_dist_kvstore', true)
+ utils.docker_run('ubuntu_gpu_cu91', 'integrationtest_ubuntu_gpu_dist_kvstore', true)
utils.publish_test_coverage()
}
}
diff --git a/dev_menu.py b/dev_menu.py
index 27db9e8..448a2e3 100755
--- a/dev_menu.py
+++ b/dev_menu.py
@@ -100,8 +100,8 @@ COMMANDS = OrderedDict([
]),
('[Docker] Python3 GPU unittests',
[
- "ci/build.py --platform ubuntu_gpu /work/runtime_functions.sh build_ubuntu_gpu",
- "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_python3_gpu",
+ "ci/build.py --platform ubuntu_gpu_cu91 /work/runtime_functions.sh build_ubuntu_gpu",
+ "ci/build.py --nvidiadocker --platform ubuntu_gpu_cu91 /work/runtime_functions.sh unittest_ubuntu_python3_gpu",
]),
('[Docker] Python3 CPU Intel MKLDNN unittests',
[