You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by ma...@apache.org on 2018/03/21 13:30:19 UTC
[incubator-mxnet] branch master updated: [MXNET-99] Upgrade CI to
cuda 9.1 cudnn 7 (#10108)
This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new b0a6760 [MXNET-99] Upgrade CI to cuda 9.1 cudnn 7 (#10108)
b0a6760 is described below
commit b0a6760efa141aeca87b03ecf34dae924bd1af46
Author: Marco de Abreu <ma...@users.noreply.github.com>
AuthorDate: Wed Mar 21 14:30:06 2018 +0100
[MXNET-99] Upgrade CI to cuda 9.1 cudnn 7 (#10108)
* Upgrade to cuda 9.1 cudnn 7
* Pin pylint version
* Remove aws cli installation
* Address review comments
---
Jenkinsfile | 5 +++--
ci/docker/Dockerfile.build.centos7_gpu | 2 +-
ci/docker/Dockerfile.build.ubuntu_build_cuda | 6 ++----
ci/docker/Dockerfile.build.ubuntu_cpu | 2 --
ci/docker/Dockerfile.build.ubuntu_gpu | 4 +---
ci/docker/install/ubuntu_core.sh | 4 +---
ci/docker/install/ubuntu_lint.sh | 25 ------------------------
ci/docker/install/ubuntu_nvidia.sh | 5 +----
ci/docker/install/ubuntu_python.sh | 4 ++--
ci/docker/runtime_functions.sh | 2 +-
python/mxnet/kvstore.py | 4 ++--
python/mxnet/model.py | 2 +-
python/mxnet/ndarray/ndarray.py | 2 +-
python/mxnet/symbol/register.py | 2 +-
python/mxnet/symbol/symbol.py | 1 +
python/mxnet/torch.py | 20 ++++++++++---------
tests/ci_build/deploy/aws | 29 ----------------------------
tests/ci_build/deploy/ci_deploy_doc.sh | 7 ++-----
18 files changed, 31 insertions(+), 95 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 73e73f2..134a4df 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -211,11 +211,11 @@ try {
}
}
},
- 'GPU: CUDA8.0+cuDNN5': {
+ 'GPU: CUDA9.1+cuDNN7': {
node('mxnetlinux-cpu') {
ws('workspace/build-gpu') {
init_git()
- sh "ci/build.py --build --platform ubuntu_build_cuda /work/runtime_functions.sh build_ubuntu_gpu_cuda8_cudnn5"
+ sh "ci/build.py --build --platform ubuntu_build_cuda /work/runtime_functions.sh build_ubuntu_gpu_cuda91_cudnn7"
pack_lib('gpu')
stash includes: 'build/cpp-package/example/test_score', name: 'cpp_test_score'
stash includes: 'build/cpp-package/example/test_optimizer', name: 'cpp_test_optimizer'
@@ -637,6 +637,7 @@ try {
}
}
}
+
// set build status to success at the end
currentBuild.result = "SUCCESS"
} catch (caughtError) {
diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/docker/Dockerfile.build.centos7_gpu
index 3d74821..4dcf5bf 100755
--- a/ci/docker/Dockerfile.build.centos7_gpu
+++ b/ci/docker/Dockerfile.build.centos7_gpu
@@ -18,7 +18,7 @@
#
# Dockerfile to build and run MXNet on CentOS 7 for GPU
-FROM nvidia/cuda:8.0-cudnn5-devel-centos7
+FROM nvidia/cuda:9.1-cudnn7-devel-centos7
ARG USER_ID=0
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda
index 18c8af7..8bafed4 100755
--- a/ci/docker/Dockerfile.build.ubuntu_build_cuda
+++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda
@@ -21,7 +21,7 @@
# package generation, requiring the actual CUDA library to be
# present
-FROM nvidia/cuda:8.0-cudnn5-devel
+FROM nvidia/cuda:9.1-cudnn7-devel
ARG USER_ID=0
@@ -37,8 +37,6 @@ COPY install/ubuntu_r.sh /work/
RUN /work/ubuntu_r.sh
COPY install/ubuntu_perl.sh /work/
RUN /work/ubuntu_perl.sh
-COPY install/ubuntu_lint.sh /work/
-RUN /work/ubuntu_lint.sh
COPY install/ubuntu_clang.sh /work/
RUN /work/ubuntu_clang.sh
COPY install/ubuntu_mklml.sh /work/
@@ -54,4 +52,4 @@ RUN /work/ubuntu_nvidia.sh
COPY runtime_functions.sh /work/
WORKDIR /work/mxnet
-ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
\ No newline at end of file
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu
index d652a0d..f706f88 100755
--- a/ci/docker/Dockerfile.build.ubuntu_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu
@@ -34,8 +34,6 @@ COPY install/ubuntu_r.sh /work/
RUN /work/ubuntu_r.sh
COPY install/ubuntu_perl.sh /work/
RUN /work/ubuntu_perl.sh
-COPY install/ubuntu_lint.sh /work/
-RUN /work/ubuntu_lint.sh
COPY install/ubuntu_clang.sh /work/
RUN /work/ubuntu_clang.sh
COPY install/ubuntu_mklml.sh /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu
index 826836c..625d570 100755
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu
@@ -18,7 +18,7 @@
#
# Dockerfile to run MXNet on Ubuntu 16.04 for CPU
-FROM nvidia/cuda:8.0-cudnn5-devel
+FROM nvidia/cuda:9.1-cudnn7-devel
ARG USER_ID=0
@@ -34,8 +34,6 @@ COPY install/ubuntu_r.sh /work/
RUN /work/ubuntu_r.sh
COPY install/ubuntu_perl.sh /work/
RUN /work/ubuntu_perl.sh
-COPY install/ubuntu_lint.sh /work/
-RUN /work/ubuntu_lint.sh
COPY install/ubuntu_clang.sh /work/
RUN /work/ubuntu_clang.sh
COPY install/ubuntu_mklml.sh /work/
diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh
index e78f29a..dc9b091 100755
--- a/ci/docker/install/ubuntu_core.sh
+++ b/ci/docker/install/ubuntu_core.sh
@@ -34,9 +34,7 @@ apt-get install -y \
unzip \
sudo \
software-properties-common \
- ninja-build \
- python-pip
+ ninja-build
# Link Openblas to Cblas as this link does not exist on ubuntu16.04
ln -s /usr/lib/libopenblas.so /usr/lib/libcblas.so
-pip install cpplint==1.3.0 pylint==1.8.2
\ No newline at end of file
diff --git a/ci/docker/install/ubuntu_lint.sh b/ci/docker/install/ubuntu_lint.sh
deleted file mode 100755
index f3a1d6b..0000000
--- a/ci/docker/install/ubuntu_lint.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-apt-get update
-apt-get install -y python-pip sudo
\ No newline at end of file
diff --git a/ci/docker/install/ubuntu_nvidia.sh b/ci/docker/install/ubuntu_nvidia.sh
index bb1c73e..e6d7926 100755
--- a/ci/docker/install/ubuntu_nvidia.sh
+++ b/ci/docker/install/ubuntu_nvidia.sh
@@ -17,13 +17,10 @@
# specific language governing permissions and limitations
# under the License.
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
set -ex
apt install -y software-properties-common
add-apt-repository -y ppa:graphics-drivers
# Retrieve ppa:graphics-drivers and install nvidia-drivers.
# Note: DEBIAN_FRONTEND required to skip the interactive setup steps
apt update
-DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends cuda-8-0
\ No newline at end of file
+DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends cuda-9-1
diff --git a/ci/docker/install/ubuntu_python.sh b/ci/docker/install/ubuntu_python.sh
index b19448e..b906b55 100755
--- a/ci/docker/install/ubuntu_python.sh
+++ b/ci/docker/install/ubuntu_python.sh
@@ -29,5 +29,5 @@ wget -nv https://bootstrap.pypa.io/get-pip.py
python3 get-pip.py
python2 get-pip.py
-pip2 install nose pylint numpy nose-timer requests h5py scipy
-pip3 install nose pylint numpy nose-timer requests h5py scipy
\ No newline at end of file
+pip2 install nose cpplint==1.3.0 pylint==1.8.3 numpy nose-timer requests h5py scipy
+pip3 install nose cpplint==1.3.0 pylint==1.8.3 numpy nose-timer requests h5py scipy
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 39809f2..f12772e 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -256,7 +256,7 @@ build_ubuntu_gpu_mkldnn() {
-j$(nproc)
}
-build_ubuntu_gpu_cuda8_cudnn5() {
+build_ubuntu_gpu_cuda91_cudnn7() {
set -ex
make \
DEV=1 \
diff --git a/python/mxnet/kvstore.py b/python/mxnet/kvstore.py
index 221b94f..5520597 100644
--- a/python/mxnet/kvstore.py
+++ b/python/mxnet/kvstore.py
@@ -424,7 +424,7 @@ class KVStore(object):
Other keys in this dictionary are optional and specific to the type
of gradient compression.
"""
- if ('device' in self.type) or ('dist' in self.type):
+ if ('device' in self.type) or ('dist' in self.type): # pylint: disable=unsupported-membership-test
ckeys, cvals = _ctype_dict(compression_params)
check_call(_LIB.MXKVStoreSetGradientCompression(self.handle,
mx_uint(len(compression_params)),
@@ -466,7 +466,7 @@ class KVStore(object):
check_call(_LIB.MXKVStoreIsWorkerNode(ctypes.byref(is_worker)))
# pylint: disable=invalid-name
- if 'dist' in self.type and is_worker.value:
+ if 'dist' in self.type and is_worker.value: # pylint: disable=unsupported-membership-test
# send the optimizer to server
try:
# use ASCII protocol 0, might be slower, but not a big ideal
diff --git a/python/mxnet/model.py b/python/mxnet/model.py
index 33dae17..c66b17c 100644
--- a/python/mxnet/model.py
+++ b/python/mxnet/model.py
@@ -848,7 +848,7 @@ class FeedForward(BASE_ESTIMATOR):
# init optmizer
if isinstance(self.optimizer, str):
batch_size = data.batch_size
- if kvstore and 'dist' in kvstore.type and not '_async' in kvstore.type:
+ if kvstore and 'dist' in kvstore.type and '_async' not in kvstore.type:
batch_size *= kvstore.num_workers
optimizer = opt.create(self.optimizer,
rescale_grad=(1.0/batch_size),
diff --git a/python/mxnet/ndarray/ndarray.py b/python/mxnet/ndarray/ndarray.py
index 5367845..7dc2acf 100644
--- a/python/mxnet/ndarray/ndarray.py
+++ b/python/mxnet/ndarray/ndarray.py
@@ -1683,7 +1683,7 @@ fixed-size items.
pdata = ctypes.POINTER(mx_uint)()
check_call(_LIB.MXNDArrayGetShape(
self.handle, ctypes.byref(ndim), ctypes.byref(pdata)))
- return tuple(pdata[:ndim.value])
+ return tuple(pdata[:ndim.value]) # pylint: disable=invalid-slice-index
@property
diff --git a/python/mxnet/symbol/register.py b/python/mxnet/symbol/register.py
index 7b293f5..6f9e868 100644
--- a/python/mxnet/symbol/register.py
+++ b/python/mxnet/symbol/register.py
@@ -126,7 +126,7 @@ def %s(*%s, **kwargs):"""%(func_name, arr_name))
else:
keys.append(k)
vals.append(v)"""%(func_name.lower()))
- if key_var_num_args:
+ if key_var_num_args: # pylint: disable=using-constant-test
code.append("""
if '%s' not in kwargs:
keys.append('%s')
diff --git a/python/mxnet/symbol/symbol.py b/python/mxnet/symbol/symbol.py
index 59402e6..6d5c8c3 100644
--- a/python/mxnet/symbol/symbol.py
+++ b/python/mxnet/symbol/symbol.py
@@ -750,6 +750,7 @@ class Symbol(SymbolBase):
self.handle, ctypes.byref(size), ctypes.byref(sarr)))
return [py_str(sarr[i]) for i in range(size.value)]
+ # pylint: disable=invalid-length-returned
def __len__(self):
"""Get number of outputs for the symbol.
diff --git a/python/mxnet/torch.py b/python/mxnet/torch.py
index fc815b1..dde1eed 100644
--- a/python/mxnet/torch.py
+++ b/python/mxnet/torch.py
@@ -142,18 +142,20 @@ def _make_torch_function(handle):
for k in kwargs:
kwargs[k] = str(kwargs[k])
- check_call(_LIB.MXFuncInvokeEx( \
- handle, \
- c_handle_array(ndargs[n_mutate_vars:]), \
- c_array(mx_float, []), \
- c_handle_array(ndargs[:n_mutate_vars]),
- ctypes.c_int(len(kwargs)),
- c_str_array(kwargs.keys()),
- c_str_array(kwargs.values())))
+ check_call(_LIB.MXFuncInvokeEx(
+ handle,
+ c_handle_array(ndargs[n_mutate_vars:]), # pylint: disable=invalid-slice-index
+ c_array(mx_float, []),
+ c_handle_array(ndargs[:n_mutate_vars]), # pylint: disable=invalid-slice-index
+ ctypes.c_int(len(kwargs)),
+ c_str_array(kwargs.keys()),
+ c_str_array(kwargs.values())))
+
if n_mutate_vars == 1:
return ndargs[0]
else:
- return ndargs[:n_mutate_vars]
+ return ndargs[:n_mutate_vars] # pylint: disable=invalid-slice-index
+
# End of function declaration
ret_function = generic_torch_function
ret_function.__name__ = func_name[4:]
diff --git a/tests/ci_build/deploy/aws b/tests/ci_build/deploy/aws
deleted file mode 100755
index 810a0b9..0000000
--- a/tests/ci_build/deploy/aws
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env python
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-import sys
-import os
-
-if os.environ.get('LC_CTYPE', '') == 'UTF-8':
- os.environ['LC_CTYPE'] = 'en_US.UTF-8'
-import awscli.clidriver
-
-main = awscli.clidriver.main
-
-if __name__ == '__main__':
- sys.exit(main())
diff --git a/tests/ci_build/deploy/ci_deploy_doc.sh b/tests/ci_build/deploy/ci_deploy_doc.sh
index 44c8192..a300794 100755
--- a/tests/ci_build/deploy/ci_deploy_doc.sh
+++ b/tests/ci_build/deploy/ci_deploy_doc.sh
@@ -26,10 +26,7 @@
#
# BUILD_ID: the current build ID for the specified PR
#
+set -ex
-# TODO szha@: installation of awscli here should be removed once slave hosts have them during
-# bootstrap. The following line along with the "aws" script should both be removed then.
-pip install --user awscli
-
-tests/ci_build/deploy/aws s3 sync --delete docs/_build/html/ s3://mxnet-ci-doc/$1/$2 \
+aws s3 sync --delete docs/_build/html/ s3://mxnet-ci-doc/$1/$2 \
&& echo "Doc is hosted at http://mxnet-ci-doc.s3-accelerate.dualstack.amazonaws.com/$1/$2/index.html"
--
To stop receiving notification emails like this one, please contact
marcoabreu@apache.org.