You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by io...@apache.org on 2019/04/05 15:23:48 UTC
[incubator-dlab] 01/01: [GCP]: Fix Data Engine creation fails on
DeepLearning -remove old and unsupported library Caffe -change versionc of
cuda for Data Engine on deeplearning
This is an automated email from the ASF dual-hosted git repository.
ioleksandr pushed a commit to branch DLAB-523
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit d27782dd503223810edd458edc565760e544d50c
Author: Oleksandr_Isniuk <i....@gmail.com>
AuthorDate: Fri Apr 5 18:23:34 2019 +0300
[GCP]: Fix Data Engine creation fails on DeepLearning
-remove old and unsupported library Caffe
-change versionc of cuda for Data Engine on deeplearning
---
.../src/dataengine/scripts/configure_dataengine.py | 8 +++-
.../scripts/configure_deep_learning_node.py | 2 -
.../src/general/lib/os/debian/notebook_lib.py | 39 ------------------
.../src/general/lib/os/redhat/notebook_lib.py | 46 ----------------------
4 files changed, 6 insertions(+), 89 deletions(-)
diff --git a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
index 9a4fb29..e5aacb8 100644
--- a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
+++ b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
@@ -73,6 +73,12 @@ files_dir = '/root/files/'
local_spark_path = '/opt/spark/'
jars_dir = '/opt/jars/'
r_libs = ['R6', 'pbdZMQ', 'RCurl', 'devtools', 'reshape2', 'caTools', 'rJava', 'ggplot2']
+if os.environ['application'] == 'deeplearning':
+ tensorflow_version = '1.4.0'
+ cuda_version = '8.0'
+ cuda_file_name = 'cuda_8.0.44_linux-run'
+ cudnn_version = '6.0'
+ cudnn_file_name = 'cudnn-8.0-linux-x64-v6.0.tgz'
def start_spark(os_user, master_ip, node):
@@ -165,8 +171,6 @@ if __name__ == "__main__":
# INSTALL DEEP LEARNING FRAMEWORKS
if os.environ['application'] == 'deeplearning':
- print("Installing Caffe")
- install_caffe(args.os_user, args.region, caffe_version)
print("Installing Caffe2")
install_caffe2(args.os_user, caffe2_version, cmake_version)
print("Installing Torch")
diff --git a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
index ec188ed..3903bbb 100644
--- a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
+++ b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
@@ -121,8 +121,6 @@ if __name__ == "__main__":
install_theano(args.os_user, theano_version)
print("Installing Keras")
install_keras(args.os_user, keras_version)
- print("Installing Caffe")
- install_caffe(args.os_user, args.region, caffe_version)
print("Installing Caffe2")
install_caffe2(args.os_user, caffe2_version, cmake_version)
print("Installing Torch")
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index ecb5f02..91da82d 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -405,45 +405,6 @@ def get_available_os_pkgs():
sys.exit(1)
-def install_caffe(os_user, region, caffe_version):
- if not exists('/home/{}/.ensure_dir/caffe_ensured'.format(os_user)):
- env.shell = "/bin/bash -l -c -i"
- sudo('apt-get install -y python-dev')
- sudo('apt-get install -y python3-dev')
- sudo('apt-get install -y libprotobuf-dev libleveldb-dev libsnappy-dev libopencv-dev libhdf5-serial-dev protobuf-compiler')
- sudo('apt-get install -y --no-install-recommends libboost-all-dev')
- sudo('apt-get install -y libatlas-base-dev libopenblas-dev')
- sudo('apt-get install -y libgflags-dev libgoogle-glog-dev liblmdb-dev')
- with cd('/usr/lib/x86_64-linux-gnu/'):
- sudo('ln -s libhdf5_serial_hl.so.10.0.2 libhdf5_hl.so')
- sudo('ln -s libhdf5_serial.so.10.1.0 libhdf5.so')
- sudo('git clone https://github.com/BVLC/caffe.git')
- with cd('/home/{}/caffe/'.format(os_user)):
- sudo('git checkout {}'.format(caffe_version))
- sudo('pip2 install -r python/requirements.txt --no-cache-dir')
- sudo('pip3 install -r python/requirements.txt --no-cache-dir')
- sudo('echo "CUDA_DIR := /usr/local/cuda" > Makefile.config')
- cuda_arch = sudo("/opt/cuda-8.0/extras/demo_suite/deviceQuery | grep 'CUDA Capability' | tr -d ' ' | cut -f2 -d ':'")
- sudo('echo "CUDA_ARCH := -gencode arch=compute_{0},code=sm_{0}" >> Makefile.config'.format(cuda_arch.replace('.', '')))
- sudo('echo "PYTHON_INCLUDE := /usr/include/python2.7 /usr/local/lib/python2.7/dist-packages/numpy/core/include" >> Makefile.config')
- sudo('echo "BLAS := open" >> Makefile.config')
- sudo('echo "BLAS_INCLUDE := /usr/include/openblas" >> Makefile.config')
- #sudo('echo "OPENCV_VERSION := 3" >> Makefile.config')
- sudo('echo "LIBRARIES += glog gflags protobuf boost_system boost_filesystem m hdf5_serial_hl hdf5_serial" >> Makefile.config')
- sudo('echo "PYTHON_LIB := /usr/lib" >> Makefile.config')
- sudo('echo "INCLUDE_DIRS := \\\$(PYTHON_INCLUDE) /usr/local/include /usr/include/hdf5/serial/ /usr /usr/lib /usr/include/python2.7 /usr/local/lib/python2.7/dist-packages/numpy/core/include" >> Makefile.config')
- sudo('echo "LIBRARY_DIRS := \\\$(PYTHON_LIB) /usr/local/lib /usr/lib /usr /usr/lib" >> Makefile.config')
- sudo('echo "BUILD_DIR := build" >> Makefile.config')
- sudo('echo "DISTRIBUTE_DIR := distribute" >> Makefile.config')
- sudo('echo "TEST_GPUID := 0" >> Makefile.config')
- sudo('echo "Q ?= @" >> Makefile.config')
- sudo('make all -j$(nproc)')
- sudo('make test -j$(nproc)')
- run('make runtest')
- sudo('make pycaffe')
- sudo('touch /home/' + os_user + '/.ensure_dir/caffe_ensured')
-
-
def install_caffe2(os_user, caffe2_version, cmake_version):
if not exists('/home/{}/.ensure_dir/caffe2_ensured'.format(os_user)):
env.shell = "/bin/bash -l -c -i"
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
index d03710d..a103d26 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
@@ -402,52 +402,6 @@ def install_opencv(os_user):
sudo('touch /home/' + os_user + '/.ensure_dir/opencv_ensured')
-def install_caffe(os_user, region, caffe_version):
- if not exists('/home/{}/.ensure_dir/caffe_ensured'.format(os_user)):
- env.shell = "/bin/bash -l -c -i"
- install_opencv(os_user)
- with cd('/etc/yum.repos.d/'):
- if region == 'cn-north-1':
- mirror = 'mirror.lzu.edu.cn'
- else:
- mirror = 'mirror.centos.org'
- sudo('echo "[centosrepo]" > centos.repo')
- sudo('echo "name=Centos 7 Repository" >> centos.repo')
- sudo('echo "baseurl=http://{}/centos/7/os/x86_64/" >> centos.repo'.format(mirror))
- sudo('echo "enabled=1" >> centos.repo')
- sudo('echo "gpgcheck=1" >> centos.repo')
- sudo('echo "gpgkey=http://{}/centos/7/os/x86_64/RPM-GPG-KEY-CentOS-7" >> centos.repo'.format(mirror))
- sudo('yum update-minimal --security -y')
- sudo('yum install -y --nogpgcheck protobuf-devel leveldb-devel snappy-devel boost-devel hdf5-devel gcc gcc-c++')
- sudo('yum install -y gflags-devel glog-devel lmdb-devel yum-utils && package-cleanup --cleandupes')
- sudo('yum install -y openblas-devel gflags-devel glog-devel lmdb-devel')
- sudo('git clone https://github.com/BVLC/caffe.git')
- with cd('/home/{}/caffe/'.format(os_user)):
- sudo('git checkout {}'.format(caffe_version))
- sudo('pip2 install -r python/requirements.txt --no-cache-dir')
- sudo('pip3.5 install -r python/requirements.txt --no-cache-dir')
- sudo('echo "CUDA_DIR := /usr/local/cuda" > Makefile.config')
- cuda_arch = sudo("/opt/cuda-8.0/extras/demo_suite/deviceQuery | grep 'CUDA Capability' | tr -d ' ' | cut -f2 -d ':'")
- sudo('echo "CUDA_ARCH := -gencode arch=compute_{0},code=sm_{0}" >> Makefile.config'.format(cuda_arch.replace('.', '')))
- sudo('echo "PYTHON_INCLUDE := /usr/include/python2.7 /usr/lib64/python2.7/site-packages/numpy/core/include" >> Makefile.config')
- sudo('echo "BLAS := open" >> Makefile.config')
- sudo('echo "BLAS_INCLUDE := /usr/include/openblas" >> Makefile.config')
- sudo('echo "OPENCV_VERSION := 3" >> Makefile.config')
- sudo('echo "LIBRARIES += glog gflags protobuf boost_system boost_filesystem m hdf5_serial_hl hdf5_serial" >> Makefile.config')
- sudo('echo "PYTHON_LIB := /usr/lib" >> Makefile.config')
- sudo('echo "INCLUDE_DIRS := \\\$(PYTHON_INCLUDE) /usr/local/include /usr /usr/lib64 /usr/include/python2.7 /usr/lib64/python2.7/site-packages/numpy/core/include" >> Makefile.config')
- sudo('echo "LIBRARY_DIRS := \\\$(PYTHON_LIB) /usr/local/lib /usr/lib /usr /usr/lib64" >> Makefile.config')
- sudo('echo "BUILD_DIR := build" >> Makefile.config')
- sudo('echo "DISTRIBUTE_DIR := distribute" >> Makefile.config')
- sudo('echo "TEST_GPUID := 0" >> Makefile.config')
- sudo('echo "Q ?= @" >> Makefile.config')
- sudo('make all -j$(nproc)')
- sudo('make test -j$(nproc)')
- run('make runtest')
- sudo('make pycaffe')
- sudo('touch /home/' + os_user + '/.ensure_dir/caffe_ensured')
-
-
def install_caffe2(os_user, caffe2_version, cmake_version):
if not exists('/home/{}/.ensure_dir/caffe2_ensured'.format(os_user)):
env.shell = "/bin/bash -l -c -i"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org