You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/02/03 10:24:26 UTC
[incubator-datalab] 01/02: [DATALAB-2091]: replaced cd( and run(
with conn.cd( and conn.run(
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 9c368112213912c6488172a9dfdb940d0df19a54
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Feb 3 12:08:38 2021 +0200
[DATALAB-2091]: replaced cd( and run( with conn.cd( and conn.run(
---
.../scripts/deploy_repository/deploy_repository.py | 12 +++++------
.../src/dataengine/scripts/configure_dataengine.py | 2 +-
.../scripts/configure_deep_learning_node.py | 12 +++++------
.../src/general/lib/gcp/actions_lib.py | 6 +++---
.../src/general/lib/os/debian/edge_lib.py | 4 ++--
.../src/general/lib/os/debian/notebook_lib.py | 16 +++++++--------
.../src/general/lib/os/debian/ssn_lib.py | 2 +-
.../src/general/lib/os/fab.py | 4 ++--
.../src/general/lib/os/redhat/common_lib.py | 2 +-
.../src/general/lib/os/redhat/edge_lib.py | 8 ++++----
.../src/general/lib/os/redhat/notebook_lib.py | 24 +++++++++++-----------
.../src/general/lib/os/redhat/ssn_lib.py | 4 ++--
.../zeppelin_install_dataengine-service_kernels.py | 4 ++--
.../scripts/os/jupyterlab_container_start.py | 2 +-
.../src/general/scripts/os/superset_start.py | 2 +-
.../src/ssn/scripts/configure_docker.py | 2 +-
.../src/ssn/scripts/configure_ssn_node.py | 4 ++--
.../src/ssn/scripts/configure_ui.py | 4 ++--
18 files changed, 57 insertions(+), 57 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py b/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
index a30cc05..3662993 100644
--- a/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
+++ b/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
@@ -917,7 +917,7 @@ def install_nexus():
http://localhost:8081/service/rest/v1/script/configureNexus/run')
conn.sudo('systemctl stop nexus')
conn.sudo('git clone https://github.com/sonatype-nexus-community/nexus-repository-apt')
- with cd('nexus-repository-apt'):
+ with conn.cd('nexus-repository-apt'):
conn.sudo('mvn')
apt_plugin_version = conn.sudo('find nexus-repository-apt/ -name "nexus-repository-apt-*.jar" '
'-printf "%f\\n" | grep -v "sources"').replace('nexus-repository-apt-',
@@ -961,7 +961,7 @@ def install_nexus():
'''/opt/nexus/system/org/sonatype/nexus/assemblies/nexus-core-feature/{0}/nexus-core-feature-'''
'''{0}-features.xml'''.format(nexus_version))
conn.sudo('git clone https://github.com/sonatype-nexus-community/nexus-repository-r.git')
- with cd('nexus-repository-r'):
+ with conn.cd('nexus-repository-r'):
conn.sudo('mvn clean install')
r_plugin_version = conn.sudo('find nexus-repository-r/ -name "nexus-repository-r-*.jar" '
'-printf "%f\\n" | grep -v "sources"').replace('nexus-repository-r-', '').replace(
@@ -1079,9 +1079,9 @@ def mount_efs():
if not exists('/home/{}/.ensure_dir/efs_mounted'.format(configuration['conf_os_user'])):
conn.sudo('mkdir -p /opt/sonatype-work')
conn.sudo('apt-get -y install binutils')
- with cd('/tmp/'):
+ with conn.cd('/tmp/'):
conn.sudo('git clone https://github.com/aws/efs-utils')
- with cd('/tmp/efs-utils'):
+ with conn.cd('/tmp/efs-utils'):
conn.sudo('./build-deb.sh')
conn.sudo('apt-get -y install ./build/amazon-efs-utils*deb')
conn.sudo('sed -i "s/stunnel_check_cert_hostname.*/stunnel_check_cert_hostname = false/g" '
@@ -1234,7 +1234,7 @@ def download_packages():
package_name = package.split('/')[-1]
packages_list.append({'url': package, 'name': package_name})
conn.run('mkdir packages')
- with cd('packages'):
+ with conn.cd('packages'):
for package in packages_list:
conn.run('wget {0}'.format(package['url']))
conn.run('curl -v -u admin:{2} -F "raw.directory=/" -F '
@@ -1272,7 +1272,7 @@ def prepare_images():
try:
if not exists('/home/{}/.ensure_dir/images_prepared'.format(configuration['conf_os_user'])):
conn.put('files/Dockerfile', '/tmp/Dockerfile')
- with cd('/tmp/'):
+ with conn.cd('/tmp/'):
conn.sudo('docker build --file Dockerfile -t pre-base .')
conn.sudo('docker login -u {0} -p {1} localhost:8083'.format(args.nexus_service_user_name,
args.nexus_service_user_password))
diff --git a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
index 3d5b21f..7c8a5dc 100644
--- a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
+++ b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
@@ -89,7 +89,7 @@ def start_spark(os_user, master_ip, node):
if os.environ['application'] == 'deeplearning':
conn.sudo('''echo "LD_LIBRARY_PATH=/opt/cudnn/lib64:/usr/local/cuda/lib64:/usr/lib64/openmpi/lib" >> /opt/spark/conf/spark-env.sh''')
if node == 'master':
- with cd('/opt/spark/sbin/'):
+ with conn.cd('/opt/spark/sbin/'):
conn.sudo("sed -i '/start-slaves.sh/d' start-all.sh")
conn.sudo('''echo '"${}/sbin"/start-slave.sh spark://{}:7077' >> start-all.sh'''.format('{SPARK_HOME}', master_ip))
conn.put('~/templates/spark-master.service', '/tmp/spark-master.service')
diff --git a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
index bca8b15..a6c8d99 100644
--- a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
+++ b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
@@ -80,12 +80,12 @@ gitlab_certfile = os.environ['conf_gitlab_certfile']
def install_itorch(os_user):
if not exists('/home/{}/.ensure_dir/itorch_ensured'.format(os_user)):
- run('git clone https://github.com/facebook/iTorch.git')
- with cd('/home/{}/iTorch/'.format(os_user)):
- run('luarocks install luacrypto')
- run('luarocks install uuid')
- run('luarocks install lzmq')
- run('luarocks make')
+ conn.run('git clone https://github.com/facebook/iTorch.git')
+ with conn.cd('/home/{}/iTorch/'.format(os_user)):
+ conn.run('luarocks install luacrypto')
+ conn.run('luarocks install uuid')
+ conn.run('luarocks install lzmq')
+ conn.run('luarocks make')
conn.sudo('cp -rf /home/{0}/.ipython/kernels/itorch/ /home/{0}/.local/share/jupyter/kernels/'.format(os_user))
conn.sudo('chown -R {0}:{0} /home/{0}/.local/share/jupyter/'.format(os_user))
conn.sudo('touch /home/{}/.ensure_dir/itorch_ensured'.format(os_user))
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index 1e1d41e..015d354 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -1347,7 +1347,7 @@ def ensure_local_jars(os_user, jars_dir):
conn.sudo('mv /tmp/core-site.xml /opt/spark/conf/core-site.xml')
conn.put(templates_dir + 'notebook_spark-defaults_local.conf', '/tmp/notebook_spark-defaults_local.conf')
if os.environ['application'] == 'zeppelin':
- run('echo \"spark.jars $(ls -1 ' + jars_dir + '* | tr \'\\n\' \',\')\" >> /tmp/notebook_spark-defaults_local.conf')
+ conn.run('echo \"spark.jars $(ls -1 ' + jars_dir + '* | tr \'\\n\' \',\')\" >> /tmp/notebook_spark-defaults_local.conf')
conn.sudo('\cp /tmp/notebook_spark-defaults_local.conf /opt/spark/conf/spark-defaults.conf')
conn.sudo('touch /home/{}/.ensure_dir/gs_kernel_ensured'.format(os_user))
except Exception as err:
@@ -1406,7 +1406,7 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
spark_jars_paths = None
conn.put(templates_dir + 'notebook_spark-defaults_local.conf', '/tmp/notebook_spark-defaults_local.conf')
if os.environ['application'] == 'zeppelin':
- run('echo \"spark.jars $(ls -1 ' + jars_dir + '* | tr \'\\n\' \',\')\" >> /tmp/notebook_spark-defaults_local.conf')
+ conn.run('echo \"spark.jars $(ls -1 ' + jars_dir + '* | tr \'\\n\' \',\')\" >> /tmp/notebook_spark-defaults_local.conf')
conn.sudo('\cp -f /tmp/notebook_spark-defaults_local.conf /opt/spark/conf/spark-defaults.conf')
if memory_type == 'driver':
spark_memory = datalab.fab.get_spark_memory()
@@ -1415,7 +1415,7 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
spark_memory))
if not exists('/opt/spark/conf/spark-env.sh'):
conn.sudo('mv /opt/spark/conf/spark-env.sh.template /opt/spark/conf/spark-env.sh')
- java_home = run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").splitlines()[0]
+ java_home = conn.run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").splitlines()[0]
conn.sudo("echo 'export JAVA_HOME=\'{}\'' >> /opt/spark/conf/spark-env.sh".format(java_home))
if 'spark_configurations' in os.environ:
datalab_header = conn.sudo('cat /tmp/notebook_spark-defaults_local.conf | grep "^#"')
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
index d3d2628..4164b7d 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
@@ -116,7 +116,7 @@ def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak
-out /etc/ssl/certs/datalab.crt -subj "/C=US/ST=US/L=US/O=datalab/CN={}"'.format(hostname))
conn.sudo('mkdir -p /tmp/src')
- with cd('/tmp/src/'):
+ with conn.cd('/tmp/src/'):
conn.sudo('wget https://luarocks.org/releases/luarocks-3.3.1.tar.gz')
conn.sudo('tar -xzf luarocks-3.3.1.tar.gz')
@@ -125,7 +125,7 @@ def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak
conn.sudo('apt-get update')
conn.sudo('apt-get -y install openresty=1.19.3.1-1~focal1')
- with cd('/tmp/src/luarocks-3.3.1/'):
+ with conn.cd('/tmp/src/luarocks-3.3.1/'):
conn.sudo('./configure')
conn.sudo('make install')
conn.sudo('luarocks install lua-resty-jwt 0.2.2 --tree /usr/local/openresty/lualib/resty/')
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index c483ea7..0f1cc7c 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -279,7 +279,7 @@ def install_tensor(os_user, cuda_version, cuda_file_name,
with settings(warn_only=True):
reboot(wait=180)
manage_pkg('-y install', 'remote', 'dkms libglvnd-dev')
- kernel_version = run('uname -r | tr -d "[..0-9-]"')
+ kernel_version = conn.run('uname -r | tr -d "[..0-9-]"')
if kernel_version == 'azure':
manage_pkg('-y install', 'remote', 'linux-modules-`uname -r`')
else:
@@ -300,15 +300,15 @@ def install_tensor(os_user, cuda_version, cuda_file_name,
conn.sudo('ln -s /opt/cuda-{0} /usr/local/cuda-{0}'.format(cuda_version))
conn.sudo('rm -f /opt/{}'.format(cuda_file_name))
# install cuDNN
- run('wget http://developer.download.nvidia.com/compute/redist/cudnn/v{0}/{1} -O /tmp/{1}'.format(
+ conn.run('wget http://developer.download.nvidia.com/compute/redist/cudnn/v{0}/{1} -O /tmp/{1}'.format(
cudnn_version, cudnn_file_name))
- run('tar xvzf /tmp/{} -C /tmp'.format(cudnn_file_name))
+ conn.run('tar xvzf /tmp/{} -C /tmp'.format(cudnn_file_name))
conn.sudo('mkdir -p /opt/cudnn/include')
conn.sudo('mkdir -p /opt/cudnn/lib64')
conn.sudo('mv /tmp/cuda/include/cudnn.h /opt/cudnn/include')
conn.sudo('mv /tmp/cuda/lib64/libcudnn* /opt/cudnn/lib64')
conn.sudo('chmod a+r /opt/cudnn/include/cudnn.h /opt/cudnn/lib64/libcudnn*')
- run(
+ conn.run(
'echo "export LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64\"" >> ~/.bashrc')
# install TensorFlow and run TensorBoard
# conn.sudo('python2.7 -m pip install --upgrade https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp27-none-linux_x86_64.whl --no-cache-dir'.format(tensorflow_version))
@@ -316,8 +316,8 @@ def install_tensor(os_user, cuda_version, cuda_file_name,
conn.sudo('mkdir /var/log/tensorboard; chown {0}:{0} -R /var/log/tensorboard'.format(os_user))
conn.put('{}tensorboard.service'.format(templates_dir), '/tmp/tensorboard.service')
conn.sudo("sed -i 's|OS_USR|{}|' /tmp/tensorboard.service".format(os_user))
- http_proxy = run('echo $http_proxy')
- https_proxy = run('echo $https_proxy')
+ http_proxy = conn.run('echo $http_proxy')
+ https_proxy = conn.run('echo $https_proxy')
conn.sudo('sed -i \'/\[Service\]/ a\Environment=\"HTTP_PROXY={}\"\' /tmp/tensorboard.service'.format(
http_proxy))
conn.sudo('sed -i \'/\[Service\]/ a\Environment=\"HTTPS_PROXY={}\"\' /tmp/tensorboard.service'.format(
@@ -477,11 +477,11 @@ def install_caffe2(os_user, caffe2_version, cmake_version):
conn.sudo('wget https://cmake.org/files/v{2}/cmake-{1}.tar.gz -O /home/{0}/cmake-{1}.tar.gz'.format(
os_user, cmake_version, cmake_version.split('.')[0] + "." + cmake_version.split('.')[1]))
conn.sudo('tar -zxvf cmake-{}.tar.gz'.format(cmake_version))
- with cd('/home/{}/cmake-{}/'.format(os_user, cmake_version)):
+ with conn.cd('/home/{}/cmake-{}/'.format(os_user, cmake_version)):
conn.sudo('./bootstrap --prefix=/usr/local && make && make install')
conn.sudo('ln -s /usr/local/bin/cmake /bin/cmake{}'.format(cmake_version))
conn.sudo('git clone https://github.com/pytorch/pytorch.git')
- with cd('/home/{}/pytorch/'.format(os_user)):
+ with conn.cd('/home/{}/pytorch/'.format(os_user)):
conn.sudo('git submodule update --init')
with settings(warn_only=True):
conn.sudo('git checkout {}'.format(os.environ['notebook_pytorch_branch']))
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
index 34ccebe..c69622a 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
@@ -373,7 +373,7 @@ def install_build_dep():
if not exists('{}tmp/build_dep_ensured'.format(os.environ['ssn_datalab_path'])):
maven_version = '3.5.4'
manage_pkg('-y install', 'remote', 'openjdk-8-jdk git wget unzip')
- with cd('/opt/'):
+ with conn.cd('/opt/'):
conn.sudo(
'wget http://mirrors.sonic.net/apache/maven/maven-{0}/{1}/binaries/apache-maven-{1}-bin.zip'.format(
maven_version.split('.')[0], maven_version))
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index c2ded0c..aced989 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -195,7 +195,7 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
conn.sudo('pip3 install jupyter --no-cache-dir')
conn.sudo('rm -rf {}'.format(jupyter_conf_file))
conn.run('jupyter notebook --generate-config --config {}'.format(jupyter_conf_file))
- with cd('/home/{}'.format(os_user)):
+ with conn.cd('/home/{}'.format(os_user)):
conn.run('mkdir -p ~/.jupyter/custom/')
conn.run('echo "#notebook-container { width: auto; }" > ~/.jupyter/custom/custom.css')
conn.sudo('echo "c.NotebookApp.ip = \'0.0.0.0\'" >> {}'.format(jupyter_conf_file))
@@ -938,7 +938,7 @@ def configure_superset(os_user, keycloak_auth_server_url, keycloak_realm_name, k
print('Superset configuring')
try:
if not exists('/home/{}/incubator-superset'.format(os_user)):
- with cd('/home/{}'.format(os_user)):
+ with conn.cd('/home/{}'.format(os_user)):
conn.sudo('wget https://github.com/apache/incubator-superset/archive/{}.tar.gz'.format(
os.environ['notebook_superset_version']))
conn.sudo('tar -xzf {}.tar.gz'.format(os.environ['notebook_superset_version']))
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
index b4f880f..bd96aa2 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
@@ -65,7 +65,7 @@ def ensure_pkg(user, requisites='git vim gcc python-devel openssl-devel nmap lib
conn.sudo('setenforce 0')
conn.sudo("sed -i '/^SELINUX=/s/SELINUX=.*/SELINUX=disabled/g' /etc/selinux/config")
mirror = 'mirror.centos.org'
- with cd('/etc/yum.repos.d/'):
+ with conn.cd('/etc/yum.repos.d/'):
conn.sudo('echo "[Centos-repo]" > centOS-base.repo')
conn.sudo('echo "name=Centos 7 Repository" >> centOS-base.repo')
conn.sudo('echo "baseurl=http://{}/centos/7/os/x86_64/" >> centOS-base.repo'.format(mirror))
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
index 03bb708..965c4eb 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
@@ -118,7 +118,7 @@ def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak
-out /etc/ssl/certs/datalab.crt -subj "/C=US/ST=US/L=US/O=datalab/CN={}"'.format(hostname))
conn.sudo('mkdir -p /tmp/lua')
conn.sudo('mkdir -p /tmp/src')
- with cd('/tmp/src/'):
+ with conn.cd('/tmp/src/'):
conn.sudo('wget http://nginx.org/download/nginx-{}.tar.gz'.format(nginx_version))
conn.sudo('tar -xzf nginx-{}.tar.gz'.format(nginx_version))
@@ -136,11 +136,11 @@ def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak
conn.sudo('ln -sf nginx-{} nginx'.format(nginx_version))
- with cd('/tmp/src/LuaJIT-2.0.5/'):
+ with conn.cd('/tmp/src/LuaJIT-2.0.5/'):
conn.sudo('make')
conn.sudo('make install')
- with cd('/tmp/src/nginx/'), shell_env(LUAJIT_LIB='/usr/local/lib/', LUAJIT_INC='/usr/local/include/luajit-2.0'):
+ with conn.cd('/tmp/src/nginx/'), shell_env(LUAJIT_LIB='/usr/local/lib/', LUAJIT_INC='/usr/local/include/luajit-2.0'):
conn.sudo('./configure --user=nginx --group=nginx --prefix=/etc/nginx --sbin-path=/usr/sbin/nginx \
--conf-path=/etc/nginx/nginx.conf --pid-path=/run/nginx.pid --lock-path=/run/lock/subsys/nginx \
--error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log \
@@ -151,7 +151,7 @@ def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak
conn.sudo('make')
conn.sudo('make install')
- with cd('/tmp/src/luarocks-2.2.2/'):
+ with conn.cd('/tmp/src/luarocks-2.2.2/'):
conn.sudo('./configure --with-lua-include=/usr/local/include/luajit-2.0')
conn.sudo('make build')
conn.sudo('make install')
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
index 8c5d9a2..e3b3790 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
@@ -249,14 +249,14 @@ def install_tensor(os_user, cuda_version, cuda_file_name,
conn.sudo('ln -s /opt/cuda-{0} /usr/local/cuda-{0}'.format(cuda_version))
conn.sudo('rm -f /opt/{}'.format(cuda_file_name))
# install cuDNN
- run('wget http://developer.download.nvidia.com/compute/redist/cudnn/v{0}/{1} -O /tmp/{1}'.format(cudnn_version, cudnn_file_name))
- run('tar xvzf /tmp/{} -C /tmp'.format(cudnn_file_name))
+ conn.run('wget http://developer.download.nvidia.com/compute/redist/cudnn/v{0}/{1} -O /tmp/{1}'.format(cudnn_version, cudnn_file_name))
+ conn.run('tar xvzf /tmp/{} -C /tmp'.format(cudnn_file_name))
conn.sudo('mkdir -p /opt/cudnn/include')
conn.sudo('mkdir -p /opt/cudnn/lib64')
conn.sudo('mv /tmp/cuda/include/cudnn.h /opt/cudnn/include')
conn.sudo('mv /tmp/cuda/lib64/libcudnn* /opt/cudnn/lib64')
conn.sudo('chmod a+r /opt/cudnn/include/cudnn.h /opt/cudnn/lib64/libcudnn*')
- run('echo "export LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64\"" >> ~/.bashrc')
+ conn.run('echo "export LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64\"" >> ~/.bashrc')
# install TensorFlow and run TensorBoard
conn.sudo('wget https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp27-none-linux_x86_64.whl'.format(tensorflow_version))
conn.sudo('wget https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp35-cp35m-linux_x86_64.whl'.format(tensorflow_version))
@@ -402,13 +402,13 @@ def install_opencv(os_user):
manage_pkg('-y install', 'remote', 'cmake python34 python34-devel python34-pip gcc gcc-c++')
conn.sudo('pip3.4 install numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
conn.sudo('pip3.5 install numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
- run('git clone https://github.com/opencv/opencv.git')
- with cd('/home/{}/opencv/'.format(os_user)):
- run('git checkout 3.2.0')
- run('mkdir release')
- with cd('/home/{}/opencv/release/'.format(os_user)):
- run('cmake -DINSTALL_TESTS=OFF -D CUDA_GENERATION=Auto -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=$(python2 -c "import sys; print(sys.prefix)") -D PYTHON_EXECUTABLE=$(which python2) ..')
- run('make -j$(nproc)')
+ conn.run('git clone https://github.com/opencv/opencv.git')
+ with conn.cd('/home/{}/opencv/'.format(os_user)):
+ conn.run('git checkout 3.2.0')
+ conn.run('mkdir release')
+ with conn.cd('/home/{}/opencv/release/'.format(os_user)):
+ conn.run('cmake -DINSTALL_TESTS=OFF -D CUDA_GENERATION=Auto -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=$(python2 -c "import sys; print(sys.prefix)") -D PYTHON_EXECUTABLE=$(which python2) ..')
+ conn.run('make -j$(nproc)')
conn.sudo('make install')
conn.sudo('touch /home/' + os_user + '/.ensure_dir/opencv_ensured')
@@ -425,11 +425,11 @@ def install_caffe2(os_user, caffe2_version, cmake_version):
conn.sudo('wget https://cmake.org/files/v{2}/cmake-{1}.tar.gz -O /home/{0}/cmake-{1}.tar.gz'.format(
os_user, cmake_version, cmake_version.split('.')[0] + "." + cmake_version.split('.')[1]))
conn.sudo('tar -zxvf cmake-{}.tar.gz'.format(cmake_version))
- with cd('/home/{}/cmake-{}/'.format(os_user, cmake_version)):
+ with conn.cd('/home/{}/cmake-{}/'.format(os_user, cmake_version)):
conn.sudo('./bootstrap --prefix=/usr/local && make && make install')
conn.sudo('ln -s /usr/local/bin/cmake /bin/cmake{}'.format(cmake_version))
conn.sudo('git clone https://github.com/pytorch/pytorch.git')
- with cd('/home/{}/pytorch/'.format(os_user)):
+ with conn.cd('/home/{}/pytorch/'.format(os_user)):
conn.sudo('git submodule update --init')
with settings(warn_only=True):
conn.sudo('git checkout v{}'.format(caffe2_version))
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
index c7733d6..c2ee35b 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
@@ -40,7 +40,7 @@ def ensure_docker_daemon(datalab_path, os_user, region):
mirror = 'mirror.lzu.edu.cn'
else:
mirror = 'mirror.centos.org'
- with cd('/etc/yum.repos.d/'):
+ with conn.cd('/etc/yum.repos.d/'):
conn.sudo('echo "[centosrepo]" > centos.repo')
conn.sudo('echo "name=Centos 7 Repository" >> centos.repo')
conn.sudo('echo "baseurl=http://{}/centos/7/extras/x86_64/" >> centos.repo'.format(mirror))
@@ -389,7 +389,7 @@ def install_build_dep():
if not exists('{}tmp/build_dep_ensured'.format(os.environ['ssn_datalab_path'])):
maven_version = '3.5.4'
manage_pkg('-y install', 'remote', 'java-1.8.0-openjdk java-1.8.0-openjdk-devel git wget unzip')
- with cd('/opt/'):
+ with conn.cd('/opt/'):
conn.sudo(
'wget http://mirrors.sonic.net/apache/maven/maven-{0}/{1}/binaries/apache-maven-{1}-bin.zip'.format(
maven_version.split('.')[0], maven_version))
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
index d22ce7f..f022b1a 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
@@ -57,9 +57,9 @@ def configure_notebook(args):
conn.sudo('\cp /tmp/create_configs.py /usr/local/bin/create_configs.py')
conn.sudo('chmod 755 /usr/local/bin/create_configs.py')
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
- run('mkdir -p /tmp/datalab_libs/')
+ conn.run('mkdir -p /tmp/datalab_libs/')
local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
- run('chmod a+x /tmp/datalab_libs/*')
+ conn.run('chmod a+x /tmp/datalab_libs/*')
conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
conn.sudo('mkdir -p /usr/lib64/python3.8')
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py b/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
index 4dabf7d..592c98e 100644
--- a/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
+++ b/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
@@ -37,7 +37,7 @@ jupyterlab_dir = '/home/' + args.os_user + '/.jupyterlab/'
def start_jupyterlab_container(jupyterlab_dir):
try:
- with cd('{}'.format(jupyterlab_dir)):
+ with conn.cd('{}'.format(jupyterlab_dir)):
conn.run('docker build --network=host --file Dockerfile_jupyterlab -t jupyter-lab .'.format(args.os_user))
container_id = conn.run('docker ps | awk \'NR==2{print $1}\'')
if container_id != '':
diff --git a/infrastructure-provisioning/src/general/scripts/os/superset_start.py b/infrastructure-provisioning/src/general/scripts/os/superset_start.py
index 611715c..e2e974d 100644
--- a/infrastructure-provisioning/src/general/scripts/os/superset_start.py
+++ b/infrastructure-provisioning/src/general/scripts/os/superset_start.py
@@ -37,7 +37,7 @@ superset_dir = '/home/' + args.os_user + '/incubator-superset/contrib/docker'
def start_superset(superset_dir):
try:
- with cd('{}'.format(superset_dir)):
+ with conn.cd('{}'.format(superset_dir)):
conn.sudo('docker-compose run --rm superset ./docker-init.sh')
conn.sudo('cp /opt/datalab/templates/superset-notebook.service /tmp/')
conn.sudo('sed -i \'s/OS_USER/{}/g\' /tmp/superset-notebook.service'.format(args.os_user))
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
index afc5851..419b521 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
@@ -74,7 +74,7 @@ def download_toree():
def add_china_repository(datalab_path):
- with cd('{}sources/infrastructure-provisioning/src/base/'.format(datalab_path)):
+ with conn.cd('{}sources/infrastructure-provisioning/src/base/'.format(datalab_path)):
conn.sudo('sed -i "/pip install/s/$/ -i https\:\/\/{0}\/simple --trusted-host {0} --timeout 60000/g" '
'Dockerfile'.format(os.environ['conf_pypi_mirror']))
conn.sudo('sed -i "/pip install/s/jupyter/ipython==5.0.0 jupyter==1.0.0/g" Dockerfile')
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py b/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
index 9b59319..c52b7cb 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
@@ -73,7 +73,7 @@ def cp_key(keyfile, host_string, os_user):
def cp_backup_scripts(datalab_path):
try:
- with cd(datalab_path + "tmp/"):
+ with conn.cd(datalab_path + "tmp/"):
conn.put('/root/scripts/backup.py', "backup.py")
conn.put('/root/scripts/restore.py', "restore.py")
conn.run('chmod +x backup.py restore.py')
@@ -87,7 +87,7 @@ def cp_gitlab_scripts(datalab_path):
try:
if not exists('{}tmp/gitlab'.format(datalab_path)):
conn.run('mkdir -p {}tmp/gitlab'.format(datalab_path))
- with cd('{}tmp/gitlab'.format(datalab_path)):
+ with conn.cd('{}tmp/gitlab'.format(datalab_path)):
conn.put('/root/scripts/gitlab_deploy.py', 'gitlab_deploy.py')
conn.put('/root/scripts/configure_gitlab.py', 'configure_gitlab.py')
conn.run('chmod +x gitlab_deploy.py configure_gitlab.py')
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
index cb85f29..b920aac 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
@@ -146,7 +146,7 @@ def configure_mongo(mongo_passwd, default_endpoint_name):
def build_ui():
try:
# Building Front-end
- with cd(args.datalab_path + '/sources/services/self-service/src/main/resources/webapp/'):
+ with conn.cd(args.datalab_path + '/sources/services/self-service/src/main/resources/webapp/'):
conn.sudo('sed -i "s|CLOUD_PROVIDER|{}|g" src/dictionary/global.dictionary.ts'.format(args.cloud_provider))
if args.cloud_provider == 'azure' and os.environ['azure_datalake_enable'] == 'true':
@@ -158,7 +158,7 @@ def build_ui():
conn.sudo('sudo chown -R {} {}/*'.format(args.os_user, args.datalab_path))
# Building Back-end
- with cd(args.datalab_path + '/sources/'):
+ with conn.cd(args.datalab_path + '/sources/'):
conn.sudo('/opt/maven/bin/mvn -P{} -DskipTests package'.format(args.cloud_provider))
conn.sudo('mkdir -p {}/webapp/'.format(args.datalab_path))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org