You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/02/03 10:24:25 UTC
[incubator-datalab] branch DATALAB-2091 updated (559f7f6 -> 165142d)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
from 559f7f6 [DATALAB-2091]: replaced import of fabric.api with import fabric
new 9c36811 [DATALAB-2091]: replaced cd( and run( with conn.cd( and conn.run(
new 165142d [DATALAB-2091]: removed lcd(
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../scripts/deploy_datalab.py | 9 ++++----
.../scripts/deploy_repository/deploy_repository.py | 12 +++++------
.../src/dataengine/scripts/configure_dataengine.py | 2 +-
.../scripts/configure_deep_learning_node.py | 12 +++++------
.../src/general/lib/aws/actions_lib.py | 9 +++-----
.../src/general/lib/gcp/actions_lib.py | 14 ++++++-------
.../src/general/lib/os/debian/edge_lib.py | 4 ++--
.../src/general/lib/os/debian/notebook_lib.py | 16 +++++++--------
.../src/general/lib/os/debian/ssn_lib.py | 2 +-
.../src/general/lib/os/fab.py | 4 ++--
.../src/general/lib/os/redhat/common_lib.py | 2 +-
.../src/general/lib/os/redhat/edge_lib.py | 8 ++++----
.../src/general/lib/os/redhat/notebook_lib.py | 24 +++++++++++-----------
.../src/general/lib/os/redhat/ssn_lib.py | 4 ++--
.../azure/common_notebook_update_refresh_token.py | 1 -
.../zeppelin_install_dataengine-service_kernels.py | 4 ++--
.../scripts/os/jupyterlab_container_start.py | 2 +-
.../src/general/scripts/os/superset_start.py | 2 +-
.../src/ssn/scripts/configure_docker.py | 2 +-
.../src/ssn/scripts/configure_ssn_node.py | 4 ++--
.../src/ssn/scripts/configure_ui.py | 4 ++--
.../src/ssn/scripts/docker_build.py | 21 ++++++++-----------
22 files changed, 76 insertions(+), 86 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 02/02: [DATALAB-2091]: removed lcd(
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 165142dd6339c0ec3160ab3ed39b9d2dacd1d260
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Feb 3 12:24:07 2021 +0200
[DATALAB-2091]: removed lcd(
---
.../scripts/deploy_datalab.py | 9 ++++-----
.../src/general/lib/aws/actions_lib.py | 9 +++------
.../src/general/lib/gcp/actions_lib.py | 8 +++-----
.../azure/common_notebook_update_refresh_token.py | 1 -
.../src/ssn/scripts/docker_build.py | 21 +++++++++------------
5 files changed, 19 insertions(+), 29 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py
index a3a07ee..6d7a750 100644
--- a/infrastructure-provisioning/scripts/deploy_datalab.py
+++ b/infrastructure-provisioning/scripts/deploy_datalab.py
@@ -197,12 +197,11 @@ def generate_docker_command():
def build_docker_images(args):
# Building base and ssn docker images
- with lcd(args.workspace_path):
- local('sudo docker build --build-arg OS={0} --build-arg SRC_PATH="infrastructure-provisioning/src/" --file '
+ local('cd {2}; sudo docker build --build-arg OS={0} --build-arg SRC_PATH="infrastructure-provisioning/src/" --file '
'infrastructure-provisioning/src/general/files/{1}/'
- 'base_Dockerfile -t docker.datalab-base .'.format(args.conf_os_family, args.conf_cloud_provider))
- local('sudo docker build --build-arg OS={0} --file infrastructure-provisioning/src/general/files/{1}/'
- 'ssn_Dockerfile -t docker.datalab-ssn .'.format(args.conf_os_family, args.conf_cloud_provider))
+ 'base_Dockerfile -t docker.datalab-base .'.format(args.conf_os_family, args.conf_cloud_provider, args.workspace_path))
+ local('cd {2}; sudo docker build --build-arg OS={0} --file infrastructure-provisioning/src/general/files/{1}/'
+ 'ssn_Dockerfile -t docker.datalab-ssn .'.format(args.conf_os_family, args.conf_cloud_provider, args.workspace_path))
def deploy_datalab(args):
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index a3580e5..3f31ae1 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -1529,12 +1529,9 @@ def installing_python(region, bucket, user_name, cluster_name, application='', p
local('wget https://www.python.org/ftp/python/' + python_version +
'/Python-' + python_version + '.tgz -O /tmp/Python-' + python_version + '.tgz')
local('tar zxvf /tmp/Python-' + python_version + '.tgz -C /tmp/')
- with lcd('/tmp/Python-' + python_version):
- local('./configure --prefix=/opt/python/python' + python_version +
- ' --with-zlib-dir=/usr/local/lib/ --with-ensurepip=install')
- local('sudo make altinstall')
- with lcd('/tmp/'):
- local('sudo rm -rf Python-' + python_version + '/')
+ local('cd /tmp/Python-{0}; ./configure --prefix=/opt/python/python{0} --with-zlib-dir=/usr/local/lib/ --with-ensurepip=install'.format(python_version))
+ local('cd /tmp/Python-{0}; sudo make altinstall'.format(python_version))
+ local('cd /tmp/; sudo rm -rf Python-' + python_version + '/')
if region == 'cn-north-1':
local('sudo -i /opt/python/python{}/bin/python{} -m pip install -U pip=={} --no-cache-dir'.format(
python_version, python_version[0:3], os.environ['conf_pip_version']))
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index 015d354..fc95c91 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -1302,11 +1302,9 @@ class GCPActions:
if not os.path.exists('/opt/python/python{}'.format(python_version)):
local('wget https://www.python.org/ftp/python/{0}/Python-{0}.tgz -O /tmp/Python-{0}.tgz'.format(python_version))
local('tar zxvf /tmp/Python-{}.tgz -C /tmp/'.format(python_version))
- with lcd('/tmp/Python-{}'.format(python_version)):
- local('./configure --prefix=/opt/python/python{} --with-zlib-dir=/usr/local/lib/ --with-ensurepip=install'.format(python_version))
- local('sudo make altinstall')
- with lcd('/tmp/'):
- local('sudo rm -rf Python-{}/'.format(python_version))
+ local('cd /tmp/Python-{0}; ./configure --prefix=/opt/python/python{0} --with-zlib-dir=/usr/local/lib/ --with-ensurepip=install'.format(python_version))
+ local('cd /tmp/Python-{}; sudo make altinstall'.format(python_version))
+ local('cd /tmp/; sudo rm -rf Python-{}/'.format(python_version))
local('sudo -i virtualenv /opt/python/python{}'.format(python_version))
venv_command = 'source /opt/python/python{}/bin/activate'.format(python_version)
pip_command = '/opt/python/python{0}/bin/pip{1}'.format(python_version, python_version[:3])
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_notebook_update_refresh_token.py b/infrastructure-provisioning/src/general/scripts/azure/common_notebook_update_refresh_token.py
index ffef0f7..0b8d07d 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_notebook_update_refresh_token.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_notebook_update_refresh_token.py
@@ -27,7 +27,6 @@ import argparse
import os
import sys
import time
-from fabric import lcd
from fabric.contrib.files import exists
parser = argparse.ArgumentParser()
diff --git a/infrastructure-provisioning/src/ssn/scripts/docker_build.py b/infrastructure-provisioning/src/ssn/scripts/docker_build.py
index 709f61c..9347d6b 100644
--- a/infrastructure-provisioning/src/ssn/scripts/docker_build.py
+++ b/infrastructure-provisioning/src/ssn/scripts/docker_build.py
@@ -60,18 +60,15 @@ def image_build(src_path, node):
local('cp /home/datalab-user/keys/azure_auth.json {}base/azure_auth.json'.format(src_path))
else:
cloud_provider = 'gcp'
- with lcd(src_path):
- local(
- 'docker build --build-arg OS={0} --build-arg SRC_PATH= --file general/files/{1}/base_Dockerfile -t docker.datalab-base:latest .'.format(
- os_family, cloud_provider))
- try:
- for i in range(len(node)):
- local(
- 'docker build --build-arg OS={0} --file general/files/{1}/{2}_Dockerfile -t docker.datalab-{2} .'.format(
- os_family, cloud_provider, node[i]))
- except Exception as err:
- print("Failed to build {} image".format(node[i]), str(err))
- raise Exception
+ local('cd {2}; docker build --build-arg OS={0} --build-arg SRC_PATH= --file general/files/{1}/base_Dockerfile -t docker.datalab-base:latest .'.format(
+ os_family, cloud_provider, src_path))
+ try:
+ for i in range(len(node)):
+ local('cd {3}; docker build --build-arg OS={0} --file general/files/{1}/{2}_Dockerfile -t docker.datalab-{2} .'.format(
+ os_family, cloud_provider, node[i], src_path))
+ except Exception as err:
+ print("Failed to build {} image".format(node[i]), str(err))
+ raise Exception
except Exception as err:
traceback.print_exc()
sys.exit(1)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/02: [DATALAB-2091]: replaced cd( and run(
with conn.cd( and conn.run(
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 9c368112213912c6488172a9dfdb940d0df19a54
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Feb 3 12:08:38 2021 +0200
[DATALAB-2091]: replaced cd( and run( with conn.cd( and conn.run(
---
.../scripts/deploy_repository/deploy_repository.py | 12 +++++------
.../src/dataengine/scripts/configure_dataengine.py | 2 +-
.../scripts/configure_deep_learning_node.py | 12 +++++------
.../src/general/lib/gcp/actions_lib.py | 6 +++---
.../src/general/lib/os/debian/edge_lib.py | 4 ++--
.../src/general/lib/os/debian/notebook_lib.py | 16 +++++++--------
.../src/general/lib/os/debian/ssn_lib.py | 2 +-
.../src/general/lib/os/fab.py | 4 ++--
.../src/general/lib/os/redhat/common_lib.py | 2 +-
.../src/general/lib/os/redhat/edge_lib.py | 8 ++++----
.../src/general/lib/os/redhat/notebook_lib.py | 24 +++++++++++-----------
.../src/general/lib/os/redhat/ssn_lib.py | 4 ++--
.../zeppelin_install_dataengine-service_kernels.py | 4 ++--
.../scripts/os/jupyterlab_container_start.py | 2 +-
.../src/general/scripts/os/superset_start.py | 2 +-
.../src/ssn/scripts/configure_docker.py | 2 +-
.../src/ssn/scripts/configure_ssn_node.py | 4 ++--
.../src/ssn/scripts/configure_ui.py | 4 ++--
18 files changed, 57 insertions(+), 57 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py b/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
index a30cc05..3662993 100644
--- a/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
+++ b/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
@@ -917,7 +917,7 @@ def install_nexus():
http://localhost:8081/service/rest/v1/script/configureNexus/run')
conn.sudo('systemctl stop nexus')
conn.sudo('git clone https://github.com/sonatype-nexus-community/nexus-repository-apt')
- with cd('nexus-repository-apt'):
+ with conn.cd('nexus-repository-apt'):
conn.sudo('mvn')
apt_plugin_version = conn.sudo('find nexus-repository-apt/ -name "nexus-repository-apt-*.jar" '
'-printf "%f\\n" | grep -v "sources"').replace('nexus-repository-apt-',
@@ -961,7 +961,7 @@ def install_nexus():
'''/opt/nexus/system/org/sonatype/nexus/assemblies/nexus-core-feature/{0}/nexus-core-feature-'''
'''{0}-features.xml'''.format(nexus_version))
conn.sudo('git clone https://github.com/sonatype-nexus-community/nexus-repository-r.git')
- with cd('nexus-repository-r'):
+ with conn.cd('nexus-repository-r'):
conn.sudo('mvn clean install')
r_plugin_version = conn.sudo('find nexus-repository-r/ -name "nexus-repository-r-*.jar" '
'-printf "%f\\n" | grep -v "sources"').replace('nexus-repository-r-', '').replace(
@@ -1079,9 +1079,9 @@ def mount_efs():
if not exists('/home/{}/.ensure_dir/efs_mounted'.format(configuration['conf_os_user'])):
conn.sudo('mkdir -p /opt/sonatype-work')
conn.sudo('apt-get -y install binutils')
- with cd('/tmp/'):
+ with conn.cd('/tmp/'):
conn.sudo('git clone https://github.com/aws/efs-utils')
- with cd('/tmp/efs-utils'):
+ with conn.cd('/tmp/efs-utils'):
conn.sudo('./build-deb.sh')
conn.sudo('apt-get -y install ./build/amazon-efs-utils*deb')
conn.sudo('sed -i "s/stunnel_check_cert_hostname.*/stunnel_check_cert_hostname = false/g" '
@@ -1234,7 +1234,7 @@ def download_packages():
package_name = package.split('/')[-1]
packages_list.append({'url': package, 'name': package_name})
conn.run('mkdir packages')
- with cd('packages'):
+ with conn.cd('packages'):
for package in packages_list:
conn.run('wget {0}'.format(package['url']))
conn.run('curl -v -u admin:{2} -F "raw.directory=/" -F '
@@ -1272,7 +1272,7 @@ def prepare_images():
try:
if not exists('/home/{}/.ensure_dir/images_prepared'.format(configuration['conf_os_user'])):
conn.put('files/Dockerfile', '/tmp/Dockerfile')
- with cd('/tmp/'):
+ with conn.cd('/tmp/'):
conn.sudo('docker build --file Dockerfile -t pre-base .')
conn.sudo('docker login -u {0} -p {1} localhost:8083'.format(args.nexus_service_user_name,
args.nexus_service_user_password))
diff --git a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
index 3d5b21f..7c8a5dc 100644
--- a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
+++ b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
@@ -89,7 +89,7 @@ def start_spark(os_user, master_ip, node):
if os.environ['application'] == 'deeplearning':
conn.sudo('''echo "LD_LIBRARY_PATH=/opt/cudnn/lib64:/usr/local/cuda/lib64:/usr/lib64/openmpi/lib" >> /opt/spark/conf/spark-env.sh''')
if node == 'master':
- with cd('/opt/spark/sbin/'):
+ with conn.cd('/opt/spark/sbin/'):
conn.sudo("sed -i '/start-slaves.sh/d' start-all.sh")
conn.sudo('''echo '"${}/sbin"/start-slave.sh spark://{}:7077' >> start-all.sh'''.format('{SPARK_HOME}', master_ip))
conn.put('~/templates/spark-master.service', '/tmp/spark-master.service')
diff --git a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
index bca8b15..a6c8d99 100644
--- a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
+++ b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
@@ -80,12 +80,12 @@ gitlab_certfile = os.environ['conf_gitlab_certfile']
def install_itorch(os_user):
if not exists('/home/{}/.ensure_dir/itorch_ensured'.format(os_user)):
- run('git clone https://github.com/facebook/iTorch.git')
- with cd('/home/{}/iTorch/'.format(os_user)):
- run('luarocks install luacrypto')
- run('luarocks install uuid')
- run('luarocks install lzmq')
- run('luarocks make')
+ conn.run('git clone https://github.com/facebook/iTorch.git')
+ with conn.cd('/home/{}/iTorch/'.format(os_user)):
+ conn.run('luarocks install luacrypto')
+ conn.run('luarocks install uuid')
+ conn.run('luarocks install lzmq')
+ conn.run('luarocks make')
conn.sudo('cp -rf /home/{0}/.ipython/kernels/itorch/ /home/{0}/.local/share/jupyter/kernels/'.format(os_user))
conn.sudo('chown -R {0}:{0} /home/{0}/.local/share/jupyter/'.format(os_user))
conn.sudo('touch /home/{}/.ensure_dir/itorch_ensured'.format(os_user))
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index 1e1d41e..015d354 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -1347,7 +1347,7 @@ def ensure_local_jars(os_user, jars_dir):
conn.sudo('mv /tmp/core-site.xml /opt/spark/conf/core-site.xml')
conn.put(templates_dir + 'notebook_spark-defaults_local.conf', '/tmp/notebook_spark-defaults_local.conf')
if os.environ['application'] == 'zeppelin':
- run('echo \"spark.jars $(ls -1 ' + jars_dir + '* | tr \'\\n\' \',\')\" >> /tmp/notebook_spark-defaults_local.conf')
+ conn.run('echo \"spark.jars $(ls -1 ' + jars_dir + '* | tr \'\\n\' \',\')\" >> /tmp/notebook_spark-defaults_local.conf')
conn.sudo('\cp /tmp/notebook_spark-defaults_local.conf /opt/spark/conf/spark-defaults.conf')
conn.sudo('touch /home/{}/.ensure_dir/gs_kernel_ensured'.format(os_user))
except Exception as err:
@@ -1406,7 +1406,7 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
spark_jars_paths = None
conn.put(templates_dir + 'notebook_spark-defaults_local.conf', '/tmp/notebook_spark-defaults_local.conf')
if os.environ['application'] == 'zeppelin':
- run('echo \"spark.jars $(ls -1 ' + jars_dir + '* | tr \'\\n\' \',\')\" >> /tmp/notebook_spark-defaults_local.conf')
+ conn.run('echo \"spark.jars $(ls -1 ' + jars_dir + '* | tr \'\\n\' \',\')\" >> /tmp/notebook_spark-defaults_local.conf')
conn.sudo('\cp -f /tmp/notebook_spark-defaults_local.conf /opt/spark/conf/spark-defaults.conf')
if memory_type == 'driver':
spark_memory = datalab.fab.get_spark_memory()
@@ -1415,7 +1415,7 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
spark_memory))
if not exists('/opt/spark/conf/spark-env.sh'):
conn.sudo('mv /opt/spark/conf/spark-env.sh.template /opt/spark/conf/spark-env.sh')
- java_home = run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").splitlines()[0]
+ java_home = conn.run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").splitlines()[0]
conn.sudo("echo 'export JAVA_HOME=\'{}\'' >> /opt/spark/conf/spark-env.sh".format(java_home))
if 'spark_configurations' in os.environ:
datalab_header = conn.sudo('cat /tmp/notebook_spark-defaults_local.conf | grep "^#"')
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
index d3d2628..4164b7d 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
@@ -116,7 +116,7 @@ def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak
-out /etc/ssl/certs/datalab.crt -subj "/C=US/ST=US/L=US/O=datalab/CN={}"'.format(hostname))
conn.sudo('mkdir -p /tmp/src')
- with cd('/tmp/src/'):
+ with conn.cd('/tmp/src/'):
conn.sudo('wget https://luarocks.org/releases/luarocks-3.3.1.tar.gz')
conn.sudo('tar -xzf luarocks-3.3.1.tar.gz')
@@ -125,7 +125,7 @@ def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak
conn.sudo('apt-get update')
conn.sudo('apt-get -y install openresty=1.19.3.1-1~focal1')
- with cd('/tmp/src/luarocks-3.3.1/'):
+ with conn.cd('/tmp/src/luarocks-3.3.1/'):
conn.sudo('./configure')
conn.sudo('make install')
conn.sudo('luarocks install lua-resty-jwt 0.2.2 --tree /usr/local/openresty/lualib/resty/')
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index c483ea7..0f1cc7c 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -279,7 +279,7 @@ def install_tensor(os_user, cuda_version, cuda_file_name,
with settings(warn_only=True):
reboot(wait=180)
manage_pkg('-y install', 'remote', 'dkms libglvnd-dev')
- kernel_version = run('uname -r | tr -d "[..0-9-]"')
+ kernel_version = conn.run('uname -r | tr -d "[..0-9-]"')
if kernel_version == 'azure':
manage_pkg('-y install', 'remote', 'linux-modules-`uname -r`')
else:
@@ -300,15 +300,15 @@ def install_tensor(os_user, cuda_version, cuda_file_name,
conn.sudo('ln -s /opt/cuda-{0} /usr/local/cuda-{0}'.format(cuda_version))
conn.sudo('rm -f /opt/{}'.format(cuda_file_name))
# install cuDNN
- run('wget http://developer.download.nvidia.com/compute/redist/cudnn/v{0}/{1} -O /tmp/{1}'.format(
+ conn.run('wget http://developer.download.nvidia.com/compute/redist/cudnn/v{0}/{1} -O /tmp/{1}'.format(
cudnn_version, cudnn_file_name))
- run('tar xvzf /tmp/{} -C /tmp'.format(cudnn_file_name))
+ conn.run('tar xvzf /tmp/{} -C /tmp'.format(cudnn_file_name))
conn.sudo('mkdir -p /opt/cudnn/include')
conn.sudo('mkdir -p /opt/cudnn/lib64')
conn.sudo('mv /tmp/cuda/include/cudnn.h /opt/cudnn/include')
conn.sudo('mv /tmp/cuda/lib64/libcudnn* /opt/cudnn/lib64')
conn.sudo('chmod a+r /opt/cudnn/include/cudnn.h /opt/cudnn/lib64/libcudnn*')
- run(
+ conn.run(
'echo "export LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64\"" >> ~/.bashrc')
# install TensorFlow and run TensorBoard
# conn.sudo('python2.7 -m pip install --upgrade https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp27-none-linux_x86_64.whl --no-cache-dir'.format(tensorflow_version))
@@ -316,8 +316,8 @@ def install_tensor(os_user, cuda_version, cuda_file_name,
conn.sudo('mkdir /var/log/tensorboard; chown {0}:{0} -R /var/log/tensorboard'.format(os_user))
conn.put('{}tensorboard.service'.format(templates_dir), '/tmp/tensorboard.service')
conn.sudo("sed -i 's|OS_USR|{}|' /tmp/tensorboard.service".format(os_user))
- http_proxy = run('echo $http_proxy')
- https_proxy = run('echo $https_proxy')
+ http_proxy = conn.run('echo $http_proxy')
+ https_proxy = conn.run('echo $https_proxy')
conn.sudo('sed -i \'/\[Service\]/ a\Environment=\"HTTP_PROXY={}\"\' /tmp/tensorboard.service'.format(
http_proxy))
conn.sudo('sed -i \'/\[Service\]/ a\Environment=\"HTTPS_PROXY={}\"\' /tmp/tensorboard.service'.format(
@@ -477,11 +477,11 @@ def install_caffe2(os_user, caffe2_version, cmake_version):
conn.sudo('wget https://cmake.org/files/v{2}/cmake-{1}.tar.gz -O /home/{0}/cmake-{1}.tar.gz'.format(
os_user, cmake_version, cmake_version.split('.')[0] + "." + cmake_version.split('.')[1]))
conn.sudo('tar -zxvf cmake-{}.tar.gz'.format(cmake_version))
- with cd('/home/{}/cmake-{}/'.format(os_user, cmake_version)):
+ with conn.cd('/home/{}/cmake-{}/'.format(os_user, cmake_version)):
conn.sudo('./bootstrap --prefix=/usr/local && make && make install')
conn.sudo('ln -s /usr/local/bin/cmake /bin/cmake{}'.format(cmake_version))
conn.sudo('git clone https://github.com/pytorch/pytorch.git')
- with cd('/home/{}/pytorch/'.format(os_user)):
+ with conn.cd('/home/{}/pytorch/'.format(os_user)):
conn.sudo('git submodule update --init')
with settings(warn_only=True):
conn.sudo('git checkout {}'.format(os.environ['notebook_pytorch_branch']))
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
index 34ccebe..c69622a 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
@@ -373,7 +373,7 @@ def install_build_dep():
if not exists('{}tmp/build_dep_ensured'.format(os.environ['ssn_datalab_path'])):
maven_version = '3.5.4'
manage_pkg('-y install', 'remote', 'openjdk-8-jdk git wget unzip')
- with cd('/opt/'):
+ with conn.cd('/opt/'):
conn.sudo(
'wget http://mirrors.sonic.net/apache/maven/maven-{0}/{1}/binaries/apache-maven-{1}-bin.zip'.format(
maven_version.split('.')[0], maven_version))
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index c2ded0c..aced989 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -195,7 +195,7 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
conn.sudo('pip3 install jupyter --no-cache-dir')
conn.sudo('rm -rf {}'.format(jupyter_conf_file))
conn.run('jupyter notebook --generate-config --config {}'.format(jupyter_conf_file))
- with cd('/home/{}'.format(os_user)):
+ with conn.cd('/home/{}'.format(os_user)):
conn.run('mkdir -p ~/.jupyter/custom/')
conn.run('echo "#notebook-container { width: auto; }" > ~/.jupyter/custom/custom.css')
conn.sudo('echo "c.NotebookApp.ip = \'0.0.0.0\'" >> {}'.format(jupyter_conf_file))
@@ -938,7 +938,7 @@ def configure_superset(os_user, keycloak_auth_server_url, keycloak_realm_name, k
print('Superset configuring')
try:
if not exists('/home/{}/incubator-superset'.format(os_user)):
- with cd('/home/{}'.format(os_user)):
+ with conn.cd('/home/{}'.format(os_user)):
conn.sudo('wget https://github.com/apache/incubator-superset/archive/{}.tar.gz'.format(
os.environ['notebook_superset_version']))
conn.sudo('tar -xzf {}.tar.gz'.format(os.environ['notebook_superset_version']))
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
index b4f880f..bd96aa2 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
@@ -65,7 +65,7 @@ def ensure_pkg(user, requisites='git vim gcc python-devel openssl-devel nmap lib
conn.sudo('setenforce 0')
conn.sudo("sed -i '/^SELINUX=/s/SELINUX=.*/SELINUX=disabled/g' /etc/selinux/config")
mirror = 'mirror.centos.org'
- with cd('/etc/yum.repos.d/'):
+ with conn.cd('/etc/yum.repos.d/'):
conn.sudo('echo "[Centos-repo]" > centOS-base.repo')
conn.sudo('echo "name=Centos 7 Repository" >> centOS-base.repo')
conn.sudo('echo "baseurl=http://{}/centos/7/os/x86_64/" >> centOS-base.repo'.format(mirror))
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
index 03bb708..965c4eb 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
@@ -118,7 +118,7 @@ def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak
-out /etc/ssl/certs/datalab.crt -subj "/C=US/ST=US/L=US/O=datalab/CN={}"'.format(hostname))
conn.sudo('mkdir -p /tmp/lua')
conn.sudo('mkdir -p /tmp/src')
- with cd('/tmp/src/'):
+ with conn.cd('/tmp/src/'):
conn.sudo('wget http://nginx.org/download/nginx-{}.tar.gz'.format(nginx_version))
conn.sudo('tar -xzf nginx-{}.tar.gz'.format(nginx_version))
@@ -136,11 +136,11 @@ def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak
conn.sudo('ln -sf nginx-{} nginx'.format(nginx_version))
- with cd('/tmp/src/LuaJIT-2.0.5/'):
+ with conn.cd('/tmp/src/LuaJIT-2.0.5/'):
conn.sudo('make')
conn.sudo('make install')
- with cd('/tmp/src/nginx/'), shell_env(LUAJIT_LIB='/usr/local/lib/', LUAJIT_INC='/usr/local/include/luajit-2.0'):
+ with conn.cd('/tmp/src/nginx/'), shell_env(LUAJIT_LIB='/usr/local/lib/', LUAJIT_INC='/usr/local/include/luajit-2.0'):
conn.sudo('./configure --user=nginx --group=nginx --prefix=/etc/nginx --sbin-path=/usr/sbin/nginx \
--conf-path=/etc/nginx/nginx.conf --pid-path=/run/nginx.pid --lock-path=/run/lock/subsys/nginx \
--error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log \
@@ -151,7 +151,7 @@ def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak
conn.sudo('make')
conn.sudo('make install')
- with cd('/tmp/src/luarocks-2.2.2/'):
+ with conn.cd('/tmp/src/luarocks-2.2.2/'):
conn.sudo('./configure --with-lua-include=/usr/local/include/luajit-2.0')
conn.sudo('make build')
conn.sudo('make install')
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
index 8c5d9a2..e3b3790 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
@@ -249,14 +249,14 @@ def install_tensor(os_user, cuda_version, cuda_file_name,
conn.sudo('ln -s /opt/cuda-{0} /usr/local/cuda-{0}'.format(cuda_version))
conn.sudo('rm -f /opt/{}'.format(cuda_file_name))
# install cuDNN
- run('wget http://developer.download.nvidia.com/compute/redist/cudnn/v{0}/{1} -O /tmp/{1}'.format(cudnn_version, cudnn_file_name))
- run('tar xvzf /tmp/{} -C /tmp'.format(cudnn_file_name))
+ conn.run('wget http://developer.download.nvidia.com/compute/redist/cudnn/v{0}/{1} -O /tmp/{1}'.format(cudnn_version, cudnn_file_name))
+ conn.run('tar xvzf /tmp/{} -C /tmp'.format(cudnn_file_name))
conn.sudo('mkdir -p /opt/cudnn/include')
conn.sudo('mkdir -p /opt/cudnn/lib64')
conn.sudo('mv /tmp/cuda/include/cudnn.h /opt/cudnn/include')
conn.sudo('mv /tmp/cuda/lib64/libcudnn* /opt/cudnn/lib64')
conn.sudo('chmod a+r /opt/cudnn/include/cudnn.h /opt/cudnn/lib64/libcudnn*')
- run('echo "export LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64\"" >> ~/.bashrc')
+ conn.run('echo "export LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64\"" >> ~/.bashrc')
# install TensorFlow and run TensorBoard
conn.sudo('wget https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp27-none-linux_x86_64.whl'.format(tensorflow_version))
conn.sudo('wget https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp35-cp35m-linux_x86_64.whl'.format(tensorflow_version))
@@ -402,13 +402,13 @@ def install_opencv(os_user):
manage_pkg('-y install', 'remote', 'cmake python34 python34-devel python34-pip gcc gcc-c++')
conn.sudo('pip3.4 install numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
conn.sudo('pip3.5 install numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
- run('git clone https://github.com/opencv/opencv.git')
- with cd('/home/{}/opencv/'.format(os_user)):
- run('git checkout 3.2.0')
- run('mkdir release')
- with cd('/home/{}/opencv/release/'.format(os_user)):
- run('cmake -DINSTALL_TESTS=OFF -D CUDA_GENERATION=Auto -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=$(python2 -c "import sys; print(sys.prefix)") -D PYTHON_EXECUTABLE=$(which python2) ..')
- run('make -j$(nproc)')
+ conn.run('git clone https://github.com/opencv/opencv.git')
+ with conn.cd('/home/{}/opencv/'.format(os_user)):
+ conn.run('git checkout 3.2.0')
+ conn.run('mkdir release')
+ with conn.cd('/home/{}/opencv/release/'.format(os_user)):
+ conn.run('cmake -DINSTALL_TESTS=OFF -D CUDA_GENERATION=Auto -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=$(python2 -c "import sys; print(sys.prefix)") -D PYTHON_EXECUTABLE=$(which python2) ..')
+ conn.run('make -j$(nproc)')
conn.sudo('make install')
conn.sudo('touch /home/' + os_user + '/.ensure_dir/opencv_ensured')
@@ -425,11 +425,11 @@ def install_caffe2(os_user, caffe2_version, cmake_version):
conn.sudo('wget https://cmake.org/files/v{2}/cmake-{1}.tar.gz -O /home/{0}/cmake-{1}.tar.gz'.format(
os_user, cmake_version, cmake_version.split('.')[0] + "." + cmake_version.split('.')[1]))
conn.sudo('tar -zxvf cmake-{}.tar.gz'.format(cmake_version))
- with cd('/home/{}/cmake-{}/'.format(os_user, cmake_version)):
+ with conn.cd('/home/{}/cmake-{}/'.format(os_user, cmake_version)):
conn.sudo('./bootstrap --prefix=/usr/local && make && make install')
conn.sudo('ln -s /usr/local/bin/cmake /bin/cmake{}'.format(cmake_version))
conn.sudo('git clone https://github.com/pytorch/pytorch.git')
- with cd('/home/{}/pytorch/'.format(os_user)):
+ with conn.cd('/home/{}/pytorch/'.format(os_user)):
conn.sudo('git submodule update --init')
with settings(warn_only=True):
conn.sudo('git checkout v{}'.format(caffe2_version))
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
index c7733d6..c2ee35b 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
@@ -40,7 +40,7 @@ def ensure_docker_daemon(datalab_path, os_user, region):
mirror = 'mirror.lzu.edu.cn'
else:
mirror = 'mirror.centos.org'
- with cd('/etc/yum.repos.d/'):
+ with conn.cd('/etc/yum.repos.d/'):
conn.sudo('echo "[centosrepo]" > centos.repo')
conn.sudo('echo "name=Centos 7 Repository" >> centos.repo')
conn.sudo('echo "baseurl=http://{}/centos/7/extras/x86_64/" >> centos.repo'.format(mirror))
@@ -389,7 +389,7 @@ def install_build_dep():
if not exists('{}tmp/build_dep_ensured'.format(os.environ['ssn_datalab_path'])):
maven_version = '3.5.4'
manage_pkg('-y install', 'remote', 'java-1.8.0-openjdk java-1.8.0-openjdk-devel git wget unzip')
- with cd('/opt/'):
+ with conn.cd('/opt/'):
conn.sudo(
'wget http://mirrors.sonic.net/apache/maven/maven-{0}/{1}/binaries/apache-maven-{1}-bin.zip'.format(
maven_version.split('.')[0], maven_version))
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
index d22ce7f..f022b1a 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
@@ -57,9 +57,9 @@ def configure_notebook(args):
conn.sudo('\cp /tmp/create_configs.py /usr/local/bin/create_configs.py')
conn.sudo('chmod 755 /usr/local/bin/create_configs.py')
conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
- run('mkdir -p /tmp/datalab_libs/')
+ conn.run('mkdir -p /tmp/datalab_libs/')
local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
- run('chmod a+x /tmp/datalab_libs/*')
+ conn.run('chmod a+x /tmp/datalab_libs/*')
conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
conn.sudo('mkdir -p /usr/lib64/python3.8')
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py b/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
index 4dabf7d..592c98e 100644
--- a/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
+++ b/infrastructure-provisioning/src/general/scripts/os/jupyterlab_container_start.py
@@ -37,7 +37,7 @@ jupyterlab_dir = '/home/' + args.os_user + '/.jupyterlab/'
def start_jupyterlab_container(jupyterlab_dir):
try:
- with cd('{}'.format(jupyterlab_dir)):
+ with conn.cd('{}'.format(jupyterlab_dir)):
conn.run('docker build --network=host --file Dockerfile_jupyterlab -t jupyter-lab .'.format(args.os_user))
container_id = conn.run('docker ps | awk \'NR==2{print $1}\'')
if container_id != '':
diff --git a/infrastructure-provisioning/src/general/scripts/os/superset_start.py b/infrastructure-provisioning/src/general/scripts/os/superset_start.py
index 611715c..e2e974d 100644
--- a/infrastructure-provisioning/src/general/scripts/os/superset_start.py
+++ b/infrastructure-provisioning/src/general/scripts/os/superset_start.py
@@ -37,7 +37,7 @@ superset_dir = '/home/' + args.os_user + '/incubator-superset/contrib/docker'
def start_superset(superset_dir):
try:
- with cd('{}'.format(superset_dir)):
+ with conn.cd('{}'.format(superset_dir)):
conn.sudo('docker-compose run --rm superset ./docker-init.sh')
conn.sudo('cp /opt/datalab/templates/superset-notebook.service /tmp/')
conn.sudo('sed -i \'s/OS_USER/{}/g\' /tmp/superset-notebook.service'.format(args.os_user))
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
index afc5851..419b521 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
@@ -74,7 +74,7 @@ def download_toree():
def add_china_repository(datalab_path):
- with cd('{}sources/infrastructure-provisioning/src/base/'.format(datalab_path)):
+ with conn.cd('{}sources/infrastructure-provisioning/src/base/'.format(datalab_path)):
conn.sudo('sed -i "/pip install/s/$/ -i https\:\/\/{0}\/simple --trusted-host {0} --timeout 60000/g" '
'Dockerfile'.format(os.environ['conf_pypi_mirror']))
conn.sudo('sed -i "/pip install/s/jupyter/ipython==5.0.0 jupyter==1.0.0/g" Dockerfile')
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py b/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
index 9b59319..c52b7cb 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
@@ -73,7 +73,7 @@ def cp_key(keyfile, host_string, os_user):
def cp_backup_scripts(datalab_path):
try:
- with cd(datalab_path + "tmp/"):
+ with conn.cd(datalab_path + "tmp/"):
conn.put('/root/scripts/backup.py', "backup.py")
conn.put('/root/scripts/restore.py', "restore.py")
conn.run('chmod +x backup.py restore.py')
@@ -87,7 +87,7 @@ def cp_gitlab_scripts(datalab_path):
try:
if not exists('{}tmp/gitlab'.format(datalab_path)):
conn.run('mkdir -p {}tmp/gitlab'.format(datalab_path))
- with cd('{}tmp/gitlab'.format(datalab_path)):
+ with conn.cd('{}tmp/gitlab'.format(datalab_path)):
conn.put('/root/scripts/gitlab_deploy.py', 'gitlab_deploy.py')
conn.put('/root/scripts/configure_gitlab.py', 'configure_gitlab.py')
conn.run('chmod +x gitlab_deploy.py configure_gitlab.py')
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
index cb85f29..b920aac 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
@@ -146,7 +146,7 @@ def configure_mongo(mongo_passwd, default_endpoint_name):
def build_ui():
try:
# Building Front-end
- with cd(args.datalab_path + '/sources/services/self-service/src/main/resources/webapp/'):
+ with conn.cd(args.datalab_path + '/sources/services/self-service/src/main/resources/webapp/'):
conn.sudo('sed -i "s|CLOUD_PROVIDER|{}|g" src/dictionary/global.dictionary.ts'.format(args.cloud_provider))
if args.cloud_provider == 'azure' and os.environ['azure_datalake_enable'] == 'true':
@@ -158,7 +158,7 @@ def build_ui():
conn.sudo('sudo chown -R {} {}/*'.format(args.os_user, args.datalab_path))
# Building Back-end
- with cd(args.datalab_path + '/sources/'):
+ with conn.cd(args.datalab_path + '/sources/'):
conn.sudo('/opt/maven/bin/mvn -P{} -DskipTests package'.format(args.cloud_provider))
conn.sudo('mkdir -p {}/webapp/'.format(args.datalab_path))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org