You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by mh...@apache.org on 2022/03/11 08:59:07 UTC
[incubator-datalab] branch gcp-jupyter-highgpu-template-1 updated: ensure_anaconda function
This is an automated email from the ASF dual-hosted git repository.
mhladun pushed a commit to branch gcp-jupyter-highgpu-template-1
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/gcp-jupyter-highgpu-template-1 by this push:
new 7f700d7 ensure_anaconda function
new 71d969a Merge branch 'gcp-jupyter-highgpu-template-1' of https://github.com/apache/incubator-datalab into gcp-jupyter-highgpu-template-1
7f700d7 is described below
commit 7f700d775a882468cf259578dad3fb7c0d7cd148
Author: Marian_Hladun <ma...@gmail.com>
AuthorDate: Fri Mar 11 10:58:21 2022 +0200
ensure_anaconda function
---
infrastructure-provisioning/src/general/lib/os/fab.py | 18 ++++++++++++++++++
.../scripts/configure_jupyter-gpu-conda_node.py | 2 ++
2 files changed, 20 insertions(+)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index f77dfa3..8c5ed3a 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -379,6 +379,24 @@ def ensure_python_venv(python_venv_version):
traceback.print_exc()
sys.exit(1)
+def ensure_anaconda():
+ try:
+ if not exists(conn, '/opt/anaconda3'):
+ conn.sudo('wget https://repo.anaconda.com/archive/Anaconda3-2021.11-Linux-x86_64.sh -O /tmp/anaconda.sh')
+ conn.sudo('bash /tmp/anaconda.sh -b -p /opt/anaconda3')
+ conn.sudo(''' bash -l -c "echo 'export PATH=/opt/anaconda3/bin/:$PATH' >> /home/datalab-user/.bashrc" ''')
+ conn.sudo('bash -l -c "source /home/datalab-user/.bashrc"')
+ conn.sudo('chown -R datalab-user /opt/anaconda3')
+ conn.run('conda create -y -p /opt/anaconda3/envs/jupyter-gpu-conda numpy scipy pandas scikit-learn git pip')
+ conn.run('conda activate jupyter-gpu-conda && pip install transformers==4.4.2 gensim==4.0.1 tokenizers==0.10.1 python-levenshtein==0.12.2')
+ conn.run("bash -l -c 'pip install -U torch==1.10.0+cu111 torchvision==0.11.3+cu111 torchaudio==0.10.2+cu111 -f https://download.pytorch.org/whl/cu111/torch_stable.html --no-cache-dir'")
+ conn.run('conda activate jupyter-gpu-conda && conda install -y ipykernel -c anaconda')
+ conn.run('conda activate jupyter-gpu-conda && python -m ipykernel install --user --name=jupyter-gpu-conda')
+ conn.sudo('systemctl restart jupyter-notebook')
+ except Exception as err:
+ logging.error('Function ensure_anaconda error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
def install_venv_pip_pkg(pkg_name, pkg_version=''):
try:
diff --git a/infrastructure-provisioning/src/jupyter-gpu-conda/scripts/configure_jupyter-gpu-conda_node.py b/infrastructure-provisioning/src/jupyter-gpu-conda/scripts/configure_jupyter-gpu-conda_node.py
index a40fc81..74aa40f 100644
--- a/infrastructure-provisioning/src/jupyter-gpu-conda/scripts/configure_jupyter-gpu-conda_node.py
+++ b/infrastructure-provisioning/src/jupyter-gpu-conda/scripts/configure_jupyter-gpu-conda_node.py
@@ -120,6 +120,8 @@ if __name__ == "__main__":
ensure_py3spark_local_kernel(args.os_user, py3spark_local_path_dir, templates_dir, spark_version, python_venv_path, python_venv_version)
print("Install Toree-Scala kernel for Jupyter")
ensure_toree_local_kernel(args.os_user, toree_link, scala_kernel_path, files_dir, local_spark_scala_version, spark_version)
+ print("Install Anaconda and add kernel for Jupyter")
+ ensure_anaconda()
# INSTALL UNGIT
print("Install nodejs")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org