You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by ol...@apache.org on 2022/02/15 14:44:35 UTC
[incubator-datalab] branch 2.5.1-tensor-jupyterlab updated: service file
This is an automated email from the ASF dual-hosted git repository.
olehmykolaishyn pushed a commit to branch 2.5.1-tensor-jupyterlab
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/2.5.1-tensor-jupyterlab by this push:
new 96093f1 service file
96093f1 is described below
commit 96093f1467a8e0f5715e00efc7577e558ca41d04
Author: oleh_mykolaishyn <ow...@gmail.com>
AuthorDate: Tue Feb 15 16:43:55 2022 +0200
service file
---
.../general/files/aws/tensor-jupyterlab_Dockerfile | 2 +-
.../src/general/lib/os/fab.py | 45 +++++++++++-----------
.../os/debian/jupyterlab-notebook.service | 35 +++++++++++++++++
.../scripts/configure_tensor-jupyterlab_node.py | 10 ++---
4 files changed, 64 insertions(+), 28 deletions(-)
diff --git a/infrastructure-provisioning/src/general/files/aws/tensor-jupyterlab_Dockerfile b/infrastructure-provisioning/src/general/files/aws/tensor-jupyterlab_Dockerfile
index 4c9c035..eee2fa3 100644
--- a/infrastructure-provisioning/src/general/files/aws/tensor-jupyterlab_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/tensor-jupyterlab_Dockerfile
@@ -28,7 +28,7 @@ COPY tensor/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/aws/tensor_* /root/scripts/
COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
-COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/ # add jupyterlab-service file
+COPY general/templates/os/${OS}/jupyterlab-notebook.service /root/templates/ # add jupyterlab-service file
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
COPY general/templates/os/pyspark_local_template.json /root/templates/
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 6444a7b..6d0c430 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -1043,36 +1043,37 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
sys.exit(1)
+# jupyterlab
def configure_jupyterlab(os_user, jupyterlab_conf_file, templates_dir, jupyterlab_version, exploratory_name):
if not exists(conn, '/home/' + os_user + '/.ensure_dir/jupyterlab_ensured'):
try:
# conn.sudo('pip3 install notebook=={} --no-cache-dir'.format(jupyter_version))
- conn.sudo('source /opt/python/python{0}/bin/activate && /opt/python/python{0}/bin/pip{1}').format(python_version, pip_version)
+ conn.sudo('source /opt/python/python3.7.9/bin/activate')
#conn.sudo('pip3 install jupyterlab --no-cache-dir') # create external var with version
conn.sudo('rm -rf {}'.format(jupyterlab_conf_file))
#elif os.environ['application'] != 'tensor':
# conn.sudo('pip3 install environment_kernels')
#if os.environ['conf_cloud_provider'] == 'aws' and os.environ['application'] == 'deeplearning': #should be checked if for other applications any files have owner root:root in datalab-user homefolder and where it is changed to root:root on deeplearning
# conn.sudo('chown -R {0}:{0} /home/{0}/.local'.format(os_user))
- conn.run('jupyter lab --generate-config --config {}'.format(jupyter_conf_file))
- conn.run('mkdir -p ~/.jupyter/custom/')
- conn.run('echo "#notebook-container { width: auto; }" > ~/.jupyter/custom/custom.css')
- conn.sudo('echo "c.NotebookApp.ip = \'0.0.0.0\'" >> {}'.format(jupyter_conf_file))
- conn.sudo('echo "c.NotebookApp.base_url = \'/{0}/\'" >> {1}'.format(exploratory_name, jupyter_conf_file))
- conn.sudo('echo c.NotebookApp.open_browser = False >> {}'.format(jupyter_conf_file))
- conn.sudo('echo \'c.NotebookApp.cookie_secret = b"{0}"\' >> {1}'.format(id_generator(), jupyter_conf_file))
- conn.sudo('''echo "c.NotebookApp.token = u''" >> {}'''.format(jupyter_conf_file))
- conn.sudo('echo \'c.KernelSpecManager.ensure_native_kernel = False\' >> {}'.format(jupyter_conf_file))
- if os.environ['conf_deeplearning_cloud_ami'] == 'true' and os.environ['application'] == 'deeplearning':
- conn.sudo(
- '''echo "c.NotebookApp.kernel_spec_manager_class = 'environment_kernels.EnvironmentKernelSpecManager'" >> {}'''.format(
- jupyter_conf_file))
- conn.sudo(
- '''echo "c.EnvironmentKernelSpecManager.conda_env_dirs=['/home/ubuntu/anaconda3/envs']" >> {}'''.format(
- jupyter_conf_file))
- conn.put(templates_dir + 'jupyter-notebook.service', '/tmp/jupyter-notebook.service')
- conn.sudo("chmod 644 /tmp/jupyter-notebook.service")
- if os.environ['application'] == 'tensor':
+ conn.run('jupyter lab --generate-config')
+ #conn.run('mkdir -p ~/.jupyter/custom/')
+ #conn.run('echo "#notebook-container { width: auto; }" > ~/.jupyter/custom/custom.css')
+ conn.sudo('echo "c.NotebookApp.ip = \'0.0.0.0\'" >> {}'.format(jupyterlab_conf_file))
+ conn.sudo('echo "c.NotebookApp.base_url = \'/{0}/\'" >> {1}'.format(exploratory_name, jupyterlab_conf_file))
+ conn.sudo('echo c.NotebookApp.open_browser = False >> {}'.format(jupyterlab_conf_file))
+ conn.sudo('echo \'c.NotebookApp.cookie_secret = b"{0}"\' >> {1}'.format(id_generator(), jupyterlab_conf_file))
+ conn.sudo('''echo "c.NotebookApp.token = u''" >> {}'''.format(jupyterlab_conf_file))
+ conn.sudo('echo \'c.KernelSpecManager.ensure_native_kernel = False\' >> {}'.format(jupyterlab_conf_file))
+ #if os.environ['conf_deeplearning_cloud_ami'] == 'true' and os.environ['application'] == 'deeplearning':
+ # conn.sudo(
+ # '''echo "c.NotebookApp.kernel_spec_manager_class = 'environment_kernels.EnvironmentKernelSpecManager'" >> {}'''.format(
+ # jupyter_conf_file))
+ # conn.sudo(
+ # '''echo "c.EnvironmentKernelSpecManager.conda_env_dirs=['/home/ubuntu/anaconda3/envs']" >> {}'''.format(
+ # jupyter_conf_file))
+ #conn.put(templates_dir + 'jupyter-notebook.service', '/tmp/jupyter-notebook.service')
+ #conn.sudo("chmod 644 /tmp/jupyter-notebook.service")
+ if os.environ['application'] == 'tensor': # ?????
conn.sudo(
"sed -i '/ExecStart/s|-c \"|-c \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64; |g' /tmp/jupyter-notebook.service")
elif os.environ['application'] == 'deeplearning' and os.environ['conf_deeplearning_cloud_ami'] == 'false':
@@ -1085,9 +1086,9 @@ def configure_jupyterlab(os_user, jupyterlab_conf_file, templates_dir, jupyterla
java_home = conn.run("update-alternatives --query java | grep -o --color=never \'/.*/java-11.*/bin/java\'").stdout.splitlines()[0]
else:
java_home = conn.run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").stdout.splitlines()[0]
- conn.sudo('sed -i \'/\[Service\]/ a\Environment=\"JAVA_HOME={}\"\' /tmp/jupyter-notebook.service'.format(
+ conn.sudo('sed -i \'/\[Service\]/ a\Environment=\"JAVA_HOME={}\"\' /tmp/jupyterlab-notebook.service'.format(
java_home))
- conn.sudo('\cp /tmp/jupyter-notebook.service /etc/systemd/system/jupyter-notebook.service')
+ conn.sudo('\cp /tmp/jupyterlab-notebook.service /etc/systemd/system/jupyterlab-notebook.service')
conn.sudo('chown -R {0}:{0} /home/{0}/.local'.format(os_user))
conn.sudo('mkdir -p /mnt/var')
conn.sudo('chown {0}:{0} /mnt/var'.format(os_user))
diff --git a/infrastructure-provisioning/src/general/templates/os/debian/jupyterlab-notebook.service b/infrastructure-provisioning/src/general/templates/os/debian/jupyterlab-notebook.service
new file mode 100644
index 0000000..c5e2225
--- /dev/null
+++ b/infrastructure-provisioning/src/general/templates/os/debian/jupyterlab-notebook.service
@@ -0,0 +1,35 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+[Unit]
+Description=Jupyterlab notebook
+
+[Service]
+Type=simple
+PIDFile=/var/run/jupyterlab-notebook.pid
+ExecStart=/bin/bash -c "/usr/local/bin/jupyter lab
+ExecStop=/bin/bash -c "for i in $(ps aux | grep jupyter | grep -v grep | awk '{print $2}'); do kill -9 $i; done"
+User=OS_USR
+Group=OS_USR
+WorkingDirectory=/home/OS_USR
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/tensor-jupyterlab/scripts/configure_tensor-jupyterlab_node.py b/infrastructure-provisioning/src/tensor-jupyterlab/scripts/configure_tensor-jupyterlab_node.py
index 3258a89..fef030a 100644
--- a/infrastructure-provisioning/src/tensor-jupyterlab/scripts/configure_tensor-jupyterlab_node.py
+++ b/infrastructure-provisioning/src/tensor-jupyterlab/scripts/configure_tensor-jupyterlab_node.py
@@ -63,7 +63,7 @@ local_spark_path = '/opt/spark/'
jars_dir = '/opt/jars/'
templates_dir = '/root/templates/'
files_dir = '/root/files/'
-#jupyterlab_conf_file = '/home/' + args.os_user + '/.local/share/jupyter/jupyterlab_notebook_config.py'
+jupyterlab_conf_file = '/home/' + args.os_user + '/.jupyter/jupyter_notebook_config.py'
#
gitlab_certfile = os.environ['conf_gitlab_certfile']
@@ -71,7 +71,7 @@ cuda_version = os.environ['notebook_cuda_version']
cuda_file_name = os.environ['notebook_cuda_file_name']
cudnn_version = os.environ['notebook_cudnn_version']
cudnn_file_name = os.environ['notebook_cudnn_file_name']
-venv_libs = 'numpy scipy matplotlib pandas scikit-learn opencv-python'
+venv_libs = 'numpy scipy matplotlib pandas scikit-learn opencv-python' # python3-opencv
jupyterlab_pip = 'jupyterlab'
##############
@@ -113,8 +113,8 @@ if __name__ == "__main__":
install_keras(args.os_user, keras_version)
# INSTALL JUPYTER NOTEBOOK
- print("Install Jupyter")
- configure_jupyterlab(args.os_user, jupyterlab_conf_file, templates_dir, jupyterlab_version, args.exploratory_name)
+ #print("Install Jupyter")
+ #configure_jupyterlab(args.os_user, jupyterlab_conf_file, templates_dir, jupyterlab_version, args.exploratory_name)
# INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK
print("Install local Spark")
@@ -154,7 +154,7 @@ if __name__ == "__main__":
print("Install python venv required libs")
ensure_venv_libs(args.os_user, venv_libs)
- print("Install python venv required libs")
+ print("Install jupyterlab")
ensure_venv_libs(args.os_user, 'jupyterlab')
#POST INSTALLATION PROCESS
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org