You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by my...@apache.org on 2021/04/02 10:36:27 UTC
[incubator-datalab] branch DATALAB-2091-with-dataproc updated:
[DataLab-2091] - [GCP] Jupyter/Dataengine kernels for Jupyter fixed
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DATALAB-2091-with-dataproc
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/DATALAB-2091-with-dataproc by this push:
new 119cae5 [DataLab-2091] - [GCP] Jupyter/Dataengine kernels for Jupyter fixed
119cae5 is described below
commit 119cae5e13fcfca27e0555762e4f080eb0d78cc0
Author: bodnarmykola <bo...@gmail.com>
AuthorDate: Fri Apr 2 13:36:06 2021 +0300
[DataLab-2091] - [GCP] Jupyter/Dataengine kernels for Jupyter fixed
---
.../src/general/lib/os/fab.py | 20 ++++++++++----------
.../src/general/templates/os/livy-env.sh | 3 ++-
.../src/jupyter/scripts/configure_jupyter_node.py | 2 ++
3 files changed, 14 insertions(+), 11 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 87625bc..368b949 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -228,7 +228,6 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
conn.sudo('mkdir -p /mnt/var')
conn.sudo('chown {0}:{0} /mnt/var'.format(os_user))
if os.environ['application'] == 'jupyter':
- conn.sudo('jupyter-kernelspec remove -f python2 || echo "Such kernel doesnt exists"')
conn.sudo('jupyter-kernelspec remove -f python3 || echo "Such kernel doesnt exists"')
conn.sudo("systemctl daemon-reload")
conn.sudo("systemctl enable jupyter-notebook")
@@ -245,6 +244,14 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
print('Error:', str(err))
sys.exit(1)
+
+def remove_unexisting_kernel():
+ try:
+ conn.sudo('jupyter-kernelspec remove -f python3')
+ except Exception as err:
+ print('Error:', str(err))
+ sys.exit(1)
+
def configure_docker(os_user):
try:
if not exists(conn,'/home/' + os_user + '/.ensure_dir/docker_ensured'):
@@ -832,15 +839,8 @@ def get_spark_memory(creds=False, os_user='', hostname='', keyfile=''):
else:
mem = conn.sudo('free -m | grep Mem | tr -s " " ":" | cut -f 2 -d ":"').stdout.replace('\n','')
instance_memory = int(mem)
- try:
- if instance_memory > int(os.environ['dataengine_expl_instance_memory']):
- spark_memory = instance_memory - int(os.environ['dataengine_os_expl_memory'])
- else:
- spark_memory = instance_memory * int(os.environ['dataengine_os_memory']) / 100
- return spark_memory
- except Exception as err:
- print('Error:', str(err))
- return err
+ spark_memory = round(instance_memory*90/100)
+ return spark_memory
def replace_multi_symbols(string, symbol, symbol_cut=False):
diff --git a/infrastructure-provisioning/src/general/templates/os/livy-env.sh b/infrastructure-provisioning/src/general/templates/os/livy-env.sh
index e9a5738..38dfc4d 100644
--- a/infrastructure-provisioning/src/general/templates/os/livy-env.sh
+++ b/infrastructure-provisioning/src/general/templates/os/livy-env.sh
@@ -19,4 +19,5 @@
#
export SPARK_HOME=/opt/spark
export SPARK_CONF_DIR=/opt/spark/conf
-export LIVY_LOG_DIR=/var/log/livy
\ No newline at end of file
+export LIVY_LOG_DIR=/var/log/livy
+export PYSPARK_PYTHON=python3
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py b/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
index a8f0cde..4848951 100644
--- a/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
+++ b/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
@@ -147,5 +147,7 @@ if __name__ == "__main__":
#POST INSTALLATION PROCESS
print("Updating pyOpenSSL library")
update_pyopenssl_lib(args.os_user)
+ print("Removing unexisting kernels")
+ removing_unexisting_kernel()
conn.close()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org