You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/01/28 13:34:24 UTC
[incubator-datalab] branch DATALAB-2091 updated: [DATALAB-2091]:
replaced python3.6 with python 3.8
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/DATALAB-2091 by this push:
new 4a899ea [DATALAB-2091]: replaced python3.6 with python 3.8
4a899ea is described below
commit 4a899ea8981a57617393fdc7eaec6c56c8df7e6c
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Jan 28 15:34:08 2021 +0200
[DATALAB-2091]: replaced python3.6 with python 3.8
---
README.md | 2 +-
.../src/general/files/aws/base_Dockerfile | 8 ++++----
.../src/general/files/aws/dataengine-service_Dockerfile | 4 ++--
.../src/general/files/aws/dataengine_Dockerfile | 2 +-
.../src/general/files/aws/deeplearning_Dockerfile | 2 +-
.../src/general/files/aws/edge_Dockerfile | 2 +-
.../src/general/files/aws/jupyter_Dockerfile | 2 +-
.../src/general/files/aws/jupyterlab_Dockerfile | 2 +-
.../src/general/files/aws/project_Dockerfile | 2 +-
.../src/general/files/aws/rstudio_Dockerfile | 2 +-
.../src/general/files/aws/ssn_Dockerfile | 2 +-
.../src/general/files/aws/tensor-rstudio_Dockerfile | 2 +-
.../src/general/files/aws/tensor_Dockerfile | 2 +-
.../src/general/files/aws/zeppelin_Dockerfile | 2 +-
.../src/general/files/azure/base_Dockerfile | 8 ++++----
.../src/general/files/azure/dataengine_Dockerfile | 2 +-
.../src/general/files/azure/deeplearning_Dockerfile | 2 +-
.../src/general/files/azure/edge_Dockerfile | 2 +-
.../src/general/files/azure/jupyter_Dockerfile | 2 +-
.../src/general/files/azure/jupyterlab_Dockerfile | 2 +-
.../src/general/files/azure/project_Dockerfile | 2 +-
.../src/general/files/azure/rstudio_Dockerfile | 2 +-
.../src/general/files/azure/ssn_Dockerfile | 2 +-
.../src/general/files/azure/tensor_Dockerfile | 2 +-
.../src/general/files/azure/zeppelin_Dockerfile | 2 +-
.../src/general/files/gcp/base_Dockerfile | 8 ++++----
.../src/general/files/gcp/dataengine-service_Dockerfile | 2 +-
.../src/general/files/gcp/dataengine_Dockerfile | 2 +-
.../src/general/files/gcp/deeplearning_Dockerfile | 2 +-
.../src/general/files/gcp/edge_Dockerfile | 2 +-
.../src/general/files/gcp/jupyter_Dockerfile | 2 +-
.../src/general/files/gcp/jupyterlab_Dockerfile | 2 +-
.../src/general/files/gcp/project_Dockerfile | 2 +-
.../src/general/files/gcp/rstudio_Dockerfile | 2 +-
.../src/general/files/gcp/ssn_Dockerfile | 2 +-
.../src/general/files/gcp/superset_Dockerfile | 2 +-
.../src/general/files/gcp/tensor-rstudio_Dockerfile | 2 +-
.../src/general/files/gcp/tensor_Dockerfile | 2 +-
.../src/general/files/gcp/zeppelin_Dockerfile | 2 +-
.../src/general/lib/os/debian/notebook_lib.py | 2 +-
infrastructure-provisioning/src/general/lib/os/fab.py | 8 +++++---
.../src/general/lib/os/redhat/notebook_lib.py | 8 ++++----
.../aws/jupyter_install_dataengine-service_kernels.py | 8 ++++----
.../aws/rstudio_install_dataengine-service_kernels.py | 8 ++++----
.../aws/zeppelin_install_dataengine-service_kernels.py | 8 ++++----
.../gcp/jupyter_install_dataengine-service_kernels.py | 8 ++++----
.../gcp/rstudio_install_dataengine-service_kernels.py | 8 ++++----
.../gcp/zeppelin_install_dataengine-service_kernels.py | 8 ++++----
.../scripts/os/deeplearning_dataengine_create_configs.py | 14 +++++++-------
.../scripts/os/deeplearning_install_dataengine_kernels.py | 10 +++++-----
.../src/general/scripts/os/get_list_available_pkgs.py | 4 ++--
.../scripts/os/jupyter_dataengine_create_configs.py | 14 +++++++-------
.../scripts/os/jupyter_install_dataengine_kernels.py | 10 +++++-----
.../scripts/os/rstudio_install_dataengine_kernels.py | 10 +++++-----
.../os/tensor-rstudio_install_dataengine_kernels.py | 10 +++++-----
.../general/scripts/os/tensor_dataengine_create_configs.py | 14 +++++++-------
.../scripts/os/tensor_install_dataengine_kernels.py | 10 +++++-----
.../scripts/os/zeppelin_dataengine_create_configs.py | 4 ++--
.../scripts/os/zeppelin_install_dataengine_kernels.py | 10 +++++-----
.../src/general/templates/os/py3spark_local_template.json | 4 ++--
.../src/general/templates/os/pyspark_local_template.json | 6 +++---
.../src/ssn/scripts/configure_ui.py | 8 ++++----
.../terraform/gcp/ssn-gke/README.md | 2 +-
63 files changed, 148 insertions(+), 146 deletions(-)
diff --git a/README.md b/README.md
index ed6535a..24d52b5 100644
--- a/README.md
+++ b/README.md
@@ -2432,7 +2432,7 @@ ARG OS
COPY ssn/ /root/
COPY general/scripts/aws/ssn_* /root/scripts/
-COPY general/lib/os/${OS}/ssn_lib.py /usr/lib/python3.6/datalab/ssn_lib.py
+COPY general/lib/os/${OS}/ssn_lib.py /usr/lib/python3.8/datalab/ssn_lib.py
COPY general/files/aws/ssn_policy.json /root/files/
COPY general/templates/aws/jenkins_jobs /root/templates/jenkins_jobs
diff --git a/infrastructure-provisioning/src/general/files/aws/base_Dockerfile b/infrastructure-provisioning/src/general/files/aws/base_Dockerfile
index 58bb0c5..3a2ef91 100644
--- a/infrastructure-provisioning/src/general/files/aws/base_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/base_Dockerfile
@@ -58,16 +58,16 @@ RUN mkdir -p /root/conf && \
mkdir -p /root/scripts && \
mkdir -p /root/templates && \
mkdir -p /root/files && \
- mkdir -p /usr/lib/python3.6/datalab && \
+ mkdir -p /usr/lib/python3.8/datalab && \
mkdir -p /root/keys/.ssh
COPY ${SRC_PATH}base/ /root
COPY ${SRC_PATH}general/conf/* /root/conf/
COPY ${SRC_PATH}general/api/*.py /bin/
COPY ${SRC_PATH}general/scripts/aws/common_* /root/scripts/
-COPY ${SRC_PATH}general/lib/aws/* /usr/lib/python3.6/datalab/
-COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python3.6/datalab/common_lib.py
-COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python3.6/datalab/fab.py
+COPY ${SRC_PATH}general/lib/aws/* /usr/lib/python3.8/datalab/
+COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python3.8/datalab/common_lib.py
+COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python3.8/datalab/fab.py
COPY ${SRC_PATH}general/files/os/${OS}/sources.list /root/files/
COPY ${SRC_PATH}general/files/os/ivysettings.xml /root/templates/
COPY ${SRC_PATH}general/files/os/local_endpoint.json /root/files/
diff --git a/infrastructure-provisioning/src/general/files/aws/dataengine-service_Dockerfile b/infrastructure-provisioning/src/general/files/aws/dataengine-service_Dockerfile
index 065015d..5cbefa5 100644
--- a/infrastructure-provisioning/src/general/files/aws/dataengine-service_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/dataengine-service_Dockerfile
@@ -30,8 +30,8 @@ COPY general/scripts/os/common_configure_proxy.py /root/scripts/
COPY general/scripts/os/install_additional_libs.py /root/scripts/install_additional_libs.py
COPY general/scripts/os/get_list_available_pkgs.py /root/scripts/get_list_available_pkgs.py
COPY general/scripts/os/common_* /root/scripts/
-COPY general/lib/os/redhat/common_lib.py /usr/lib/python3.6/datalab/common_lib.py
-COPY general/lib/os/redhat/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/redhat/common_lib.py /usr/lib/python3.8/datalab/common_lib.py
+COPY general/lib/os/redhat/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/inactive.sh /root/templates/
COPY general/templates/os/inactive.service /root/templates/
COPY general/templates/os/inactive.timer /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/dataengine_Dockerfile b/infrastructure-provisioning/src/general/files/aws/dataengine_Dockerfile
index 45044c7..5af2a5c 100644
--- a/infrastructure-provisioning/src/general/files/aws/dataengine_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/dataengine_Dockerfile
@@ -29,7 +29,7 @@ COPY general/scripts/os/update_inactivity_on_start.py /root/scripts/
COPY general/scripts/os/reconfigure_spark.py /root/scripts/
COPY general/scripts/os/install_additional_libs.py /root/scripts/install_additional_libs.py
COPY general/scripts/os/get_list_available_pkgs.py /root/scripts/get_list_available_pkgs.py
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/scripts/os/common_* /root/scripts/
COPY general/scripts/aws/dataengine_* /root/scripts/
COPY general/scripts/os/notebook_reconfigure_dataengine_spark.py /root/scripts/
diff --git a/infrastructure-provisioning/src/general/files/aws/deeplearning_Dockerfile b/infrastructure-provisioning/src/general/files/aws/deeplearning_Dockerfile
index 8470fc6..0df60c4 100644
--- a/infrastructure-provisioning/src/general/files/aws/deeplearning_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/deeplearning_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY deeplearning/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/aws/deeplearning_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/edge_Dockerfile b/infrastructure-provisioning/src/general/files/aws/edge_Dockerfile
index 82012c4..c31e48e 100644
--- a/infrastructure-provisioning/src/general/files/aws/edge_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/edge_Dockerfile
@@ -26,7 +26,7 @@ ARG OS
COPY edge/ /root/
COPY general/scripts/aws/edge_* /root/scripts/
-COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python3.6/datalab/edge_lib.py
+COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python3.8/datalab/edge_lib.py
COPY general/templates/aws/edge_s3_policy.json /root/templates/edge_s3_policy.json
RUN chmod a+x /root/fabfile.py; \
diff --git a/infrastructure-provisioning/src/general/files/aws/jupyter_Dockerfile b/infrastructure-provisioning/src/general/files/aws/jupyter_Dockerfile
index eaa6791..8312b72 100644
--- a/infrastructure-provisioning/src/general/files/aws/jupyter_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/jupyter_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY jupyter/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/aws/jupyter_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/jupyterlab_Dockerfile b/infrastructure-provisioning/src/general/files/aws/jupyterlab_Dockerfile
index 9d111bd..6c4f0aa 100644
--- a/infrastructure-provisioning/src/general/files/aws/jupyterlab_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/jupyterlab_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY jupyterlab/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/aws/jupyter* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
COPY general/templates/os/pyspark_local_template.json /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/project_Dockerfile b/infrastructure-provisioning/src/general/files/aws/project_Dockerfile
index 4713edf..5390023 100644
--- a/infrastructure-provisioning/src/general/files/aws/project_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/project_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY project/ /root/
COPY general/scripts/aws/project_* /root/scripts/
COPY general/scripts/aws/edge_* /root/scripts/
-COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python3.6/datalab/edge_lib.py
+COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python3.8/datalab/edge_lib.py
COPY general/templates/aws/edge_s3_policy.json /root/templates/edge_s3_policy.json
COPY general/templates/os/manage_step_certs.sh /root/templates/
COPY general/templates/os/step-cert-manager.service /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/rstudio_Dockerfile b/infrastructure-provisioning/src/general/files/aws/rstudio_Dockerfile
index 28117cf..93041d3 100644
--- a/infrastructure-provisioning/src/general/files/aws/rstudio_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/rstudio_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY rstudio/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/aws/rstudio_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
COPY general/templates/os/inactive_rs.sh /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/ssn_Dockerfile b/infrastructure-provisioning/src/general/files/aws/ssn_Dockerfile
index 1a8ab72..b9c1548 100644
--- a/infrastructure-provisioning/src/general/files/aws/ssn_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/ssn_Dockerfile
@@ -25,7 +25,7 @@ ARG OS
COPY infrastructure-provisioning/src/ssn/ /root/
COPY infrastructure-provisioning/src/general/scripts/aws/ssn_* /root/scripts/
-COPY infrastructure-provisioning/src/general/lib/os/${OS}/ssn_lib.py /usr/lib/python3.6/datalab/ssn_lib.py
+COPY infrastructure-provisioning/src/general/lib/os/${OS}/ssn_lib.py /usr/lib/python3.8/datalab/ssn_lib.py
COPY infrastructure-provisioning/src/general/files/aws/ssn_policy.json /root/files/
COPY infrastructure-provisioning/src/general/templates/aws/jenkins_jobs /root/templates/jenkins_jobs
COPY infrastructure-provisioning/src/general/templates/os/manage_step_certs.sh /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/tensor-rstudio_Dockerfile b/infrastructure-provisioning/src/general/files/aws/tensor-rstudio_Dockerfile
index c063f76..5c7e95b 100644
--- a/infrastructure-provisioning/src/general/files/aws/tensor-rstudio_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/tensor-rstudio_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY tensor-rstudio/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/aws/tensor-rstudio_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/tensor_Dockerfile b/infrastructure-provisioning/src/general/files/aws/tensor_Dockerfile
index 808cdf4..6bd820c 100644
--- a/infrastructure-provisioning/src/general/files/aws/tensor_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/tensor_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY tensor/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/aws/tensor_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/aws/zeppelin_Dockerfile b/infrastructure-provisioning/src/general/files/aws/zeppelin_Dockerfile
index 038bc1a..abcb89e 100644
--- a/infrastructure-provisioning/src/general/files/aws/zeppelin_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/zeppelin_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY zeppelin/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/aws/zeppelin_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/aws/interpreter_livy.json /root/templates/
COPY general/templates/aws/interpreter_spark.json /root/templates/
COPY general/templates/aws/dataengine-service_interpreter_* /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/azure/base_Dockerfile b/infrastructure-provisioning/src/general/files/azure/base_Dockerfile
index 611231e..dc402ff 100644
--- a/infrastructure-provisioning/src/general/files/azure/base_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/base_Dockerfile
@@ -58,16 +58,16 @@ RUN mkdir -p /root/conf && \
mkdir -p /root/scripts && \
mkdir -p /root/templates && \
mkdir -p /root/files && \
- mkdir -p /usr/lib/python3.6/datalab && \
+ mkdir -p /usr/lib/python3.8/datalab && \
mkdir -p /root/keys/.ssh
COPY ${SRC_PATH}base/ /root
COPY ${SRC_PATH}general/conf/* /root/conf/
COPY ${SRC_PATH}general/api/*.py /bin/
COPY ${SRC_PATH}general/scripts/azure/common_* /root/scripts/
-COPY ${SRC_PATH}general/lib/azure/* /usr/lib/python3.6/datalab/
-COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python3.6/datalab/common_lib.py
-COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python3.6/datalab/fab.py
+COPY ${SRC_PATH}general/lib/azure/* /usr/lib/python3.8/datalab/
+COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python3.8/datalab/common_lib.py
+COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python3.8/datalab/fab.py
COPY ${SRC_PATH}general/files/os/${OS}/sources.list /root/files/
COPY ${SRC_PATH}general/files/os/ivysettings.xml /root/templates/
COPY ${SRC_PATH}general/files/os/local_endpoint.json /root/files/
diff --git a/infrastructure-provisioning/src/general/files/azure/dataengine_Dockerfile b/infrastructure-provisioning/src/general/files/azure/dataengine_Dockerfile
index a16396f..d1eec41 100644
--- a/infrastructure-provisioning/src/general/files/azure/dataengine_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/dataengine_Dockerfile
@@ -24,7 +24,7 @@ FROM docker.datalab-base:latest
ARG OS
COPY dataengine/ /root/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/scripts/os/dataengine_* /root/scripts/
COPY general/scripts/os/update_inactivity_on_start.py /root/scripts/
COPY general/scripts/os/reconfigure_spark.py /root/scripts/
diff --git a/infrastructure-provisioning/src/general/files/azure/deeplearning_Dockerfile b/infrastructure-provisioning/src/general/files/azure/deeplearning_Dockerfile
index b8b127c..a9d548f 100644
--- a/infrastructure-provisioning/src/general/files/azure/deeplearning_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/deeplearning_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY deeplearning/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/azure/deeplearning_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/azure/edge_Dockerfile b/infrastructure-provisioning/src/general/files/azure/edge_Dockerfile
index bfeea38..eeb5215 100644
--- a/infrastructure-provisioning/src/general/files/azure/edge_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/edge_Dockerfile
@@ -26,7 +26,7 @@ ARG OS
COPY edge/ /root/
COPY general/scripts/azure/edge_* /root/scripts/
-COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python3.6/datalab/edge_lib.py
+COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python3.8/datalab/edge_lib.py
RUN chmod a+x /root/fabfile.py; \
chmod a+x /root/scripts/*
diff --git a/infrastructure-provisioning/src/general/files/azure/jupyter_Dockerfile b/infrastructure-provisioning/src/general/files/azure/jupyter_Dockerfile
index 6bdccfe..821ec24 100644
--- a/infrastructure-provisioning/src/general/files/azure/jupyter_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/jupyter_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY jupyter/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/azure/jupyter_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/azure/jupyterlab_Dockerfile b/infrastructure-provisioning/src/general/files/azure/jupyterlab_Dockerfile
index 990e3c5..4fd212b 100644
--- a/infrastructure-provisioning/src/general/files/azure/jupyterlab_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/jupyterlab_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY jupyterlab/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/azure/jupyter* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/azure/project_Dockerfile b/infrastructure-provisioning/src/general/files/azure/project_Dockerfile
index 841c3f8..9f13f4b 100644
--- a/infrastructure-provisioning/src/general/files/azure/project_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/project_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY project/ /root/
COPY general/scripts/azure/project_* /root/scripts/
COPY general/scripts/azure/edge_* /root/scripts/
-COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python3.6/datalab/edge_lib.py
+COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python3.8/datalab/edge_lib.py
COPY general/templates/os/manage_step_certs.sh /root/templates/
COPY general/templates/os/step-cert-manager.service /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/azure/rstudio_Dockerfile b/infrastructure-provisioning/src/general/files/azure/rstudio_Dockerfile
index 3f3bdfd..0c39847 100644
--- a/infrastructure-provisioning/src/general/files/azure/rstudio_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/rstudio_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY rstudio/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/azure/rstudio_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
COPY general/templates/os/inactive_rs.sh /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/azure/ssn_Dockerfile b/infrastructure-provisioning/src/general/files/azure/ssn_Dockerfile
index 06ebc90..8eb5885 100644
--- a/infrastructure-provisioning/src/general/files/azure/ssn_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/ssn_Dockerfile
@@ -25,7 +25,7 @@ ARG OS
COPY infrastructure-provisioning/src/ssn/ /root/
COPY infrastructure-provisioning/src/general/scripts/azure/ssn_* /root/scripts/
-COPY infrastructure-provisioning/src/general/lib/os/${OS}/ssn_lib.py /usr/lib/python3.6/datalab/ssn_lib.py
+COPY infrastructure-provisioning/src/general/lib/os/${OS}/ssn_lib.py /usr/lib/python3.8/datalab/ssn_lib.py
COPY infrastructure-provisioning/src/general/templates/azure/jenkins_jobs /root/templates/jenkins_jobs
COPY infrastructure-provisioning/src/general/templates/os/manage_step_certs.sh /root/templates/
COPY infrastructure-provisioning/src/general/templates/os/step-cert-manager.service /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/azure/tensor_Dockerfile b/infrastructure-provisioning/src/general/files/azure/tensor_Dockerfile
index d1a27de..4bb12ee 100644
--- a/infrastructure-provisioning/src/general/files/azure/tensor_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/tensor_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY tensor/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/azure/tensor_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/azure/zeppelin_Dockerfile b/infrastructure-provisioning/src/general/files/azure/zeppelin_Dockerfile
index 11cf654..b2a0a6f 100644
--- a/infrastructure-provisioning/src/general/files/azure/zeppelin_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/zeppelin_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY zeppelin/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/azure/zeppelin_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/azure/interpreter_livy.json /root/templates/
COPY general/templates/azure/interpreter_spark.json /root/templates/
COPY general/templates/os/dataengine_interpreter_livy.json /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/gcp/base_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/base_Dockerfile
index 8dae17b..6f324eb 100644
--- a/infrastructure-provisioning/src/general/files/gcp/base_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/base_Dockerfile
@@ -69,16 +69,16 @@ RUN mkdir -p /root/conf && \
mkdir -p /root/scripts && \
mkdir -p /root/templates && \
mkdir -p /root/files && \
- mkdir -p /usr/lib/python3.6/datalab && \
+ mkdir -p /usr/lib/python3.8/datalab && \
mkdir -p /root/keys/.ssh
COPY ${SRC_PATH}base/ /root
COPY ${SRC_PATH}general/conf/* /root/conf/
COPY ${SRC_PATH}general/api/*.py /bin/
COPY ${SRC_PATH}general/scripts/gcp/common_* /root/scripts/
-COPY ${SRC_PATH}general/lib/gcp/* /usr/lib/python3.6/datalab/
-COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python3.6/datalab/common_lib.py
-COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python3.6/datalab/fab.py
+COPY ${SRC_PATH}general/lib/gcp/* /usr/lib/python3.8/datalab/
+COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python3.8/datalab/common_lib.py
+COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python3.8/datalab/fab.py
COPY ${SRC_PATH}general/files/os/${OS}/sources.list /root/files/
COPY ${SRC_PATH}general/files/os/ivysettings.xml /root/templates/
COPY ${SRC_PATH}general/files/os/local_endpoint.json /root/files/
diff --git a/infrastructure-provisioning/src/general/files/gcp/dataengine-service_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/dataengine-service_Dockerfile
index 3d4a974..953b929 100644
--- a/infrastructure-provisioning/src/general/files/gcp/dataengine-service_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/dataengine-service_Dockerfile
@@ -26,7 +26,7 @@ ARG OS
COPY dataengine-service/fabfile.py /root/
COPY dataengine-service/description.json /root/
COPY general/scripts/gcp/dataengine-service_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/scripts/os/common_* /root/scripts/
COPY general/scripts/os/install_additional_libs.py /root/scripts/install_additional_libs.py
COPY general/scripts/os/get_list_available_pkgs.py /root/scripts/get_list_available_pkgs.py
diff --git a/infrastructure-provisioning/src/general/files/gcp/dataengine_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/dataengine_Dockerfile
index f0d04c9..8445b20 100644
--- a/infrastructure-provisioning/src/general/files/gcp/dataengine_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/dataengine_Dockerfile
@@ -29,7 +29,7 @@ COPY general/scripts/os/update_inactivity_on_start.py /root/scripts/
COPY general/scripts/os/reconfigure_spark.py /root/scripts/
COPY general/scripts/os/install_additional_libs.py /root/scripts/install_additional_libs.py
COPY general/scripts/os/get_list_available_pkgs.py /root/scripts/get_list_available_pkgs.py
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/scripts/os/notebook_reconfigure_dataengine_spark.py /root/scripts/
COPY general/scripts/os/common_* /root/scripts/
COPY general/scripts/gcp/dataengine_* /root/scripts/
diff --git a/infrastructure-provisioning/src/general/files/gcp/deeplearning_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/deeplearning_Dockerfile
index 62b8544..45a327b 100644
--- a/infrastructure-provisioning/src/general/files/gcp/deeplearning_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/deeplearning_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY deeplearning/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/gcp/deeplearning_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/gcp/edge_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/edge_Dockerfile
index 7068f14..ef3c1ac 100644
--- a/infrastructure-provisioning/src/general/files/gcp/edge_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/edge_Dockerfile
@@ -26,7 +26,7 @@ ARG OS
COPY edge/ /root/
COPY general/scripts/gcp/edge_* /root/scripts/
-COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python3.6/datalab/edge_lib.py
+COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python3.8/datalab/edge_lib.py
COPY general/files/gcp/ps_policy.json /root/files/ps_policy.json
COPY general/files/gcp/ps_roles.json /root/files/ps_roles.json
diff --git a/infrastructure-provisioning/src/general/files/gcp/jupyter_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/jupyter_Dockerfile
index 28e3275..0de5dc7 100644
--- a/infrastructure-provisioning/src/general/files/gcp/jupyter_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/jupyter_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY jupyter/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/gcp/jupyter_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/gcp/jupyterlab_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/jupyterlab_Dockerfile
index b639d09..4ab9fd2 100644
--- a/infrastructure-provisioning/src/general/files/gcp/jupyterlab_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/jupyterlab_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY jupyterlab/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/gcp/jupyter* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
COPY general/templates/os/pyspark_local_template.json /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/gcp/project_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/project_Dockerfile
index a87cf10..8c8df3c 100644
--- a/infrastructure-provisioning/src/general/files/gcp/project_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/project_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY project/ /root/
COPY general/scripts/gcp/project_* /root/scripts/
COPY general/scripts/gcp/edge_* /root/scripts/
-COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python3.6/datalab/edge_lib.py
+COPY general/lib/os/${OS}/edge_lib.py /usr/lib/python3.8/datalab/edge_lib.py
COPY general/files/gcp/ps_policy.json /root/files/ps_policy.json
COPY general/files/gcp/ps_roles.json /root/files/ps_roles.json
COPY general/templates/os/manage_step_certs.sh /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/gcp/rstudio_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/rstudio_Dockerfile
index 993a881..5738cba 100644
--- a/infrastructure-provisioning/src/general/files/gcp/rstudio_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/rstudio_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY rstudio/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/gcp/rstudio_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
COPY general/templates/gcp/core-site.xml /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/gcp/ssn_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/ssn_Dockerfile
index 49cb146..a330912 100644
--- a/infrastructure-provisioning/src/general/files/gcp/ssn_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/ssn_Dockerfile
@@ -25,7 +25,7 @@ ARG OS
COPY infrastructure-provisioning/src/ssn/ /root/
COPY infrastructure-provisioning/src/general/scripts/gcp/ssn_* /root/scripts/
-COPY infrastructure-provisioning/src/general/lib/os/${OS}/ssn_lib.py /usr/lib/python3.6/datalab/ssn_lib.py
+COPY infrastructure-provisioning/src/general/lib/os/${OS}/ssn_lib.py /usr/lib/python3.8/datalab/ssn_lib.py
COPY infrastructure-provisioning/src/general/files/gcp/ssn_policy.json /root/files/
COPY infrastructure-provisioning/src/general/files/gcp/ssn_roles.json /root/files/
COPY infrastructure-provisioning/src/general/templates/gcp/jenkins_jobs /root/templates/jenkins_jobs
diff --git a/infrastructure-provisioning/src/general/files/gcp/superset_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/superset_Dockerfile
index 54e4eb4..8c63e0a 100644
--- a/infrastructure-provisioning/src/general/files/gcp/superset_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/superset_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY superset/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/gcp/superset_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/superset-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/gcp/tensor-rstudio_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/tensor-rstudio_Dockerfile
index a5d1151..0e85ff3 100644
--- a/infrastructure-provisioning/src/general/files/gcp/tensor-rstudio_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/tensor-rstudio_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY tensor-rstudio/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/gcp/tensor-rstudio_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/gcp/tensor_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/tensor_Dockerfile
index a6eef70..b585dd1 100644
--- a/infrastructure-provisioning/src/general/files/gcp/tensor_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/tensor_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY tensor/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/gcp/tensor_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/os/${OS}/jupyter-notebook.service /root/templates/
COPY general/templates/os/${OS}/ungit.service /root/templates/
COPY general/templates/os/notebook_spark-defaults_local.conf /root/templates/
diff --git a/infrastructure-provisioning/src/general/files/gcp/zeppelin_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/zeppelin_Dockerfile
index 179dacb..3ae5220 100644
--- a/infrastructure-provisioning/src/general/files/gcp/zeppelin_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/zeppelin_Dockerfile
@@ -27,7 +27,7 @@ ARG OS
COPY zeppelin/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/gcp/zeppelin_* /root/scripts/
-COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.6/datalab/notebook_lib.py
+COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
COPY general/templates/gcp/interpreter_livy.json /root/templates/
COPY general/templates/gcp/interpreter_spark.json /root/templates/
COPY general/templates/os/dataengine_interpreter_livy.json /root/templates/
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index 95b0bfb..72f0fbd 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -175,7 +175,7 @@ def ensure_matplot(os_user):
manage_pkg('-y build-dep', 'remote', 'python3-matplotlib')
sudo('pip3 install matplotlib==2.0.2 --no-cache-dir')
if os.environ['application'] in ('tensor', 'deeplearning'):
- sudo('python3.6 -m pip install -U numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
+ sudo('python3.8 -m pip install -U numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
sudo('touch /home/' + os_user + '/.ensure_dir/matplot_ensured')
except:
sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index c323c79..ea42c4e 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -384,9 +384,9 @@ def pyspark_kernel(kernels_dir, dataengine_service_version, cluster_name, spark_
text = text.replace('CLUSTER_NAME', cluster_name)
text = text.replace('SPARK_VERSION', 'Spark-' + spark_version)
text = text.replace('SPARK_PATH', spark_path)
- text = text.replace('PYTHON_SHORT_VERSION', '3.6')
- text = text.replace('PYTHON_FULL_VERSION', '3.6')
- text = text.replace('PYTHON_PATH', '/usr/bin/python3.6')
+ text = text.replace('PYTHON_SHORT_VERSION', '3.8')
+ text = text.replace('PYTHON_FULL_VERSION', '3.8')
+ text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
text = text.replace('DATAENGINE-SERVICE_VERSION', dataengine_service_version)
with open(kernel_path, 'w') as f:
f.write(text)
@@ -737,6 +737,8 @@ def configure_data_engine_service_pip(hostname, os_user, keyfile, emr=False):
sudo('ln -s /usr/bin/pip-3.6 /usr/bin/pip3')
elif not exists('/usr/bin/pip3') and sudo("python3.7 -V 2>/dev/null | awk '{print $2}'"):
sudo('ln -s /usr/bin/pip-3.7 /usr/bin/pip3')
+ elif not exists('/usr/bin/pip3') and sudo("python3.8 -V 2>/dev/null | awk '{print $2}'"):
+ sudo('ln -s /usr/bin/pip-3.8 /usr/bin/pip3')
if emr:
sudo('pip3 install -U pip=={}'.format(os.environ['conf_pip_version']))
sudo('ln -s /usr/local/bin/pip3.7 /bin/pip3.7')
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
index ea8eae3..50e2193 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
@@ -144,7 +144,7 @@ def ensure_matplot(os_user):
try:
sudo('python3.5 -m pip install matplotlib==2.0.2 --no-cache-dir')
if os.environ['application'] in ('tensor', 'deeplearning'):
- sudo('python3.6 -m pip install -U numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
+ sudo('python3.8 -m pip install -U numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
sudo('touch /home/{}/.ensure_dir/matplot_ensured'.format(os_user))
except:
sys.exit(1)
@@ -188,7 +188,7 @@ def ensure_additional_python_libs(os_user):
if os.environ['application'] in ('jupyter', 'zeppelin'):
sudo('python3.5 -m pip install NumPy=={} SciPy pandas Sympy Pillow sklearn --no-cache-dir'.format(os.environ['notebook_numpy_version']))
if os.environ['application'] in ('tensor', 'deeplearning'):
- sudo('python3.6 -m pip install opencv-python h5py --no-cache-dir')
+ sudo('python3.8 -m pip install opencv-python h5py --no-cache-dir')
sudo('touch /home/' + os_user + '/.ensure_dir/additional_python_libs_ensured')
except:
sys.exit(1)
@@ -260,7 +260,7 @@ def install_tensor(os_user, cuda_version, cuda_file_name,
# install TensorFlow and run TensorBoard
sudo('wget https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp27-none-linux_x86_64.whl'.format(tensorflow_version))
sudo('wget https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp35-cp35m-linux_x86_64.whl'.format(tensorflow_version))
- sudo('python3.6 -m pip install --upgrade tensorflow_gpu-{}-cp35-cp35m-linux_x86_64.whl --no-cache-dir'.format(tensorflow_version))
+ sudo('python3.8 -m pip install --upgrade tensorflow_gpu-{}-cp35-cp35m-linux_x86_64.whl --no-cache-dir'.format(tensorflow_version))
sudo('rm -rf /home/{}/tensorflow_gpu-*'.format(os_user))
sudo('mkdir /var/log/tensorboard; chown {0}:{0} -R /var/log/tensorboard'.format(os_user))
put('{}tensorboard.service'.format(templates_dir), '/tmp/tensorboard.service')
@@ -456,7 +456,7 @@ def install_keras(os_user, keras_version):
def install_theano(os_user, theano_version):
if not exists('/home/{}/.ensure_dir/theano_ensured'.format(os_user)):
- sudo('python3.6 -m pip install Theano=={} --no-cache-dir'.format(theano_version))
+ sudo('python3.8 -m pip install Theano=={} --no-cache-dir'.format(theano_version))
sudo('touch /home/{}/.ensure_dir/theano_ensured'.format(os_user))
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py
index c6514fb..3c54783 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py
@@ -62,13 +62,13 @@ def configure_notebook(args):
sudo(
'\cp /tmp/jupyter_dataengine-service_create_configs.py /usr/local/bin/jupyter_dataengine-service_create_configs.py')
sudo('chmod 755 /usr/local/bin/jupyter_dataengine-service_create_configs.py')
- sudo('mkdir -p /usr/lib/python3.6/datalab/')
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
run('mkdir -p /tmp/datalab_libs/')
- local('scp -i {} /usr/lib/python3.6/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
run('chmod a+x /tmp/datalab_libs/*')
- sudo('mv /tmp/datalab_libs/* /usr/lib/python3.6/datalab/')
+ sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python3.6/datalab /usr/lib64/python3.6/datalab')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
if __name__ == "__main__":
diff --git a/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py
index b5f4b4c..1b3d850 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py
@@ -51,13 +51,13 @@ def configure_notebook(args):
sudo(
'\cp /tmp/rstudio_dataengine-service_create_configs.py /usr/local/bin/rstudio_dataengine-service_create_configs.py')
sudo('chmod 755 /usr/local/bin/rstudio_dataengine-service_create_configs.py')
- sudo('mkdir -p /usr/lib/python3.6/datalab/')
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
run('mkdir -p /tmp/datalab_libs/')
- local('scp -i {} /usr/lib/python3.6/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
run('chmod a+x /tmp/datalab_libs/*')
- sudo('mv /tmp/datalab_libs/* /usr/lib/python3.6/datalab/')
+ sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python3.6/datalab /usr/lib64/python3.6/datalab')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
if __name__ == "__main__":
diff --git a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
index 377940a..3ff681f 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
@@ -57,13 +57,13 @@ def configure_notebook(args):
sudo(
'\cp /tmp/zeppelin_dataengine-service_create_configs.py /usr/local/bin/zeppelin_dataengine-service_create_configs.py')
sudo('chmod 755 /usr/local/bin/zeppelin_dataengine-service_create_configs.py')
- sudo('mkdir -p /usr/lib/python3.6/datalab/')
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
run('mkdir -p /tmp/datalab_libs/')
- local('scp -i {} /usr/lib/python3.6/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
run('chmod a+x /tmp/datalab_libs/*')
- sudo('mv /tmp/datalab_libs/* /usr/lib/python3.6/datalab/')
+ sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python3.6/datalab /usr/lib64/python3.6/datalab')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
if __name__ == "__main__":
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
index 6aad7bf..650e64d 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
@@ -60,13 +60,13 @@ def configure_notebook(args):
# put(templates_dir + 'run_template.sh', '/tmp/run_template.sh')
sudo('\cp /tmp/create_configs.py /usr/local/bin/create_configs.py')
sudo('chmod 755 /usr/local/bin/create_configs.py')
- sudo('mkdir -p /usr/lib/python3.6/datalab/')
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
run('mkdir -p /tmp/datalab_libs/')
- local('scp -i {} /usr/lib/python3.6/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
run('chmod a+x /tmp/datalab_libs/*')
- sudo('mv /tmp/datalab_libs/* /usr/lib/python3.6/datalab/')
+ sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python3.6/datalab /usr/lib64/python3.6/datalab')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
if __name__ == "__main__":
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py
index 9cc946a..b5183f9 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py
@@ -51,13 +51,13 @@ def configure_notebook(args):
put(scripts_dir + '{}_dataengine-service_create_configs.py'.format(args.application), '/tmp/create_configs.py')
sudo('\cp /tmp/create_configs.py /usr/local/bin/create_configs.py')
sudo('chmod 755 /usr/local/bin/create_configs.py')
- sudo('mkdir -p /usr/lib/python3.6/datalab/')
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
run('mkdir -p /tmp/datalab_libs/')
- local('scp -i {} /usr/lib/python3.6/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
run('chmod a+x /tmp/datalab_libs/*')
- sudo('mv /tmp/datalab_libs/* /usr/lib/python3.6/datalab/')
+ sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python3.6/datalab /usr/lib64/python3.6/datalab')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
if __name__ == "__main__":
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
index cf4b15d..dde90be 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
@@ -56,13 +56,13 @@ def configure_notebook(args):
put(scripts_dir + '{}_dataengine-service_create_configs.py'.format(args.application), '/tmp/create_configs.py')
sudo('\cp /tmp/create_configs.py /usr/local/bin/create_configs.py')
sudo('chmod 755 /usr/local/bin/create_configs.py')
- sudo('mkdir -p /usr/lib/python3.6/datalab/')
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
run('mkdir -p /tmp/datalab_libs/')
- local('scp -i {} /usr/lib/python3.6/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
run('chmod a+x /tmp/datalab_libs/*')
- sudo('mv /tmp/datalab_libs/* /usr/lib/python3.6/datalab/')
+ sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python3.6/datalab /usr/lib64/python3.6/datalab')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
if __name__ == "__main__":
diff --git a/infrastructure-provisioning/src/general/scripts/os/deeplearning_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/deeplearning_dataengine_create_configs.py
index 066c6f2..68c3ae0 100644
--- a/infrastructure-provisioning/src/general/scripts/os/deeplearning_dataengine_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/deeplearning_dataengine_create_configs.py
@@ -62,10 +62,10 @@ def pyspark_kernel(args):
text = text.replace('CLUSTER_NAME', args.cluster_name)
text = text.replace('SPARK_VERSION', 'Spark-' + spark_version)
text = text.replace('SPARK_PATH', spark_path)
- text = text.replace('PYTHON_SHORT_VERSION', '3.6')
- text = text.replace('PYTHON_FULL_VERSION', '3.6')
+ text = text.replace('PYTHON_SHORT_VERSION', '3.8')
+ text = text.replace('PYTHON_FULL_VERSION', '3.8')
text = text.replace('MASTER', args.spark_master)
- text = text.replace('PYTHON_PATH', '/usr/bin/python3.6')
+ text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
with open(kernel_path, 'w') as f:
f.write(text)
local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
@@ -83,9 +83,9 @@ def pyspark_kernel(args):
text = text.replace('SPARK_VERSION', 'Spark-' + spark_version)
text = text.replace('SPARK_PATH', spark_path)
text = text.replace('MASTER', args.spark_master)
- text = text.replace('PYTHON_SHORT_VERSION', '3.6')
- text = text.replace('PYTHON_FULL_VERSION', '3.6')
- text = text.replace('PYTHON_PATH', '/usr/bin/python3.6')
+ text = text.replace('PYTHON_SHORT_VERSION', '3.8')
+ text = text.replace('PYTHON_FULL_VERSION', '3.8')
+ text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
with open(kernel_path, 'w') as f:
f.write(text)
local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
@@ -101,7 +101,7 @@ def install_sparkamagic_kernels(args):
local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkkernel --user'.format(sparkmagic_dir))
local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/pysparkkernel --user'.format(sparkmagic_dir))
- pyspark_kernel_name = 'PySpark (Python-3.6 / Spark-{0} ) [{1}]'.format(args.spark_version,
+ pyspark_kernel_name = 'PySpark (Python-3.8 / Spark-{0} ) [{1}]'.format(args.spark_version,
args.cluster_name)
local('sed -i \'s|PySpark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/pysparkkernel/kernel.json'.format(
pyspark_kernel_name, args.os_user))
diff --git a/infrastructure-provisioning/src/general/scripts/os/deeplearning_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/deeplearning_install_dataengine_kernels.py
index 3a55591..035e8c5 100644
--- a/infrastructure-provisioning/src/general/scripts/os/deeplearning_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/deeplearning_install_dataengine_kernels.py
@@ -58,12 +58,12 @@ def configure_notebook(keyfile, hoststring):
put(scripts_dir + 'deeplearning_dataengine_create_configs.py',
'/usr/local/bin/deeplearning_dataengine_create_configs.py', use_sudo=True)
sudo('chmod 755 /usr/local/bin/deeplearning_dataengine_create_configs.py')
- if not exists('/usr/lib/python3.6/datalab/'):
- sudo('mkdir -p /usr/lib/python3.6/datalab/')
- put('/usr/lib/python3.6/datalab/*', '/usr/lib/python3.6/datalab/', use_sudo=True)
- sudo('chmod a+x /usr/lib/python3.6/datalab/*')
+ if not exists('/usr/lib/python3.8/datalab/'):
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
+ put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
+ sudo('chmod a+x /usr/lib/python3.8/datalab/*')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python3.6/datalab /usr/lib64/python3.6/datalab')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
def create_inactivity_log(master_ip, hoststring):
reworked_ip = master_ip.replace('.', '-')
diff --git a/infrastructure-provisioning/src/general/scripts/os/get_list_available_pkgs.py b/infrastructure-provisioning/src/general/scripts/os/get_list_available_pkgs.py
index 79d3ccc..67ca9c9 100644
--- a/infrastructure-provisioning/src/general/scripts/os/get_list_available_pkgs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/get_list_available_pkgs.py
@@ -85,10 +85,10 @@ if __name__ == "__main__":
elif args.group == 'pip2':
all_pkgs['pip2'] = get_available_pip_pkgs("2.7")
elif args.group == 'pip3':
- all_pkgs['pip3'] = get_available_pip_pkgs("3.6")
+ all_pkgs['pip3'] = get_available_pip_pkgs("3.8")
elif args.group == 'others':
all_pkgs['pip2'] = get_available_pip_pkgs("2.7")
- all_pkgs['pip3'] = get_available_pip_pkgs("3.6")
+ all_pkgs['pip3'] = get_available_pip_pkgs("3.8")
all_pkgs['others'] = get_uncategorised_pip_pkgs(all_pkgs['pip2'], all_pkgs['pip3'])
elif args.group == 'r_pkg':
all_pkgs['r_pkg'] = get_available_r_pkgs()
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py
index 4daecbf..1c9328d 100644
--- a/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/jupyter_dataengine_create_configs.py
@@ -117,10 +117,10 @@ def pyspark_kernel(args):
text = text.replace('CLUSTER_NAME', args.cluster_name)
text = text.replace('SPARK_VERSION', 'Spark-' + spark_version)
text = text.replace('SPARK_PATH', spark_path)
- text = text.replace('PYTHON_SHORT_VERSION', '3.6')
- text = text.replace('PYTHON_FULL_VERSION', '3.6')
+ text = text.replace('PYTHON_SHORT_VERSION', '3.8')
+ text = text.replace('PYTHON_FULL_VERSION', '3.8')
text = text.replace('MASTER', args.spark_master)
- text = text.replace('PYTHON_PATH', '/usr/bin/python3.6')
+ text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
with open(kernel_path, 'w') as f:
f.write(text)
local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
@@ -138,9 +138,9 @@ def pyspark_kernel(args):
text = text.replace('SPARK_VERSION', 'Spark-' + spark_version)
text = text.replace('SPARK_PATH', spark_path)
text = text.replace('MASTER', args.spark_master)
- text = text.replace('PYTHON_SHORT_VERSION', '3.6')
- text = text.replace('PYTHON_FULL_VERSION', '3.6')
- text = text.replace('PYTHON_PATH', '/usr/bin/python3.6')
+ text = text.replace('PYTHON_SHORT_VERSION', '3.8')
+ text = text.replace('PYTHON_FULL_VERSION', '3.8')
+ text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
with open(kernel_path, 'w') as f:
f.write(text)
local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
@@ -156,7 +156,7 @@ def install_sparkamagic_kernels(args):
local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkkernel --user'.format(sparkmagic_dir))
local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/pysparkkernel --user'.format(sparkmagic_dir))
local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkrkernel --user'.format(sparkmagic_dir))
- pyspark_kernel_name = 'PySpark (Python-3.6 / Spark-{0} ) [{1}]'.format(args.spark_version,
+ pyspark_kernel_name = 'PySpark (Python-3.8 / Spark-{0} ) [{1}]'.format(args.spark_version,
args.cluster_name)
local('sed -i \'s|PySpark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/pysparkkernel/kernel.json'.format(
pyspark_kernel_name, args.os_user))
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py
index 7079d4d..5a21371 100644
--- a/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py
@@ -62,12 +62,12 @@ def configure_notebook(keyfile, hoststring):
put(scripts_dir + 'jupyter_dataengine_create_configs.py', '/usr/local/bin/jupyter_dataengine_create_configs.py',
use_sudo=True)
sudo('chmod 755 /usr/local/bin/jupyter_dataengine_create_configs.py')
- if not exists('/usr/lib/python3.6/datalab/'):
- sudo('mkdir -p /usr/lib/python3.6/datalab/')
- put('/usr/lib/python3.6/datalab/*', '/usr/lib/python3.6/datalab/', use_sudo=True)
- sudo('chmod a+x /usr/lib/python3.6/datalab/*')
+ if not exists('/usr/lib/python3.8/datalab/'):
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
+ put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
+ sudo('chmod a+x /usr/lib/python3.8/datalab/*')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python3.6/datalab /usr/lib64/python3.6/datalab')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
def create_inactivity_log(master_ip, hoststring):
reworked_ip = master_ip.replace('.', '-')
diff --git a/infrastructure-provisioning/src/general/scripts/os/rstudio_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/rstudio_install_dataengine_kernels.py
index 11dea24..d934a7c 100644
--- a/infrastructure-provisioning/src/general/scripts/os/rstudio_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/rstudio_install_dataengine_kernels.py
@@ -55,12 +55,12 @@ def configure_notebook(keyfile, hoststring):
put(scripts_dir + 'rstudio_dataengine_create_configs.py', '/usr/local/bin/rstudio_dataengine_create_configs.py',
use_sudo=True)
sudo('chmod 755 /usr/local/bin/rstudio_dataengine_create_configs.py')
- if not exists('/usr/lib/python3.6/datalab/'):
- sudo('mkdir -p /usr/lib/python3.6/datalab/')
- put('/usr/lib/python3.6/datalab/*', '/usr/lib/python3.6/datalab/', use_sudo=True)
- sudo('chmod a+x /usr/lib/python3.6/datalab/*')
+ if not exists('/usr/lib/python3.8/datalab/'):
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
+ put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
+ sudo('chmod a+x /usr/lib/python3.8/datalab/*')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python3.6/datalab /usr/lib64/python3.6/datalab')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
def create_inactivity_log(master_ip, hoststring):
reworked_ip = master_ip.replace('.', '-')
diff --git a/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_install_dataengine_kernels.py
index be0dab4..e1d49b0 100644
--- a/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/tensor-rstudio_install_dataengine_kernels.py
@@ -55,12 +55,12 @@ def configure_notebook(keyfile, hoststring):
put(scripts_dir + 'tensor-rstudio_dataengine_create_configs.py',
'/usr/local/bin/tensor-rstudio_dataengine_create_configs.py', use_sudo=True)
sudo('chmod 755 /usr/local/bin/tensor-rstudio_dataengine_create_configs.py')
- if not exists('/usr/lib/python3.6/datalab/'):
- sudo('mkdir -p /usr/lib/python3.6/datalab/')
- put('/usr/lib/python3.6/datalab/*', '/usr/lib/python3.6/datalab/', use_sudo=True)
- sudo('chmod a+x /usr/lib/python3.6/datalab/*')
+ if not exists('/usr/lib/python3.8/datalab/'):
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
+ put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
+ sudo('chmod a+x /usr/lib/python3.8/datalab/*')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python3.6/datalab /usr/lib64/python3.6/datalab')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
def create_inactivity_log(master_ip, hoststring):
reworked_ip = master_ip.replace('.', '-')
diff --git a/infrastructure-provisioning/src/general/scripts/os/tensor_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/tensor_dataengine_create_configs.py
index 540525f..e93d125 100644
--- a/infrastructure-provisioning/src/general/scripts/os/tensor_dataengine_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/tensor_dataengine_create_configs.py
@@ -62,10 +62,10 @@ def pyspark_kernel(args):
text = text.replace('CLUSTER_NAME', args.cluster_name)
text = text.replace('SPARK_VERSION', 'Spark-' + spark_version)
text = text.replace('SPARK_PATH', spark_path)
- text = text.replace('PYTHON_SHORT_VERSION', '3.6')
- text = text.replace('PYTHON_FULL_VERSION', '3.6')
+ text = text.replace('PYTHON_SHORT_VERSION', '3.8')
+ text = text.replace('PYTHON_FULL_VERSION', '3.8')
text = text.replace('MASTER', args.spark_master)
- text = text.replace('PYTHON_PATH', '/usr/bin/python3.6')
+ text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
with open(kernel_path, 'w') as f:
f.write(text)
local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
@@ -83,9 +83,9 @@ def pyspark_kernel(args):
text = text.replace('SPARK_VERSION', 'Spark-' + spark_version)
text = text.replace('SPARK_PATH', spark_path)
text = text.replace('MASTER', args.spark_master)
- text = text.replace('PYTHON_SHORT_VERSION', '3.6')
- text = text.replace('PYTHON_FULL_VERSION', '3.6')
- text = text.replace('PYTHON_PATH', '/usr/bin/python3.6')
+ text = text.replace('PYTHON_SHORT_VERSION', '3.8')
+ text = text.replace('PYTHON_FULL_VERSION', '3.8')
+ text = text.replace('PYTHON_PATH', '/usr/bin/python3.8')
with open(kernel_path, 'w') as f:
f.write(text)
local('touch /tmp/{}/kernel_var.json'.format(args.cluster_name))
@@ -101,7 +101,7 @@ def install_sparkamagic_kernels(args):
local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkkernel --user'.format(sparkmagic_dir))
local('sudo jupyter-kernelspec install {}/sparkmagic/kernels/pysparkkernel --user'.format(sparkmagic_dir))
- pyspark_kernel_name = 'PySpark (Python-3.6 / Spark-{0} ) [{1}]'.format(args.spark_version,
+ pyspark_kernel_name = 'PySpark (Python-3.8 / Spark-{0} ) [{1}]'.format(args.spark_version,
args.cluster_name)
local('sed -i \'s|PySpark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/pysparkkernel/kernel.json'.format(
pyspark_kernel_name, args.os_user))
diff --git a/infrastructure-provisioning/src/general/scripts/os/tensor_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/tensor_install_dataengine_kernels.py
index cb4f4c0..44dce5a 100644
--- a/infrastructure-provisioning/src/general/scripts/os/tensor_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/tensor_install_dataengine_kernels.py
@@ -55,12 +55,12 @@ def configure_notebook(keyfile, hoststring):
put(scripts_dir + 'tensor_dataengine_create_configs.py', '/usr/local/bin/tensor_dataengine_create_configs.py',
use_sudo=True)
sudo('chmod 755 /usr/local/bin/tensor_dataengine_create_configs.py')
- if not exists('/usr/lib/python3.6/datalab/'):
- sudo('mkdir -p /usr/lib/python3.6/datalab/')
- put('/usr/lib/python3.6/datalab/*', '/usr/lib/python3.6/datalab/', use_sudo=True)
- sudo('chmod a+x /usr/lib/python3.6/datalab/*')
+ if not exists('/usr/lib/python3.8/datalab/'):
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
+ put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
+ sudo('chmod a+x /usr/lib/python3.8/datalab/*')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python3.6/datalab /usr/lib64/python3.6/datalab')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
def create_inactivity_log(master_ip, hoststring):
reworked_ip = master_ip.replace('.', '-')
diff --git a/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py
index d33173b..5c917a2 100644
--- a/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py
@@ -91,7 +91,7 @@ def configure_zeppelin_dataengine_interpreter(cluster_name, cluster_dir, os_user
if os.path.exists(livy_path + 'conf/spark-blacklist.conf'):
local('sudo sed -i "s/^/#/g" ' + livy_path + 'conf/spark-blacklist.conf')
local(''' sudo echo "export SPARK_HOME=''' + cluster_dir + '''spark/" >> ''' + livy_path + '''conf/livy-env.sh''')
- local(''' sudo echo "export PYSPARK3_PYTHON=python3.6" >> ''' +
+ local(''' sudo echo "export PYSPARK3_PYTHON=python3.8" >> ''' +
livy_path + '''conf/livy-env.sh''')
template_file = "/tmp/{}/dataengine_interpreter.json".format(args.cluster_name)
fr = open(template_file, 'r+')
@@ -122,7 +122,7 @@ def configure_zeppelin_dataengine_interpreter(cluster_name, cluster_dir, os_user
local('sudo systemctl start livy-server-' + str(livy_port))
else:
template_file = "/tmp/{}/dataengine_interpreter.json".format(args.cluster_name)
- p_versions = ["2", "3.6"]
+ p_versions = ["2", "3.8"]
for p_version in p_versions:
fr = open(template_file, 'r+')
text = fr.read()
diff --git a/infrastructure-provisioning/src/general/scripts/os/zeppelin_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/zeppelin_install_dataengine_kernels.py
index 5a5ff20..4d32e76 100644
--- a/infrastructure-provisioning/src/general/scripts/os/zeppelin_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/zeppelin_install_dataengine_kernels.py
@@ -61,12 +61,12 @@ def configure_notebook(keyfile, hoststring):
put(scripts_dir + 'zeppelin_dataengine_create_configs.py',
'/usr/local/bin/zeppelin_dataengine_create_configs.py', use_sudo=True)
sudo('chmod 755 /usr/local/bin/zeppelin_dataengine_create_configs.py')
- if not exists('/usr/lib/python3.6/datalab/'):
- sudo('mkdir -p /usr/lib/python3.6/datalab/')
- put('/usr/lib/python3.6/datalab/*', '/usr/lib/python3.6/datalab/', use_sudo=True)
- sudo('chmod a+x /usr/lib/python3.6/datalab/*')
+ if not exists('/usr/lib/python3.8/datalab/'):
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
+ put('/usr/lib/python3.8/datalab/*', '/usr/lib/python3.8/datalab/', use_sudo=True)
+ sudo('chmod a+x /usr/lib/python3.8/datalab/*')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python3.6/datalab /usr/lib64/python3.6/datalab')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
def create_inactivity_log(master_ip, hoststring):
reworked_ip = master_ip.replace('.', '-')
diff --git a/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json b/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
index e03c629..45c1213 100644
--- a/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
+++ b/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
@@ -7,9 +7,9 @@
"{connection_file}"
],
"language": "python",
- "display_name": "Local PySpark (Python-3.6 / Spark-SP_VER )",
+ "display_name": "Local PySpark (Python-3.8 / Spark-SP_VER )",
"env": {
- "PYSPARK_PYTHON": "python3.6",
+ "PYSPARK_PYTHON": "python3.8",
"SPARK_HOME": "/opt/spark/",
"PYTHONPATH": "PY4J:/opt/spark/python/:",
"PYTHONSTARTUP": "/opt/spark/python/pyspark/shell.py",
diff --git a/infrastructure-provisioning/src/general/templates/os/pyspark_local_template.json b/infrastructure-provisioning/src/general/templates/os/pyspark_local_template.json
index 7533f3a..1001f0f 100644
--- a/infrastructure-provisioning/src/general/templates/os/pyspark_local_template.json
+++ b/infrastructure-provisioning/src/general/templates/os/pyspark_local_template.json
@@ -1,15 +1,15 @@
{
"argv": [
- "/usr/bin/python3.6",
+ "/usr/bin/python3.8",
"-m",
"ipykernel",
"-f",
"{connection_file}"
],
"language": "python",
- "display_name": "Local PySpark (Python-3.6 / Spark-SP_VER )",
+ "display_name": "Local PySpark (Python-3.8 / Spark-SP_VER )",
"env": {
- "PYSPARK_PYTHON": "python3.6",
+ "PYSPARK_PYTHON": "python3.8",
"SPARK_HOME": "/opt/spark/",
"PYTHONPATH": "PY4J:/opt/spark/python/:",
"PYTHONSTARTUP": "/opt/spark/python/pyspark/shell.py",
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
index 4c209ff..43d7b8e 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
@@ -90,13 +90,13 @@ keystore_passwd = id_generator()
def copy_ssn_libraries():
try:
- sudo('mkdir -p /usr/lib/python3.6/datalab/')
+ sudo('mkdir -p /usr/lib/python3.8/datalab/')
run('mkdir -p /tmp/datalab_libs/')
- local('scp -i {} /usr/lib/python3.6/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
run('chmod a+x /tmp/datalab_libs/*')
- sudo('mv /tmp/datalab_libs/* /usr/lib/python3.6/datalab/')
+ sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python3.6/datalab /usr/lib64/python3.6/datalab')
+ sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
except Exception as err:
traceback.print_exc()
print('Failed to copy ssn libraries: ', str(err))
diff --git a/infrastructure-provisioning/terraform/gcp/ssn-gke/README.md b/infrastructure-provisioning/terraform/gcp/ssn-gke/README.md
index 82fdab2..8e8766f 100644
--- a/infrastructure-provisioning/terraform/gcp/ssn-gke/README.md
+++ b/infrastructure-provisioning/terraform/gcp/ssn-gke/README.md
@@ -34,7 +34,7 @@ snap install kubectl --classic
```
add-apt-repository ppa:deadsnakes/ppa
apt-get install python3.7
-update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1
+update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 2
update-alternatives --config python3
apt-get install python3-pip
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org