You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by my...@apache.org on 2021/05/28 09:20:52 UTC
[incubator-datalab] 02/03: [DATALAB-2398] - Jupyter PySpark kernel
from python venv implemented
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DATALAB-2398
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 9fc649a2be054869883d30646f0c014629163ee0
Author: bodnarmykola <bo...@gmail.com>
AuthorDate: Wed May 26 19:06:43 2021 +0300
[DATALAB-2398] - Jupyter PySpark kernel from python venv implemented
---
.../src/general/conf/datalab.ini | 2 ++
.../src/general/lib/os/fab.py | 37 ++++++++++------------
.../templates/os/py3spark_local_template.json | 10 +++---
.../src/jupyter/scripts/configure_jupyter_node.py | 9 ++++--
4 files changed, 32 insertions(+), 26 deletions(-)
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini
index 2679b22..2040dc8 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -248,6 +248,8 @@ spark_version = 3.0.1
hadoop_version = 2.7
### Version of Jupyter to be installed on notebook
jupyter_version = 6.1.6
+### Version of Python to be installed as virualenv on notebook
+python_venv_version = 3.7.9
### Version of TensorFlow to be installed on notebook
tensorflow_version = 2.3.2
### Version of Zeppelin to be installed on notebook
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index a9daee1..8b54a2e 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -38,26 +38,20 @@ from fabric import *
from patchwork.files import exists
from patchwork import files
-def ensure_python_venv(python_version):
+def ensure_python_venv(python_venv_version):
try:
- if not exist(conn, '/opt/python/python{}'.format(python_version)):
- conn.sudo('wget https://www.python.org/ftp/python/{0}/Python-{0}.tgz -O /tmp/Python-{0}.tgz'.format(python_version))
- conn.sudo('tar zxvf /tmp/Python-{}.tgz -C /tmp/'.format(python_version))
- conn.sudo('cd /tmp/Python-{0} && ./configure --prefix=/opt/python/python{0} --with-zlib-dir=/usr/local/lib/ --with-ensurepip=install'.format(
- python_version)
- conn.sudo('cd /tmp/Python-{0} make altinstall'.format(python_version))
- conn.sudo('cd /tmp && rm -rf Python-{}'.format(python_version))
- conn.sudo('virtualenv /opt/python/python{}'.format(python_version))
- venv_command = 'source /opt/python/python{}/bin/activate'.format(python_version)
- pip_command = '/opt/python/python{0}/bin/pip{1}'.format(python_version, python_version[:3])
- conn.sudo('{0} && install -U pip=={}'.format(venv_command, pip_command, os.environ['pip_version']))
- subprocess.run({0} && {1} install pyzmq==17.0.0'.format(venv_command, pip_command))
- subprocess.run(
- '{0} && sudo -i {1} install ipython ipykernel --no-cache-dir'.format(venv_command, pip_command),
- shell=True, check=True)
- subprocess.run(
- '{0} && sudo -i {1} install boto boto3 NumPy=={2} SciPy Matplotlib pandas Sympy Pillow sklearn --no-cache-dir'
- .format(venv_command, pip_command, numpy_version), shell=True, check=True)
+ if not exists(conn, '/opt/python/python{}'.format(python_venv_version)):
+ conn.sudo('wget https://www.python.org/ftp/python/{0}/Python-{0}.tgz -O /tmp/Python-{0}.tgz'.format(python_venv_version))
+ conn.sudo('tar zxvf /tmp/Python-{}.tgz -C /tmp/'.format(python_venv_version))
+ conn.sudo('''bash -l -c 'cd /tmp/Python-{0} && ./configure --prefix=/opt/python/python{0} --with-zlib-dir=/usr/local/lib/ --with-ensurepip=install' '''.format(python_venv_version))
+ conn.sudo('''bash -l -c 'cd /tmp/Python-{0} && make altinstall' '''.format(python_venv_version))
+ conn.sudo('''bash -l -c 'cd /tmp && rm -rf Python-{}' '''.format(python_venv_version))
+ conn.sudo('virtualenv /opt/python/python{}'.format(python_venv_version))
+ venv_command = 'source /opt/python/python{}/bin/activate'.format(python_venv_version)
+ pip_command = '/opt/python/python{0}/bin/pip{1}'.format(python_venv_version, python_venv_version[:3])
+ conn.sudo('''bash -l -c '{0} && {1} install -U pip=={2}' '''.format(venv_command, pip_command, os.environ['conf_pip_version']))
+ conn.sudo('''bash -l -c '{0} && {1} install ipython ipykernel --no-cache-dir' '''.format(venv_command, pip_command))
+ conn.sudo('''bash -l -c '{0} && {1} install NumPy=={2} SciPy Matplotlib pandas Sympy Pillow sklearn --no-cache-dir' '''.format(venv_command, pip_command, os.environ['notebook_numpy_version']))
except Exception as err:
print('Error:', str(err))
@@ -404,7 +398,7 @@ def ensure_pyspark_local_kernel(os_user, pyspark_local_path_dir, templates_dir,
sys.exit(1)
-def ensure_py3spark_local_kernel(os_user, py3spark_local_path_dir, templates_dir, spark_version):
+def ensure_py3spark_local_kernel(os_user, py3spark_local_path_dir, templates_dir, spark_version, python_venv_path, python_venv_version):
if not exists(conn,'/home/' + os_user + '/.ensure_dir/py3spark_local_kernel_ensured'):
try:
conn.sudo('mkdir -p ' + py3spark_local_path_dir)
@@ -412,6 +406,9 @@ def ensure_py3spark_local_kernel(os_user, py3spark_local_path_dir, templates_dir
conn.put(templates_dir + 'py3spark_local_template.json', '/tmp/py3spark_local_template.json')
conn.sudo(
'''bash -l -c "PYJ=`find /opt/spark/ -name '*py4j*.zip' | tr '\\n' ':' | sed 's|:$||g'`; sed -i 's|PY4J|'$PYJ'|g' /tmp/py3spark_local_template.json" ''')
+ conn.sudo('sed -i "s|PYTHON_VENV_PATH|' + python_venv_path + '|g" /tmp/py3spark_local_template.json')
+ conn.sudo('sed -i "s|PYTHON_VENV_VERSION|' + python_venv_version + '|g" /tmp/py3spark_local_template.json')
+ conn.sudo('sed -i "s|PYTHON_VENV_SHORT_VERSION|' + python_venv_version[:3] + '|g" /tmp/py3spark_local_template.json')
conn.sudo('sed -i "s|SP_VER|' + spark_version + '|g" /tmp/py3spark_local_template.json')
conn.sudo('sed -i \'/PYTHONPATH\"\:/s|\(.*\)"|\\1/home/{0}/caffe/python:/home/{0}/pytorch/build:"|\' /tmp/py3spark_local_template.json'.format(os_user))
conn.sudo('\cp /tmp/py3spark_local_template.json ' + py3spark_local_path_dir + 'kernel.json')
diff --git a/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json b/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
index 45c1213..428524d 100644
--- a/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
+++ b/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
@@ -1,18 +1,20 @@
{
"argv": [
- "/usr/bin/python3",
+ "PYTHON_VENV_PATH",
"-m",
"ipykernel",
"-f",
"{connection_file}"
],
"language": "python",
- "display_name": "Local PySpark (Python-3.8 / Spark-SP_VER )",
+ "display_name": "Local PySpark (Python-PYTHON_VENV_VERSION / Spark-SP_VER )",
"env": {
- "PYSPARK_PYTHON": "python3.8",
+ "PYSPARK_PYTHON": "pythonPYTHON_VENV_SHORT_VERSION",
"SPARK_HOME": "/opt/spark/",
"PYTHONPATH": "PY4J:/opt/spark/python/:",
"PYTHONSTARTUP": "/opt/spark/python/pyspark/shell.py",
"PYSPARK_SUBMIT_ARGS": "--name LocalPySpark pyspark-shell"
}
-}
\ No newline at end of file
+}
+
+
diff --git a/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py b/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
index 2791b78..76c5fbf 100644
--- a/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
+++ b/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
@@ -45,6 +45,7 @@ args = parser.parse_args()
spark_version = args.spark_version
hadoop_version = args.hadoop_version
jupyter_version = os.environ['notebook_jupyter_version']
+python_venv_version = os.environ['notebook_python_venv_version']
scala_link = "https://www.scala-lang.org/files/archive/"
if args.region == 'cn-north-1':
spark_link = "http://mirrors.hust.edu.cn/apache/spark/spark-" + spark_version + "/spark-" + spark_version + \
@@ -52,7 +53,7 @@ if args.region == 'cn-north-1':
else:
spark_link = "https://archive.apache.org/dist/spark/spark-" + spark_version + "/spark-" + spark_version + \
"-bin-hadoop" + hadoop_version + ".tgz"
-
+python_venv_path = '/opt/python/python{0}/bin/python{1}'.format(python_venv_version, python_venv_version[:3])
pyspark_local_path_dir = '/home/' + args.os_user + '/.local/share/jupyter/kernels/pyspark_local/'
py3spark_local_path_dir = '/home/' + args.os_user + '/.local/share/jupyter/kernels/py3spark_local/'
jupyter_conf_file = '/home/' + args.os_user + '/.local/share/jupyter/jupyter_notebook_config.py'
@@ -96,6 +97,10 @@ if __name__ == "__main__":
print("Install Python 3 modules")
ensure_python3_libraries(args.os_user)
+ # INSTALL PYTHON IN VIRTUALENV
+ print("Configure Python Virtualenv")
+ ensure_python_venv(python_venv_version)
+
# INSTALL JUPYTER NOTEBOOK
print("Install Jupyter")
configure_jupyter(args.os_user, jupyter_conf_file, templates_dir, jupyter_version, args.exploratory_name)
@@ -115,7 +120,7 @@ if __name__ == "__main__":
#print("Install pyspark local kernel for Jupyter")
#ensure_pyspark_local_kernel(args.os_user, pyspark_local_path_dir, templates_dir, spark_version)
print("Install py3spark local kernel for Jupyter")
- ensure_py3spark_local_kernel(args.os_user, py3spark_local_path_dir, templates_dir, spark_version)
+ ensure_py3spark_local_kernel(args.os_user, py3spark_local_path_dir, templates_dir, spark_version, python_venv_path, python_venv_version)
print("Install Toree-Scala kernel for Jupyter")
ensure_toree_local_kernel(args.os_user, toree_link, scala_kernel_path, files_dir, local_spark_scala_version, spark_version)
if os.environ['notebook_r_enabled'] == 'true':
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org