You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by my...@apache.org on 2021/05/28 09:20:50 UTC
[incubator-datalab] branch DATALAB-2398 created (now 2943160)
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a change to branch DATALAB-2398
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
at 2943160 [DATALAB-2398] - Installing libs for python venv fixed
This branch includes the following new commits:
new b64143d [DATALAB-2398] - Python installation in virtualenv added
new 9fc649a [DATALAB-2398] - Jupyter PySpark kernel from python venv implemented
new 2943160 [DATALAB-2398] - Installing libs for python venv fixed
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/03: [DATALAB-2398] - Python installation in
virtualenv added
Posted by my...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DATALAB-2398
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit b64143d162b8a5776934fa4d43173b316d490f71
Author: bodnarmykola <bo...@gmail.com>
AuthorDate: Thu May 20 11:51:38 2021 +0300
[DATALAB-2398] - Python installation in virtualenv added
---
.../src/general/lib/os/fab.py | 25 ++++++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 3185b65..a9daee1 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -38,6 +38,31 @@ from fabric import *
from patchwork.files import exists
from patchwork import files
+def ensure_python_venv(python_version):
+ try:
+ if not exist(conn, '/opt/python/python{}'.format(python_version)):
+ conn.sudo('wget https://www.python.org/ftp/python/{0}/Python-{0}.tgz -O /tmp/Python-{0}.tgz'.format(python_version))
+ conn.sudo('tar zxvf /tmp/Python-{}.tgz -C /tmp/'.format(python_version))
+ conn.sudo('cd /tmp/Python-{0} && ./configure --prefix=/opt/python/python{0} --with-zlib-dir=/usr/local/lib/ --with-ensurepip=install'.format(
+ python_version)
+ conn.sudo('cd /tmp/Python-{0} make altinstall'.format(python_version))
+ conn.sudo('cd /tmp && rm -rf Python-{}'.format(python_version))
+ conn.sudo('virtualenv /opt/python/python{}'.format(python_version))
+ venv_command = 'source /opt/python/python{}/bin/activate'.format(python_version)
+ pip_command = '/opt/python/python{0}/bin/pip{1}'.format(python_version, python_version[:3])
+ conn.sudo('{0} && install -U pip=={}'.format(venv_command, pip_command, os.environ['pip_version']))
+ subprocess.run({0} && {1} install pyzmq==17.0.0'.format(venv_command, pip_command))
+ subprocess.run(
+ '{0} && sudo -i {1} install ipython ipykernel --no-cache-dir'.format(venv_command, pip_command),
+ shell=True, check=True)
+ subprocess.run(
+ '{0} && sudo -i {1} install boto boto3 NumPy=={2} SciPy Matplotlib pandas Sympy Pillow sklearn --no-cache-dir'
+ .format(venv_command, pip_command, numpy_version), shell=True, check=True)
+
+ except Exception as err:
+ print('Error:', str(err))
+ sys.exit(1)
+
def ensure_pip(requisites):
try:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 03/03: [DATALAB-2398] - Installing libs for
python venv fixed
Posted by my...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DATALAB-2398
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 294316085f3257cdc4929f88c1b4187b59326cd6
Author: bodnarmykola <bo...@gmail.com>
AuthorDate: Fri May 28 12:19:29 2021 +0300
[DATALAB-2398] - Installing libs for python venv fixed
---
.../src/general/lib/os/fab.py | 54 ++++++++++++++--------
1 file changed, 34 insertions(+), 20 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 8b54a2e..9447242 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -79,13 +79,14 @@ def install_pip_pkg(requisites, pip_version, lib_group):
status = list()
error_parser = "Could not|No matching|ImportError:|failed|EnvironmentError:|requires|FileNotFoundError:|RuntimeError:|error:"
try:
- if pip_version == 'pip3' and not exists(conn, '/bin/pip3'):
- for v in range(4, 8):
- if exists(conn, '/bin/pip3.{}'.format(v)):
- conn.sudo('ln -s /bin/pip3.{} /bin/pip3'.format(v))
- conn.sudo('{} install -U pip=={} setuptools=={}'.format(pip_version, os.environ['conf_pip_version'], os.environ['notebook_setuptools_version']))
- conn.sudo('{} install -U pip=={} --no-cache-dir'.format(pip_version, os.environ['conf_pip_version']))
- conn.sudo('{} install --upgrade pip=={}'.format(pip_version, os.environ['conf_pip_version']))
+ venv_install_command = 'source /opt/python/python{0}/bin/activate && /opt/python/python{0}/bin/pip{1}'.format(os.environ['notebook_python_venv_version'], os.environ['notebook_python_venv_version'][:3])
+ #if pip_version == 'pip3' and not exists(conn, '/bin/pip3'):
+ # for v in range(4, 8):
+ # if exists(conn, '/bin/pip3.{}'.format(v)):
+ # conn.sudo('ln -s /bin/pip3.{} /bin/pip3'.format(v))
+ #conn.sudo('{} install -U pip=={} setuptools=={}'.format(pip_version, os.environ['conf_pip_version'], os.environ['notebook_setuptools_version']))
+ #conn.sudo('{} install -U pip=={} --no-cache-dir'.format(pip_version, os.environ['conf_pip_version']))
+ #conn.sudo('{} install --upgrade pip=={}'.format(pip_version, os.environ['conf_pip_version']))
for pip_pkg in requisites:
name, vers = pip_pkg
if pip_pkg[1] == '' or pip_pkg[1] == 'N/A':
@@ -94,20 +95,30 @@ def install_pip_pkg(requisites, pip_version, lib_group):
else:
version = pip_pkg[1]
pip_pkg = "{}=={}".format(pip_pkg[0], pip_pkg[1])
- conn.sudo('{0} install -U {1} --use-deprecated=legacy-resolver --no-cache-dir 2>&1 | tee /tmp/tee.tmp; if ! grep -w -i -E "({2})" /tmp/tee.tmp > '
- ' /tmp/{0}install_{3}.log; then echo "" > /tmp/{0}install_{3}.log;fi'.format(pip_version, pip_pkg, error_parser, name))
- err = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, pip_pkg.split("==")[0])).stdout.replace('"', "'").replace('\n', ' ')
- conn.sudo('{0} freeze --all | if ! grep -w -i {1} > /tmp/{0}install_{1}.list; then echo "not_found" > /tmp/{0}install_{1}.list;fi'.format(pip_version, name))
- res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, name)).stdout.replace('\n', '')
- conn.sudo('cat /tmp/tee.tmp | if ! grep "Successfully installed" > /tmp/{0}install_{1}.list; then echo "not_installed" > /tmp/{0}install_{1}.list;fi'.format(pip_version, name))
- installed_out = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, name)).stdout.replace('\n', '')
+ conn.sudo(
+ '''bash -l -c '{0} install -U {1} --use-deprecated=legacy-resolver --no-cache-dir 2>&1 | tee /tmp/tee.tmp; if ! grep -w -i -E "({2})" /tmp/tee.tmp > /tmp/{4}install_{3}.log; then echo "" > /tmp/{4}install_{3}.log;fi' '''.format(
+ venv_install_command, pip_pkg, error_parser, name, pip_version))
+ err = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, pip_pkg.split("==")[0])).stdout.replace(
+ '"', "'").replace('\n', ' ')
+ conn.sudo(
+ '''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}install_{1}.list; then echo "not_found" > /tmp/{2}install_{1}.list;fi' '''.format(
+ venv_install_command, name, pip_version))
+ res = conn.sudo('''bash -l -c 'cat /tmp/{0}install_{1}.list' '''.format(pip_version, name)).stdout.replace(
+ '\n', '')
+ conn.sudo(
+ '''bash -l -c 'cat /tmp/tee.tmp | if ! grep "Successfully installed" > /tmp/{0}install_{1}.list; then echo "not_installed" > /tmp/{0}install_{1}.list;fi' '''.format(
+ pip_version, name))
+ installed_out = conn.sudo(
+ '''bash -l -c 'cat /tmp/{0}install_{1}.list' '''.format(pip_version, name)).stdout.replace('\n', '')
changed_pip_pkg = False
if 'not_found' in res:
changed_pip_pkg = pip_pkg.split("==")[0].replace("_", "-").split('-')
changed_pip_pkg = changed_pip_pkg[0]
- conn.sudo('{0} freeze --all | if ! grep -w -i {1} > /tmp/{0}install_{1}.list; then echo "" > '
- '/tmp/{0}install_{1}.list;fi'.format(pip_version, changed_pip_pkg))
- res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, changed_pip_pkg)).stdout.replace('\n', '')
+ conn.sudo(
+ '''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}install_{1}.list; then echo "" > /tmp/{2}install_{1}.list;fi' '''.format(
+ venv_install_command, changed_pip_pkg, pip_version))
+ res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, changed_pip_pkg)).stdout.replace(
+ '\n', '')
if err and name not in installed_out:
status_msg = 'installation_error'
if 'ERROR: No matching distribution found for {}'.format(name) in err:
@@ -120,10 +131,11 @@ def install_pip_pkg(requisites, pip_version, lib_group):
version = [i for i in ver if changed_pip_pkg.lower() in i][0].split('==')[1]
else:
version = \
- [i for i in ver if pip_pkg.split("==")[0].lower() in i][0].split('==')[1]
+ [i for i in ver if pip_pkg.split("==")[0].lower() in i][0].split('==')[1]
status_msg = "installed"
versions = []
- if 'Could not find a version that satisfies the requirement' in err and 'ERROR: No matching distribution found for {}=='.format(name) in err:
+ if 'Could not find a version that satisfies the requirement' in err and 'ERROR: No matching distribution found for {}=='.format(
+ name) in err:
versions = err[err.find("(from versions: ") + 16: err.find(") ")]
if versions != '' and versions != 'none':
versions = versions.split(', ')
@@ -132,7 +144,9 @@ def install_pip_pkg(requisites, pip_version, lib_group):
else:
versions = []
- conn.sudo('cat /tmp/tee.tmp | if ! grep -w -i -E "Installing collected packages:" > /tmp/{0}install_{1}.log; then echo "" > /tmp/{0}install_{1}.log;fi'.format(pip_version, name))
+ conn.sudo(
+ '''bash -l -c 'cat /tmp/tee.tmp | if ! grep -w -i -E "Installing collected packages:" > /tmp/{0}install_{1}.log; then echo "" > /tmp/{0}install_{1}.log;fi' '''.format(
+ pip_version, name))
dep = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, name)).stdout.replace('\n', '').strip()[31:]
if dep == '':
dep = []
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 02/03: [DATALAB-2398] - Jupyter PySpark kernel
from python venv implemented
Posted by my...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DATALAB-2398
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 9fc649a2be054869883d30646f0c014629163ee0
Author: bodnarmykola <bo...@gmail.com>
AuthorDate: Wed May 26 19:06:43 2021 +0300
[DATALAB-2398] - Jupyter PySpark kernel from python venv implemented
---
.../src/general/conf/datalab.ini | 2 ++
.../src/general/lib/os/fab.py | 37 ++++++++++------------
.../templates/os/py3spark_local_template.json | 10 +++---
.../src/jupyter/scripts/configure_jupyter_node.py | 9 ++++--
4 files changed, 32 insertions(+), 26 deletions(-)
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini
index 2679b22..2040dc8 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -248,6 +248,8 @@ spark_version = 3.0.1
hadoop_version = 2.7
### Version of Jupyter to be installed on notebook
jupyter_version = 6.1.6
+### Version of Python to be installed as virualenv on notebook
+python_venv_version = 3.7.9
### Version of TensorFlow to be installed on notebook
tensorflow_version = 2.3.2
### Version of Zeppelin to be installed on notebook
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index a9daee1..8b54a2e 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -38,26 +38,20 @@ from fabric import *
from patchwork.files import exists
from patchwork import files
-def ensure_python_venv(python_version):
+def ensure_python_venv(python_venv_version):
try:
- if not exist(conn, '/opt/python/python{}'.format(python_version)):
- conn.sudo('wget https://www.python.org/ftp/python/{0}/Python-{0}.tgz -O /tmp/Python-{0}.tgz'.format(python_version))
- conn.sudo('tar zxvf /tmp/Python-{}.tgz -C /tmp/'.format(python_version))
- conn.sudo('cd /tmp/Python-{0} && ./configure --prefix=/opt/python/python{0} --with-zlib-dir=/usr/local/lib/ --with-ensurepip=install'.format(
- python_version)
- conn.sudo('cd /tmp/Python-{0} make altinstall'.format(python_version))
- conn.sudo('cd /tmp && rm -rf Python-{}'.format(python_version))
- conn.sudo('virtualenv /opt/python/python{}'.format(python_version))
- venv_command = 'source /opt/python/python{}/bin/activate'.format(python_version)
- pip_command = '/opt/python/python{0}/bin/pip{1}'.format(python_version, python_version[:3])
- conn.sudo('{0} && install -U pip=={}'.format(venv_command, pip_command, os.environ['pip_version']))
- subprocess.run({0} && {1} install pyzmq==17.0.0'.format(venv_command, pip_command))
- subprocess.run(
- '{0} && sudo -i {1} install ipython ipykernel --no-cache-dir'.format(venv_command, pip_command),
- shell=True, check=True)
- subprocess.run(
- '{0} && sudo -i {1} install boto boto3 NumPy=={2} SciPy Matplotlib pandas Sympy Pillow sklearn --no-cache-dir'
- .format(venv_command, pip_command, numpy_version), shell=True, check=True)
+ if not exists(conn, '/opt/python/python{}'.format(python_venv_version)):
+ conn.sudo('wget https://www.python.org/ftp/python/{0}/Python-{0}.tgz -O /tmp/Python-{0}.tgz'.format(python_venv_version))
+ conn.sudo('tar zxvf /tmp/Python-{}.tgz -C /tmp/'.format(python_venv_version))
+ conn.sudo('''bash -l -c 'cd /tmp/Python-{0} && ./configure --prefix=/opt/python/python{0} --with-zlib-dir=/usr/local/lib/ --with-ensurepip=install' '''.format(python_venv_version))
+ conn.sudo('''bash -l -c 'cd /tmp/Python-{0} && make altinstall' '''.format(python_venv_version))
+ conn.sudo('''bash -l -c 'cd /tmp && rm -rf Python-{}' '''.format(python_venv_version))
+ conn.sudo('virtualenv /opt/python/python{}'.format(python_venv_version))
+ venv_command = 'source /opt/python/python{}/bin/activate'.format(python_venv_version)
+ pip_command = '/opt/python/python{0}/bin/pip{1}'.format(python_venv_version, python_venv_version[:3])
+ conn.sudo('''bash -l -c '{0} && {1} install -U pip=={2}' '''.format(venv_command, pip_command, os.environ['conf_pip_version']))
+ conn.sudo('''bash -l -c '{0} && {1} install ipython ipykernel --no-cache-dir' '''.format(venv_command, pip_command))
+ conn.sudo('''bash -l -c '{0} && {1} install NumPy=={2} SciPy Matplotlib pandas Sympy Pillow sklearn --no-cache-dir' '''.format(venv_command, pip_command, os.environ['notebook_numpy_version']))
except Exception as err:
print('Error:', str(err))
@@ -404,7 +398,7 @@ def ensure_pyspark_local_kernel(os_user, pyspark_local_path_dir, templates_dir,
sys.exit(1)
-def ensure_py3spark_local_kernel(os_user, py3spark_local_path_dir, templates_dir, spark_version):
+def ensure_py3spark_local_kernel(os_user, py3spark_local_path_dir, templates_dir, spark_version, python_venv_path, python_venv_version):
if not exists(conn,'/home/' + os_user + '/.ensure_dir/py3spark_local_kernel_ensured'):
try:
conn.sudo('mkdir -p ' + py3spark_local_path_dir)
@@ -412,6 +406,9 @@ def ensure_py3spark_local_kernel(os_user, py3spark_local_path_dir, templates_dir
conn.put(templates_dir + 'py3spark_local_template.json', '/tmp/py3spark_local_template.json')
conn.sudo(
'''bash -l -c "PYJ=`find /opt/spark/ -name '*py4j*.zip' | tr '\\n' ':' | sed 's|:$||g'`; sed -i 's|PY4J|'$PYJ'|g' /tmp/py3spark_local_template.json" ''')
+ conn.sudo('sed -i "s|PYTHON_VENV_PATH|' + python_venv_path + '|g" /tmp/py3spark_local_template.json')
+ conn.sudo('sed -i "s|PYTHON_VENV_VERSION|' + python_venv_version + '|g" /tmp/py3spark_local_template.json')
+ conn.sudo('sed -i "s|PYTHON_VENV_SHORT_VERSION|' + python_venv_version[:3] + '|g" /tmp/py3spark_local_template.json')
conn.sudo('sed -i "s|SP_VER|' + spark_version + '|g" /tmp/py3spark_local_template.json')
conn.sudo('sed -i \'/PYTHONPATH\"\:/s|\(.*\)"|\\1/home/{0}/caffe/python:/home/{0}/pytorch/build:"|\' /tmp/py3spark_local_template.json'.format(os_user))
conn.sudo('\cp /tmp/py3spark_local_template.json ' + py3spark_local_path_dir + 'kernel.json')
diff --git a/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json b/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
index 45c1213..428524d 100644
--- a/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
+++ b/infrastructure-provisioning/src/general/templates/os/py3spark_local_template.json
@@ -1,18 +1,20 @@
{
"argv": [
- "/usr/bin/python3",
+ "PYTHON_VENV_PATH",
"-m",
"ipykernel",
"-f",
"{connection_file}"
],
"language": "python",
- "display_name": "Local PySpark (Python-3.8 / Spark-SP_VER )",
+ "display_name": "Local PySpark (Python-PYTHON_VENV_VERSION / Spark-SP_VER )",
"env": {
- "PYSPARK_PYTHON": "python3.8",
+ "PYSPARK_PYTHON": "pythonPYTHON_VENV_SHORT_VERSION",
"SPARK_HOME": "/opt/spark/",
"PYTHONPATH": "PY4J:/opt/spark/python/:",
"PYTHONSTARTUP": "/opt/spark/python/pyspark/shell.py",
"PYSPARK_SUBMIT_ARGS": "--name LocalPySpark pyspark-shell"
}
-}
\ No newline at end of file
+}
+
+
diff --git a/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py b/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
index 2791b78..76c5fbf 100644
--- a/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
+++ b/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
@@ -45,6 +45,7 @@ args = parser.parse_args()
spark_version = args.spark_version
hadoop_version = args.hadoop_version
jupyter_version = os.environ['notebook_jupyter_version']
+python_venv_version = os.environ['notebook_python_venv_version']
scala_link = "https://www.scala-lang.org/files/archive/"
if args.region == 'cn-north-1':
spark_link = "http://mirrors.hust.edu.cn/apache/spark/spark-" + spark_version + "/spark-" + spark_version + \
@@ -52,7 +53,7 @@ if args.region == 'cn-north-1':
else:
spark_link = "https://archive.apache.org/dist/spark/spark-" + spark_version + "/spark-" + spark_version + \
"-bin-hadoop" + hadoop_version + ".tgz"
-
+python_venv_path = '/opt/python/python{0}/bin/python{1}'.format(python_venv_version, python_venv_version[:3])
pyspark_local_path_dir = '/home/' + args.os_user + '/.local/share/jupyter/kernels/pyspark_local/'
py3spark_local_path_dir = '/home/' + args.os_user + '/.local/share/jupyter/kernels/py3spark_local/'
jupyter_conf_file = '/home/' + args.os_user + '/.local/share/jupyter/jupyter_notebook_config.py'
@@ -96,6 +97,10 @@ if __name__ == "__main__":
print("Install Python 3 modules")
ensure_python3_libraries(args.os_user)
+ # INSTALL PYTHON IN VIRTUALENV
+ print("Configure Python Virtualenv")
+ ensure_python_venv(python_venv_version)
+
# INSTALL JUPYTER NOTEBOOK
print("Install Jupyter")
configure_jupyter(args.os_user, jupyter_conf_file, templates_dir, jupyter_version, args.exploratory_name)
@@ -115,7 +120,7 @@ if __name__ == "__main__":
#print("Install pyspark local kernel for Jupyter")
#ensure_pyspark_local_kernel(args.os_user, pyspark_local_path_dir, templates_dir, spark_version)
print("Install py3spark local kernel for Jupyter")
- ensure_py3spark_local_kernel(args.os_user, py3spark_local_path_dir, templates_dir, spark_version)
+ ensure_py3spark_local_kernel(args.os_user, py3spark_local_path_dir, templates_dir, spark_version, python_venv_path, python_venv_version)
print("Install Toree-Scala kernel for Jupyter")
ensure_toree_local_kernel(args.os_user, toree_link, scala_kernel_path, files_dir, local_spark_scala_version, spark_version)
if os.environ['notebook_r_enabled'] == 'true':
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org