You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/06/08 13:49:19 UTC
[incubator-datalab] branch DATALAB-2414 created (now 972ee62)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2414
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
at 972ee62 [DATALAB-2414]: fixed error
This branch includes the following new commits:
new 6c694ce [DATALAB-2398]: virtualenv usage for rstudio
new 7ba3184 [DATALAB-2414]: fixed rstudio virtualenv errors
new 972ee62 [DATALAB-2414]: fixed error
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 02/03: [DATALAB-2414]: fixed rstudio virtualenv
errors
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2414
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 7ba3184ee387f093844f155bdacf82d962a29036
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Jun 8 10:53:16 2021 +0300
[DATALAB-2414]: fixed rstudio virtualenv errors
---
.../src/general/lib/os/debian/notebook_lib.py | 2 --
infrastructure-provisioning/src/general/lib/os/fab.py | 7 +++++--
.../src/rstudio/scripts/configure_rstudio_node.py | 2 +-
3 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index 4bcd2d9..8396256 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -256,8 +256,6 @@ def ensure_python3_libraries(os_user):
#manage_pkg('-y install', 'remote', 'python3-setuptools')
manage_pkg('-y install', 'remote', 'python3-pip')
manage_pkg('-y install', 'remote', 'libkrb5-dev')
- #datalab.fab.conn.sudo('add-apt-repository ppa:deadsnakes/ppa')
- #manage_pkg('-y install', 'remote', 'libpython3.7-dev')
datalab.fab.conn.sudo('pip3 install -U keyrings.alt backoff')
datalab.fab.conn.sudo('pip3 install setuptools=={}'.format(os.environ['notebook_setuptools_version']))
try:
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 43759b0..80f10fd 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -43,10 +43,13 @@ def ensure_python_venv(python_venv_version):
if not exists(conn, '/opt/python/python{}'.format(python_venv_version)):
conn.sudo('wget https://www.python.org/ftp/python/{0}/Python-{0}.tgz -O /tmp/Python-{0}.tgz'.format(python_venv_version))
conn.sudo('tar zxvf /tmp/Python-{}.tgz -C /tmp/'.format(python_venv_version))
- conn.sudo('''bash -l -c 'cd /tmp/Python-{0} && ./configure --prefix=/opt/python/python{0} --with-zlib-dir=/usr/local/lib/ --with-ensurepip=install' '''.format(python_venv_version))
+ conn.sudo('''bash -l -c 'cd /tmp/Python-{0} && ./configure --prefix=/opt/python/python{0}
+ --with-zlib-dir=/usr/local/lib/ --with-ensurepip=install --enable-shared' '''.format(python_venv_version))
conn.sudo('''bash -l -c 'cd /tmp/Python-{0} && make altinstall' '''.format(python_venv_version))
conn.sudo('''bash -l -c 'cd /tmp && rm -rf Python-{}' '''.format(python_venv_version))
- conn.sudo('virtualenv /opt/python/python{}'.format(python_venv_version))
+ conn.sudo(
+ '''bash -l -c 'echo "export LD_LIBRARY_PATH=/opt/python/python{}/lib" >> /etc/profile' '''.format(python_venv_version))
+ conn.sudo('''bash -l -c 'virtualenv /opt/python/python{0}' '''.format(python_venv_version))
venv_command = 'source /opt/python/python{}/bin/activate'.format(python_venv_version)
pip_command = '/opt/python/python{0}/bin/pip{1}'.format(python_venv_version, python_venv_version[:3])
conn.sudo('''bash -l -c '{0} && {1} install -U pip=={2}' '''.format(venv_command, pip_command, os.environ['conf_pip_version']))
diff --git a/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py b/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py
index 9ea4f12..9cf8d98 100644
--- a/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py
+++ b/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py
@@ -92,7 +92,7 @@ if __name__ == "__main__":
# INSTALL PYTHON IN VIRTUALENV
print("Configure Python Virtualenv")
- #ensure_python_venv(python_venv_version)
+ ensure_python_venv(python_venv_version)
# INSTALL RSTUDIO
print("Install RStudio")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/03: [DATALAB-2398]: virtualenv usage for
rstudio
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2414
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 6c694ce1044d11c44f34cc6329f1b6802b98210c
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Jun 2 14:58:03 2021 +0300
[DATALAB-2398]: virtualenv usage for rstudio
---
.../src/general/lib/os/debian/notebook_lib.py | 5 ++++-
.../src/rstudio/scripts/configure_rstudio_node.py | 8 +++++++-
2 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index d6d7d63..4bcd2d9 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -122,7 +122,7 @@ def ensure_r(os_user, r_libs, region, r_mirror):
sys.exit(1)
-def install_rstudio(os_user, local_spark_path, rstudio_pass, rstudio_version):
+def install_rstudio(os_user, local_spark_path, rstudio_pass, rstudio_version, python_venv_path=''):
if not exists(datalab.fab.conn,'/home/' + os_user + '/.ensure_dir/rstudio_ensured'):
try:
manage_pkg('-y install', 'remote', 'r-base')
@@ -150,6 +150,7 @@ def install_rstudio(os_user, local_spark_path, rstudio_pass, rstudio_version):
datalab.fab.conn.sudo('chown {0}:{0} /home/{0}/.Renviron'.format(os_user))
datalab.fab.conn.sudo('''echo 'SPARK_HOME="{0}"' >> /home/{1}/.Renviron'''.format(local_spark_path, os_user))
datalab.fab.conn.sudo('''echo 'JAVA_HOME="{0}"' >> /home/{1}/.Renviron'''.format(java_home, os_user))
+ #datalab.fab.conn.sudo('''echo 'RETICULATE_PYTHON="{0}"' >> /home/{1}/.Renviron'''.format(python_venv_path, os_user))
datalab.fab.conn.sudo('touch /home/{}/.Rprofile'.format(os_user))
datalab.fab.conn.sudo('chown {0}:{0} /home/{0}/.Rprofile'.format(os_user))
datalab.fab.conn.sudo('''echo 'library(SparkR, lib.loc = c(file.path(Sys.getenv("SPARK_HOME"), "R", "lib")))' >> /home/{}/.Rprofile'''.format(os_user))
@@ -255,6 +256,8 @@ def ensure_python3_libraries(os_user):
#manage_pkg('-y install', 'remote', 'python3-setuptools')
manage_pkg('-y install', 'remote', 'python3-pip')
manage_pkg('-y install', 'remote', 'libkrb5-dev')
+ #datalab.fab.conn.sudo('add-apt-repository ppa:deadsnakes/ppa')
+ #manage_pkg('-y install', 'remote', 'libpython3.7-dev')
datalab.fab.conn.sudo('pip3 install -U keyrings.alt backoff')
datalab.fab.conn.sudo('pip3 install setuptools=={}'.format(os.environ['notebook_setuptools_version']))
try:
diff --git a/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py b/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py
index cf0679e..9ea4f12 100644
--- a/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py
+++ b/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py
@@ -47,6 +47,8 @@ args = parser.parse_args()
spark_version = os.environ['notebook_spark_version']
hadoop_version = os.environ['notebook_hadoop_version']
+python_venv_version = os.environ['notebook_python_venv_version']
+python_venv_path = '/opt/python/python{0}/bin/python{1}'.format(python_venv_version, python_venv_version[:3])
if args.region == 'cn-north-1':
spark_link = "http://mirrors.hust.edu.cn/apache/spark/spark-" + spark_version + "/spark-" + spark_version + \
"-bin-hadoop" + hadoop_version + ".tgz"
@@ -88,9 +90,13 @@ if __name__ == "__main__":
print("Install Python 3 modules")
ensure_python3_libraries(args.os_user)
+ # INSTALL PYTHON IN VIRTUALENV
+ print("Configure Python Virtualenv")
+ #ensure_python_venv(python_venv_version)
+
# INSTALL RSTUDIO
print("Install RStudio")
- install_rstudio(args.os_user, local_spark_path, args.rstudio_pass, args.rstudio_version)
+ install_rstudio(args.os_user, local_spark_path, args.rstudio_pass, args.rstudio_version, python_venv_path)
# INSTALL SPARK AND CLOUD STORAGE JARS FOR SPARK
print("Install local Spark")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 03/03: [DATALAB-2414]: fixed error
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2414
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 972ee622b6d8ca85a607c945ae02cb807ebec76c
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Jun 8 16:48:29 2021 +0300
[DATALAB-2414]: fixed error
---
infrastructure-provisioning/src/general/lib/os/fab.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 80f10fd..3f5ef1e 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -43,8 +43,7 @@ def ensure_python_venv(python_venv_version):
if not exists(conn, '/opt/python/python{}'.format(python_venv_version)):
conn.sudo('wget https://www.python.org/ftp/python/{0}/Python-{0}.tgz -O /tmp/Python-{0}.tgz'.format(python_venv_version))
conn.sudo('tar zxvf /tmp/Python-{}.tgz -C /tmp/'.format(python_venv_version))
- conn.sudo('''bash -l -c 'cd /tmp/Python-{0} && ./configure --prefix=/opt/python/python{0}
- --with-zlib-dir=/usr/local/lib/ --with-ensurepip=install --enable-shared' '''.format(python_venv_version))
+ conn.sudo('''bash -l -c 'cd /tmp/Python-{0} && ./configure --prefix=/opt/python/python{0} --with-zlib-dir=/usr/local/lib/ --with-ensurepip=install --enable-shared' '''.format(python_venv_version))
conn.sudo('''bash -l -c 'cd /tmp/Python-{0} && make altinstall' '''.format(python_venv_version))
conn.sudo('''bash -l -c 'cd /tmp && rm -rf Python-{}' '''.format(python_venv_version))
conn.sudo(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org