You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/07/02 09:40:04 UTC
[incubator-datalab] branch DATALAB-2448 created (now 5fceef5)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2448
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
at 5fceef5 [DATALAB-2448]: fixed R devtools installation issues for dataproc
This branch includes the following new commits:
new 4d77abe [DATALAB-2448]: changed how devtools dependencies are installed
new 638ed5e [DATALAB-2465]: fixed permission denied to tmp file during lib install
new 5fceef5 [DATALAB-2448]: fixed R devtools installation issues for dataproc
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 03/03: [DATALAB-2448]: fixed R devtools
installation issues for dataproc
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2448
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 5fceef5aafbe0775e51589c4b2818c8f9e1e28dd
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Fri Jul 2 12:39:46 2021 +0300
[DATALAB-2448]: fixed R devtools installation issues for dataproc
---
infrastructure-provisioning/src/general/lib/os/fab.py | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 9829cc4..5682832 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -536,10 +536,11 @@ def install_r_pkg(requisites):
if os.environ['conf_resource'] in ('dataengine-service'):
if os.environ['conf_cloud_provider'] in ('aws'):
manage_pkg('-y install', 'remote', 'libcurl libcurl-devel')
- elif os.environ['conf_cloud_provider'] in ('gcp'):
- # manage_pkg('-y build-dep', 'remote', 'libcurl4-gnutls-dev libxml2-dev')
- manage_pkg('-y install', 'remote', 'libxml2-dev libcurl4-openssl-dev')
- conn.sudo('R -e "install.packages(\'devtools\', repos = \'cloud.r-project.org\')"')
+ elif (os.environ['conf_cloud_provider'] in 'gcp') and ('-w-' in conn.sudo('hostname').stdout.replace('\n', '')):
+ #manage_pkg('-y build-dep', 'remote', 'libcurl4-gnutls-dev libxml2-dev')
+ manage_pkg('-y install', 'remote', 'libxml2-dev libcurl4-openssl-dev pkg-config')
+ conn.sudo('R -e ".libPaths(\'/usr/lib/R/site-library\')"')
+ conn.sudo('R -e "install.packages(\'devtools\', repos = \'cloud.r-project.org\'"')
for r_pkg in requisites:
name, vers = r_pkg
version = vers
@@ -552,7 +553,9 @@ def install_r_pkg(requisites):
'dependencies = NA)\' 2>&1 | tee /tmp/install_{0}.tmp; if ! grep -w -E "({2})" /tmp/install_{0}.tmp '
'> /tmp/install_{0}.log; then echo "" > /tmp/install_{0}.log;fi'.format(name, vers, error_parser))
else:
- conn.sudo('R -e \'devtools::install_version("{0}", version = {1}, repos = "https://cloud.r-project.org", dependencies = NA)\' 2>&1 | tee /tmp/install_{0}.tmp; if ! grep -w -E "({2})" /tmp/install_{0}.tmp > /tmp/install_{0}.log; then echo "" > /tmp/install_{0}.log;fi'.format(name, vers, error_parser))
+ conn.sudo('R -e \'devtools::install_version("{0}", version = {1}, repos = "https://cloud.r-project.org", '
+ 'dependencies = NA)\' 2>&1 | tee /tmp/install_{0}.tmp; if ! grep -w -E "({2})" /tmp/install_{0}.tmp > '
+ '/tmp/install_{0}.log; then echo "" > /tmp/install_{0}.log;fi'.format(name, vers, error_parser))
dep = conn.sudo('grep "(NA.*->". /tmp/install_' + name + '.tmp | awk \'{print $1}\'').stdout.replace('\n', ' ')
dep_ver = conn.sudo('grep "(NA.*->". /tmp/install_' + name + '.tmp | awk \'{print $4}\'').stdout.replace('\n', ' ').replace(')', '').split(' ')
if dep == '':
@@ -565,6 +568,7 @@ def install_r_pkg(requisites):
else:
dep[n] = '{} v.{}'.format(dep[n], dep_ver[n])
dep = [i for i in dep if i]
+ conn.sudo('hostname')
err = conn.sudo('cat /tmp/install_{0}.log'.format(name)).stdout.replace('"', "'").replace('\n', '')
conn.sudo('R -e \'installed.packages()[,c(3:4)]\' | if ! grep -w {0} > /tmp/install_{0}.list; then echo "" > /tmp/install_{0}.list;fi'.format(name))
res = conn.sudo('cat /tmp/install_{0}.list'.format(name)).stdout.replace('\n', '')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 02/03: [DATALAB-2465]: fixed permission denied
to tmp file during lib install
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2448
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 638ed5e695e038b28077a9c653582fb13a1cbbd9
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Jul 1 12:36:22 2021 +0300
[DATALAB-2465]: fixed permission denied to tmp file during lib install
---
.../src/general/lib/os/debian/notebook_lib.py | 12 ++++--
.../src/general/lib/os/fab.py | 48 +++++++++++-----------
2 files changed, 33 insertions(+), 27 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index df11ceb..94c86a1 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -420,11 +420,14 @@ def install_os_pkg(requisites):
else:
version = 'N/A'
os_pkg = os_pkg[0]
- datalab.fab.conn.sudo('DEBIAN_FRONTEND=noninteractive apt-get -y install --allow-downgrades {0} 2>&1 | tee /tmp/tee.tmp; if ! grep -w -E "({1})" /tmp/tee.tmp > '
- '/tmp/os_install_{2}.log; then echo "no_error" > /tmp/os_install_{2}.log;fi'.format(os_pkg, error_parser, name))
+ datalab.fab.conn.sudo('DEBIAN_FRONTEND=noninteractive apt-get -y install --allow-downgrades {0} 2>&1 | '
+ 'tee /tmp/os_install_{2}.tmp; if ! grep -w -E "({1})" /tmp/os_install_{2}.tmp > '
+ '/tmp/os_install_{2}.log; then echo "no_error" > /tmp/os_install_{2}.log;fi'
+ .format(os_pkg, error_parser, name))
err = datalab.fab.conn.sudo('cat /tmp/os_install_{}.log'.format(name)).stdout.replace('"', "'")
- datalab.fab.conn.sudo('cat /tmp/tee.tmp | if ! grep -w -E -A 30 "({1})" /tmp/tee.tmp > '
- '/tmp/os_install_{0}.log; then echo "no_new_pkgs" > /tmp/os_install_{0}.log;fi'.format(name, new_pkgs_parser))
+ datalab.fab.conn.sudo('cat /tmp/os_install_{0}.tmp | if ! grep -w -E -A 30 "({1})" /tmp/os_install_{0}.tmp > '
+ '/tmp/os_install_{0}.log; then echo "no_new_pkgs" > /tmp/os_install_{0}.log;fi'
+ .format(name, new_pkgs_parser))
dep = datalab.fab.conn.sudo('cat /tmp/os_install_{}.log'.format(name)).stdout
if 'no_new_pkgs' in dep:
dep = []
@@ -460,6 +463,7 @@ def install_os_pkg(requisites):
"error_message": err, "add_pkgs": dep, "available_versions": versions})
datalab.fab.conn.sudo('unattended-upgrades -v')
#datalab.fab.conn.sudo('export LC_ALL=C')
+ datalab.fab.conn.sudo('rm /tmp/*{}*'.format(name))
return status
except Exception as err:
for os_pkg in requisites:
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 71e9c66..9829cc4 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -127,31 +127,32 @@ def install_pip_pkg(requisites, pip_version, lib_group, dataengine_service = Fal
pip_pkg = "{}=={}".format(pip_pkg[0], pip_pkg[1])
conn.sudo(
'''bash -l -c '{0} install -U {1} --use-deprecated=legacy-resolver --no-cache-dir 2>&1 | '''
- '''tee /tmp/tee.tmp; if ! grep -w -i -E "({2})" /tmp/tee.tmp > /tmp/{4}install_{3}.log; '''
- '''then echo "" > /tmp/{4}install_{3}.log;fi' '''.format(
+ '''tee /tmp/{4}_install_{3}.tmp; if ! grep -w -i -E "({2})" /tmp/{4}_install_{3}.tmp > /tmp/{4}_install_{3}.log; '''
+ '''then echo "" > /tmp/{4}_install_{3}.log;fi' '''.format(
install_command, pip_pkg, error_parser, name, pip_version))
- err = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, pip_pkg.split("==")[0])).stdout.replace(
+ err = conn.sudo('cat /tmp/{0}_install_{1}.log'.format(pip_version, pip_pkg.split("==")[0])).stdout.replace(
'"', "'").replace('\n', ' ')
conn.sudo(
- '''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}install_{1}.list; '''
- '''then echo "not_found" > /tmp/{2}install_{1}.list;fi' '''.format(
+ '''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}_install_{1}.list; '''
+ '''then echo "not_found" > /tmp/{2}_install_{1}.list;fi' '''.format(
install_command, name, pip_version))
- res = conn.sudo('''bash -l -c 'cat /tmp/{0}install_{1}.list' '''.format(pip_version, name)).stdout.replace(
+ res = conn.sudo('''bash -l -c 'cat /tmp/{0}_install_{1}.list' '''.format(pip_version, name)).stdout.replace(
'\n', '')
conn.sudo(
- '''bash -l -c 'cat /tmp/tee.tmp | if ! grep -w -i -E "(Successfully installed|up-to-date)" > '''
- '''/tmp/{0}install_{1}.list; then echo "not_installed" > /tmp/{0}install_{1}.list;fi' '''.format(
+ '''bash -l -c 'cat /tmp/{0}_install_{1}.tmp | if ! grep -w -i -E "(Successfully installed|up-to-date)" > '''
+ '''/tmp/{0}_install_{1}.list; then echo "not_installed" > /tmp/{0}_install_{1}.list;fi' '''.format(
pip_version, name))
installed_out = conn.sudo(
- '''bash -l -c 'cat /tmp/{0}install_{1}.list' '''.format(pip_version, name)).stdout.replace('\n', '')
+ '''bash -l -c 'cat /tmp/{0}_install_{1}.list' '''.format(pip_version, name)).stdout.replace('\n', '')
changed_pip_pkg = False
if 'not_found' in res:
changed_pip_pkg = pip_pkg.split("==")[0].replace("_", "-").split('-')
changed_pip_pkg = changed_pip_pkg[0]
conn.sudo(
- '''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}install_{1}.list; then echo "" > /tmp/{2}install_{1}.list;fi' '''.format(
+ '''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}_install_{1}.list; then echo "" > '''
+ '''/tmp/{2}_install_{1}.list;fi' '''.format(
install_command, changed_pip_pkg, pip_version))
- res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, changed_pip_pkg)).stdout.replace(
+ res = conn.sudo('cat /tmp/{0}_install_{1}.list'.format(pip_version, changed_pip_pkg)).stdout.replace(
'\n', '')
if err and name not in installed_out:
status_msg = 'installation_error'
@@ -178,11 +179,9 @@ def install_pip_pkg(requisites, pip_version, lib_group, dataengine_service = Fal
else:
versions = []
- conn.sudo(
- '''bash -l -c 'cat /tmp/tee.tmp | if ! grep -w -i -E "Installing collected packages:" > '''
- '''/tmp/{0}install_{1}.dep; then echo "" > /tmp/{0}install_{1}.dep;fi' '''.format(
- pip_version, name))
- dep = conn.sudo('cat /tmp/{0}install_{1}.dep'.format(pip_version, name)).stdout.replace('\n', '').strip()[31:]
+ conn.sudo('cat /tmp/{0}_install_{1}.tmp | if ! grep -w -i -E "Installing collected packages:" > '
+ '/tmp/{0}_install_{1}.dep; then echo "" > /tmp/{0}_install_{1}.dep;fi'.format(pip_version, name))
+ dep = conn.sudo('cat /tmp/{0}_install_{1}.dep'.format(pip_version, name)).stdout.replace('\n', '').strip()[31:]
if dep == '':
dep = []
else:
@@ -197,6 +196,7 @@ def install_pip_pkg(requisites, pip_version, lib_group, dataengine_service = Fal
dep = [i for i in dep if i]
status.append({"group": lib_group, "name": name, "version": version, "status": status_msg,
"error_message": err, "available_versions": versions, "add_pkgs": dep})
+ conn.sudo('rm -rf /tmp/*{}*'.format(name))
return status
except Exception as err:
for pip_pkg in requisites:
@@ -548,13 +548,13 @@ def install_r_pkg(requisites):
else:
vers = '"{}"'.format(vers)
if name == 'sparklyr':
- conn.run('sudo -i R -e \'devtools::install_version("{0}", version = {1}, repos = "http://cran.us.r-project.org", dependencies = NA)\' 2>&1 | '
- 'tee /tmp/tee.tmp; if ! grep -w -E "({2})" /tmp/tee.tmp > /tmp/install_{0}.log; then echo "" > /tmp/install_{0}.log;fi'.format(name, vers, error_parser))
+ conn.run('sudo -i R -e \'devtools::install_version("{0}", version = {1}, repos = "http://cran.us.r-project.org", '
+ 'dependencies = NA)\' 2>&1 | tee /tmp/install_{0}.tmp; if ! grep -w -E "({2})" /tmp/install_{0}.tmp '
+ '> /tmp/install_{0}.log; then echo "" > /tmp/install_{0}.log;fi'.format(name, vers, error_parser))
else:
- conn.sudo('R -e \'devtools::install_version("{0}", version = {1}, repos = "https://cloud.r-project.org", dependencies = NA)\' 2>&1 | '
- 'tee /tmp/tee.tmp; if ! grep -w -E "({2})" /tmp/tee.tmp > /tmp/install_{0}.log; then echo "" > /tmp/install_{0}.log;fi'.format(name, vers, error_parser))
- dep = conn.sudo('grep "(NA.*->". /tmp/tee.tmp | awk \'{print $1}\'').stdout.replace('\n', ' ')
- dep_ver = conn.sudo('grep "(NA.*->". /tmp/tee.tmp | awk \'{print $4}\'').stdout.replace('\n', ' ').replace(')', '').split(' ')
+ conn.sudo('R -e \'devtools::install_version("{0}", version = {1}, repos = "https://cloud.r-project.org", dependencies = NA)\' 2>&1 | tee /tmp/install_{0}.tmp; if ! grep -w -E "({2})" /tmp/install_{0}.tmp > /tmp/install_{0}.log; then echo "" > /tmp/install_{0}.log;fi'.format(name, vers, error_parser))
+ dep = conn.sudo('grep "(NA.*->". /tmp/install_' + name + '.tmp | awk \'{print $1}\'').stdout.replace('\n', ' ')
+ dep_ver = conn.sudo('grep "(NA.*->". /tmp/install_' + name + '.tmp | awk \'{print $4}\'').stdout.replace('\n', ' ').replace(')', '').split(' ')
if dep == '':
dep = []
else:
@@ -587,6 +587,7 @@ def install_r_pkg(requisites):
else:
versions = []
status.append({"group": "r_pkg", "name": name, "version": version, "status": status_msg, "error_message": err, "available_versions": versions, "add_pkgs": dep})
+ conn.sudo('rm /tmp/*{}*'.format(name))
return status
except Exception as err:
for r_pkg in requisites:
@@ -642,7 +643,7 @@ def install_java_pkg(requisites):
conn.sudo('mkdir -p {0}'.format(ivy_cache_dir))
group, artifact, version, override = java_pkg
print("Installing package (override: {3}): {0}:{1}:{2}".format(group, artifact, version, override))
- conn.sudo('''bash -c "{8}; java -jar {0} -settings {1}/{2} -cache {3} -dependency {4} {5} {6} 2>&1 | tee /tmp/tee.tmp; if ! grep -w -E \\"({7})\\" /tmp/tee.tmp > /tmp/install_{5}.log; then echo \\"\\" > /tmp/install_{5}.log;fi" '''.format(ivy_jar, ivy_dir, ivy_settings, ivy_cache_dir, group, artifact, version, error_parser, java_proxy))
+ conn.sudo('''bash -c "{8}; java -jar {0} -settings {1}/{2} -cache {3} -dependency {4} {5} {6} 2>&1 | tee /tmp/install_{5}.tmp; if ! grep -w -E \\"({7})\\" /tmp/install_{5}.tmp > /tmp/install_{5}.log; then echo \\"\\" > /tmp/install_{5}.log;fi" '''.format(ivy_jar, ivy_dir, ivy_settings, ivy_cache_dir, group, artifact, version, error_parser, java_proxy))
err = conn.sudo('cat /tmp/install_{0}.log'.format(artifact)).stdout.replace('"', "'").strip()
conn.sudo('find {0} -name "{1}*.jar" | head -n 1 | rev | cut -f1 -d "/" | rev | \
if ! grep -w -i {1} > /tmp/install_{1}.list; then echo "" > /tmp/install_{1}.list;fi'.format(ivy_cache_dir, artifact))
@@ -653,6 +654,7 @@ def install_java_pkg(requisites):
else:
status.append({"group": "java", "name": "{0}:{1}".format(group, artifact), "status": "installation_error", "error_message": err})
update_spark_jars()
+ conn.sudo('rm -rf /tmp/*{}*'.format(artifact))
return status
except Exception as err:
for java_pkg in requisites:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/03: [DATALAB-2448]: changed how devtools
dependencies are installed
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2448
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 4d77abe6818382909a0b196307d81109f48a5271
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Jun 30 14:17:57 2021 +0300
[DATALAB-2448]: changed how devtools dependencies are installed
---
infrastructure-provisioning/src/general/lib/os/fab.py | 7 +++++++
.../src/general/scripts/os/install_additional_libs.py | 7 -------
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index e1c1392..71e9c66 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -533,6 +533,13 @@ def install_r_pkg(requisites):
status = list()
error_parser = "ERROR:|error:|Cannot|failed|Please run|requires|Error|Skipping|couldn't find"
try:
+ if os.environ['conf_resource'] in ('dataengine-service'):
+ if os.environ['conf_cloud_provider'] in ('aws'):
+ manage_pkg('-y install', 'remote', 'libcurl libcurl-devel')
+ elif os.environ['conf_cloud_provider'] in ('gcp'):
+ # manage_pkg('-y build-dep', 'remote', 'libcurl4-gnutls-dev libxml2-dev')
+ manage_pkg('-y install', 'remote', 'libxml2-dev libcurl4-openssl-dev')
+ conn.sudo('R -e "install.packages(\'devtools\', repos = \'cloud.r-project.org\')"')
for r_pkg in requisites:
name, vers = r_pkg
version = vers
diff --git a/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py b/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py
index cf443c5..1a9bf05 100644
--- a/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/install_additional_libs.py
@@ -104,13 +104,6 @@ if __name__ == "__main__":
or os.environ['application'] in ('rstudio', 'tensor-rstudio'):
try:
print('Installing R packages: {}'.format(pkgs['libraries']['r_pkg']))
- if os.environ['conf_resource'] in ('dataengine-service') :
- if os.environ['conf_cloud_provider'] in ('aws'):
- manage_pkg('-y install', 'remote', 'libcurl libcurl-devel')
- elif os.environ['conf_cloud_provider'] in ('gcp'):
- manage_pkg('-y build-dep', 'remote', 'libcurl4-gnutls-dev libxml2-dev')
- manage_pkg('-y install', 'remote', 'libcurl4-gnutls-dev libgit2-dev libxml2-dev')
- conn.sudo('R -e "install.packages(\'devtools\', repos = \'https://cloud.r-project.org\')"')
status = install_r_pkg(pkgs['libraries']['r_pkg'])
general_status = general_status + status
except KeyError:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org