You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/07/02 09:40:06 UTC
[incubator-datalab] 02/03: [DATALAB-2465]: fixed permission denied
to tmp file during lib install
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2448
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 638ed5e695e038b28077a9c653582fb13a1cbbd9
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Jul 1 12:36:22 2021 +0300
[DATALAB-2465]: fixed permission denied to tmp file during lib install
---
.../src/general/lib/os/debian/notebook_lib.py | 12 ++++--
.../src/general/lib/os/fab.py | 48 +++++++++++-----------
2 files changed, 33 insertions(+), 27 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index df11ceb..94c86a1 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@ -420,11 +420,14 @@ def install_os_pkg(requisites):
else:
version = 'N/A'
os_pkg = os_pkg[0]
- datalab.fab.conn.sudo('DEBIAN_FRONTEND=noninteractive apt-get -y install --allow-downgrades {0} 2>&1 | tee /tmp/tee.tmp; if ! grep -w -E "({1})" /tmp/tee.tmp > '
- '/tmp/os_install_{2}.log; then echo "no_error" > /tmp/os_install_{2}.log;fi'.format(os_pkg, error_parser, name))
+ datalab.fab.conn.sudo('DEBIAN_FRONTEND=noninteractive apt-get -y install --allow-downgrades {0} 2>&1 | '
+ 'tee /tmp/os_install_{2}.tmp; if ! grep -w -E "({1})" /tmp/os_install_{2}.tmp > '
+ '/tmp/os_install_{2}.log; then echo "no_error" > /tmp/os_install_{2}.log;fi'
+ .format(os_pkg, error_parser, name))
err = datalab.fab.conn.sudo('cat /tmp/os_install_{}.log'.format(name)).stdout.replace('"', "'")
- datalab.fab.conn.sudo('cat /tmp/tee.tmp | if ! grep -w -E -A 30 "({1})" /tmp/tee.tmp > '
- '/tmp/os_install_{0}.log; then echo "no_new_pkgs" > /tmp/os_install_{0}.log;fi'.format(name, new_pkgs_parser))
+ datalab.fab.conn.sudo('cat /tmp/os_install_{0}.tmp | if ! grep -w -E -A 30 "({1})" /tmp/os_install_{0}.tmp > '
+ '/tmp/os_install_{0}.log; then echo "no_new_pkgs" > /tmp/os_install_{0}.log;fi'
+ .format(name, new_pkgs_parser))
dep = datalab.fab.conn.sudo('cat /tmp/os_install_{}.log'.format(name)).stdout
if 'no_new_pkgs' in dep:
dep = []
@@ -460,6 +463,7 @@ def install_os_pkg(requisites):
"error_message": err, "add_pkgs": dep, "available_versions": versions})
datalab.fab.conn.sudo('unattended-upgrades -v')
#datalab.fab.conn.sudo('export LC_ALL=C')
+ datalab.fab.conn.sudo('rm /tmp/*{}*'.format(name))
return status
except Exception as err:
for os_pkg in requisites:
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 71e9c66..9829cc4 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -127,31 +127,32 @@ def install_pip_pkg(requisites, pip_version, lib_group, dataengine_service = Fal
pip_pkg = "{}=={}".format(pip_pkg[0], pip_pkg[1])
conn.sudo(
'''bash -l -c '{0} install -U {1} --use-deprecated=legacy-resolver --no-cache-dir 2>&1 | '''
- '''tee /tmp/tee.tmp; if ! grep -w -i -E "({2})" /tmp/tee.tmp > /tmp/{4}install_{3}.log; '''
- '''then echo "" > /tmp/{4}install_{3}.log;fi' '''.format(
+ '''tee /tmp/{4}_install_{3}.tmp; if ! grep -w -i -E "({2})" /tmp/{4}_install_{3}.tmp > /tmp/{4}_install_{3}.log; '''
+ '''then echo "" > /tmp/{4}_install_{3}.log;fi' '''.format(
install_command, pip_pkg, error_parser, name, pip_version))
- err = conn.sudo('cat /tmp/{0}install_{1}.log'.format(pip_version, pip_pkg.split("==")[0])).stdout.replace(
+ err = conn.sudo('cat /tmp/{0}_install_{1}.log'.format(pip_version, pip_pkg.split("==")[0])).stdout.replace(
'"', "'").replace('\n', ' ')
conn.sudo(
- '''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}install_{1}.list; '''
- '''then echo "not_found" > /tmp/{2}install_{1}.list;fi' '''.format(
+ '''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}_install_{1}.list; '''
+ '''then echo "not_found" > /tmp/{2}_install_{1}.list;fi' '''.format(
install_command, name, pip_version))
- res = conn.sudo('''bash -l -c 'cat /tmp/{0}install_{1}.list' '''.format(pip_version, name)).stdout.replace(
+ res = conn.sudo('''bash -l -c 'cat /tmp/{0}_install_{1}.list' '''.format(pip_version, name)).stdout.replace(
'\n', '')
conn.sudo(
- '''bash -l -c 'cat /tmp/tee.tmp | if ! grep -w -i -E "(Successfully installed|up-to-date)" > '''
- '''/tmp/{0}install_{1}.list; then echo "not_installed" > /tmp/{0}install_{1}.list;fi' '''.format(
+ '''bash -l -c 'cat /tmp/{0}_install_{1}.tmp | if ! grep -w -i -E "(Successfully installed|up-to-date)" > '''
+ '''/tmp/{0}_install_{1}.list; then echo "not_installed" > /tmp/{0}_install_{1}.list;fi' '''.format(
pip_version, name))
installed_out = conn.sudo(
- '''bash -l -c 'cat /tmp/{0}install_{1}.list' '''.format(pip_version, name)).stdout.replace('\n', '')
+ '''bash -l -c 'cat /tmp/{0}_install_{1}.list' '''.format(pip_version, name)).stdout.replace('\n', '')
changed_pip_pkg = False
if 'not_found' in res:
changed_pip_pkg = pip_pkg.split("==")[0].replace("_", "-").split('-')
changed_pip_pkg = changed_pip_pkg[0]
conn.sudo(
- '''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}install_{1}.list; then echo "" > /tmp/{2}install_{1}.list;fi' '''.format(
+ '''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}_install_{1}.list; then echo "" > '''
+ '''/tmp/{2}_install_{1}.list;fi' '''.format(
install_command, changed_pip_pkg, pip_version))
- res = conn.sudo('cat /tmp/{0}install_{1}.list'.format(pip_version, changed_pip_pkg)).stdout.replace(
+ res = conn.sudo('cat /tmp/{0}_install_{1}.list'.format(pip_version, changed_pip_pkg)).stdout.replace(
'\n', '')
if err and name not in installed_out:
status_msg = 'installation_error'
@@ -178,11 +179,9 @@ def install_pip_pkg(requisites, pip_version, lib_group, dataengine_service = Fal
else:
versions = []
- conn.sudo(
- '''bash -l -c 'cat /tmp/tee.tmp | if ! grep -w -i -E "Installing collected packages:" > '''
- '''/tmp/{0}install_{1}.dep; then echo "" > /tmp/{0}install_{1}.dep;fi' '''.format(
- pip_version, name))
- dep = conn.sudo('cat /tmp/{0}install_{1}.dep'.format(pip_version, name)).stdout.replace('\n', '').strip()[31:]
+ conn.sudo('cat /tmp/{0}_install_{1}.tmp | if ! grep -w -i -E "Installing collected packages:" > '
+ '/tmp/{0}_install_{1}.dep; then echo "" > /tmp/{0}_install_{1}.dep;fi'.format(pip_version, name))
+ dep = conn.sudo('cat /tmp/{0}_install_{1}.dep'.format(pip_version, name)).stdout.replace('\n', '').strip()[31:]
if dep == '':
dep = []
else:
@@ -197,6 +196,7 @@ def install_pip_pkg(requisites, pip_version, lib_group, dataengine_service = Fal
dep = [i for i in dep if i]
status.append({"group": lib_group, "name": name, "version": version, "status": status_msg,
"error_message": err, "available_versions": versions, "add_pkgs": dep})
+ conn.sudo('rm -rf /tmp/*{}*'.format(name))
return status
except Exception as err:
for pip_pkg in requisites:
@@ -548,13 +548,13 @@ def install_r_pkg(requisites):
else:
vers = '"{}"'.format(vers)
if name == 'sparklyr':
- conn.run('sudo -i R -e \'devtools::install_version("{0}", version = {1}, repos = "http://cran.us.r-project.org", dependencies = NA)\' 2>&1 | '
- 'tee /tmp/tee.tmp; if ! grep -w -E "({2})" /tmp/tee.tmp > /tmp/install_{0}.log; then echo "" > /tmp/install_{0}.log;fi'.format(name, vers, error_parser))
+ conn.run('sudo -i R -e \'devtools::install_version("{0}", version = {1}, repos = "http://cran.us.r-project.org", '
+ 'dependencies = NA)\' 2>&1 | tee /tmp/install_{0}.tmp; if ! grep -w -E "({2})" /tmp/install_{0}.tmp '
+ '> /tmp/install_{0}.log; then echo "" > /tmp/install_{0}.log;fi'.format(name, vers, error_parser))
else:
- conn.sudo('R -e \'devtools::install_version("{0}", version = {1}, repos = "https://cloud.r-project.org", dependencies = NA)\' 2>&1 | '
- 'tee /tmp/tee.tmp; if ! grep -w -E "({2})" /tmp/tee.tmp > /tmp/install_{0}.log; then echo "" > /tmp/install_{0}.log;fi'.format(name, vers, error_parser))
- dep = conn.sudo('grep "(NA.*->". /tmp/tee.tmp | awk \'{print $1}\'').stdout.replace('\n', ' ')
- dep_ver = conn.sudo('grep "(NA.*->". /tmp/tee.tmp | awk \'{print $4}\'').stdout.replace('\n', ' ').replace(')', '').split(' ')
+ conn.sudo('R -e \'devtools::install_version("{0}", version = {1}, repos = "https://cloud.r-project.org", dependencies = NA)\' 2>&1 | tee /tmp/install_{0}.tmp; if ! grep -w -E "({2})" /tmp/install_{0}.tmp > /tmp/install_{0}.log; then echo "" > /tmp/install_{0}.log;fi'.format(name, vers, error_parser))
+ dep = conn.sudo('grep "(NA.*->". /tmp/install_' + name + '.tmp | awk \'{print $1}\'').stdout.replace('\n', ' ')
+ dep_ver = conn.sudo('grep "(NA.*->". /tmp/install_' + name + '.tmp | awk \'{print $4}\'').stdout.replace('\n', ' ').replace(')', '').split(' ')
if dep == '':
dep = []
else:
@@ -587,6 +587,7 @@ def install_r_pkg(requisites):
else:
versions = []
status.append({"group": "r_pkg", "name": name, "version": version, "status": status_msg, "error_message": err, "available_versions": versions, "add_pkgs": dep})
+ conn.sudo('rm /tmp/*{}*'.format(name))
return status
except Exception as err:
for r_pkg in requisites:
@@ -642,7 +643,7 @@ def install_java_pkg(requisites):
conn.sudo('mkdir -p {0}'.format(ivy_cache_dir))
group, artifact, version, override = java_pkg
print("Installing package (override: {3}): {0}:{1}:{2}".format(group, artifact, version, override))
- conn.sudo('''bash -c "{8}; java -jar {0} -settings {1}/{2} -cache {3} -dependency {4} {5} {6} 2>&1 | tee /tmp/tee.tmp; if ! grep -w -E \\"({7})\\" /tmp/tee.tmp > /tmp/install_{5}.log; then echo \\"\\" > /tmp/install_{5}.log;fi" '''.format(ivy_jar, ivy_dir, ivy_settings, ivy_cache_dir, group, artifact, version, error_parser, java_proxy))
+ conn.sudo('''bash -c "{8}; java -jar {0} -settings {1}/{2} -cache {3} -dependency {4} {5} {6} 2>&1 | tee /tmp/install_{5}.tmp; if ! grep -w -E \\"({7})\\" /tmp/install_{5}.tmp > /tmp/install_{5}.log; then echo \\"\\" > /tmp/install_{5}.log;fi" '''.format(ivy_jar, ivy_dir, ivy_settings, ivy_cache_dir, group, artifact, version, error_parser, java_proxy))
err = conn.sudo('cat /tmp/install_{0}.log'.format(artifact)).stdout.replace('"', "'").strip()
conn.sudo('find {0} -name "{1}*.jar" | head -n 1 | rev | cut -f1 -d "/" | rev | \
if ! grep -w -i {1} > /tmp/install_{1}.list; then echo "" > /tmp/install_{1}.list;fi'.format(ivy_cache_dir, artifact))
@@ -653,6 +654,7 @@ def install_java_pkg(requisites):
else:
status.append({"group": "java", "name": "{0}:{1}".format(group, artifact), "status": "installation_error", "error_message": err})
update_spark_jars()
+ conn.sudo('rm -rf /tmp/*{}*'.format(artifact))
return status
except Exception as err:
for java_pkg in requisites:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org