You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/04/28 14:28:40 UTC
[incubator-datalab] branch DATALAB-2091 updated: [DATALAB-2091]:
fixed some emr related errors in aws action lib
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/DATALAB-2091 by this push:
new ef95448 [DATALAB-2091]: fixed some emr related errors in aws action lib
ef95448 is described below
commit ef954482c8c9f3a7d044dca84f4cdca25a64a42a
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Apr 28 17:28:26 2021 +0300
[DATALAB-2091]: fixed some emr related errors in aws action lib
---
infrastructure-provisioning/src/general/lib/aws/actions_lib.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index c5f8db6..e34de9d 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -1579,7 +1579,7 @@ def installing_python(region, bucket, user_name, cluster_name, application='', p
subprocess.run('sudo rm -rf /opt/python/python{}/'.format(python_version), shell=True, check=True)
sys.exit(1)
else:
- subprocess.run(venv_command + ' && sudo -i ' + pip_command + ' install -U pip==9.0.3', shell=True, check=True)
+ print(subprocess.run(venv_command + ' && sudo -i ' + pip_command + ' install -U pip==9.0.3', shell=True, check=True))
subprocess.run(venv_command + ' && sudo -i ' + pip_command + ' install pyzmq==17.0.0', shell=True, check=True)
subprocess.run(venv_command + ' && sudo -i ' + pip_command + ' install ipython ipykernel --no-cache-dir', shell=True, check=True)
subprocess.run(venv_command + ' && sudo -i ' + pip_command + ' install NumPy=={}'.format(numpy_version), shell=True, check=True)
@@ -1718,10 +1718,10 @@ def configure_zeppelin_emr_interpreter(emr_version, cluster_name, region, spark_
python_version = python_version[0:5]
livy_port = ''
livy_path = '/opt/{0}/{1}/livy/'.format(emr_version, cluster_name)
- spark_libs = "/opt/{0}/jars/usr/share/aws/aws-java-sdk/aws-java-sdk-core*.jar /opt/{0}/jars/usr/lib/hadoop" \
- "/hadoop-aws*.jar /opt/" + \
- "{0}/jars/usr/share/aws/aws-java-sdk/aws-java-sdk-s3-*.jar /opt/{0}" + \
- "/jars/usr/lib/hadoop-lzo/lib/hadoop-lzo-*.jar".format(emr_version)
+ spark_libs = "/opt/{0}/jars/usr/share/aws/aws-java-sdk/aws-java-sdk-core*.jar " \
+ "/opt/{0}/jars/usr/lib/hadoop/hadoop-aws*.jar " \
+ "/opt/{0}/jars/usr/share/aws/aws-java-sdk/aws-java-sdk-s3-*.jar " \
+ "/opt/{0}/jars/usr/lib/hadoop-lzo/lib/hadoop-lzo-*.jar".format(emr_version)
# fix due to: Multiple py4j files found under ..../spark/python/lib
# py4j-0.10.7-src.zip still in folder. Versions may varies.
subprocess.run('rm /opt/{0}/{1}/spark/python/lib/py4j-src.zip'.format(emr_version, cluster_name), shell=True, check=True)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org