You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by my...@apache.org on 2019/12/18 13:51:05 UTC
[incubator-dlab] branch epm-v2.2-rc1-versions created (now 27cda4c)
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a change to branch epm-v2.2-rc1-versions
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
at 27cda4c [DLAB-1346] - EMR version updated [DLAB-1348] - Spark version updated [DLAB-1350] - Jupyter version updated
This branch includes the following new commits:
new 27cda4c [DLAB-1346] - EMR version updated [DLAB-1348] - Spark version updated [DLAB-1350] - Jupyter version updated
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/01: [DLAB-1346] - EMR version updated
[DLAB-1348] - Spark version updated [DLAB-1350] - Jupyter version updated
Posted by my...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch epm-v2.2-rc1-versions
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 27cda4cd838cc3b80efdf0abd01212bebc29d81d
Author: Mykola_Bodnar1 <bo...@gmail.com>
AuthorDate: Wed Dec 18 15:50:30 2019 +0200
[DLAB-1346] - EMR version updated
[DLAB-1348] - Spark version updated
[DLAB-1350] - Jupyter version updated
---
infrastructure-provisioning/src/general/conf/dlab.ini | 4 ++--
.../general/files/aws/dataengine-service_description.json | 4 ++--
infrastructure-provisioning/src/general/lib/os/fab.py | 4 +++-
.../general/scripts/aws/dataengine-service_jars_parser.py | 14 ++++++--------
4 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini b/infrastructure-provisioning/src/general/conf/dlab.ini
index 7873b43..ede348d 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -216,11 +216,11 @@ docker_version = 17.09.0
### Size of the additional volume for notebook instance
disk_size = 30
### Version of Apache Spark to be installed on notebook
-spark_version = 2.3.2
+spark_version = 2.4.4
### Version of Apache Hadoop to be installed on notebook
hadoop_version = 2.7
### Version of Jupyter to be installed on notebook
-jupyter_version = 5.7.4
+jupyter_version = 6.0.2
### Version of TensorFlow to be installed on notebook
tensorflow_version = 1.8.0
### Version of Zeppelin to be installed on notebook
diff --git a/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json b/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json
index 7caee2a..b4e5ba2 100644
--- a/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json
+++ b/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json
@@ -24,7 +24,7 @@
},
"templates":
[
- {"version":"emr-5.12.0", "applications": [{"Name":"Hadoop", "Version": "2.8.3"}, {"Name":"Spark", "Version": "2.2.1"}, {"Name":"Hive", "Version": "2.3.2"}]},
- {"version":"emr-5.19.0", "applications": [{"Name":"Hadoop", "Version": "2.8.5"}, {"Name":"Spark", "Version": "2.3.2"}, {"Name":"Hive", "Version": "2.3.3"}]}
+ {"version":"emr-5.19.0", "applications": [{"Name":"Hadoop", "Version": "2.8.5"}, {"Name":"Spark", "Version": "2.3.2"}, {"Name":"Hive", "Version": "2.3.3"}]},
+ {"version":"emr-5.28.0", "applications": [{"Name":"Hadoop", "Version": "2.8.5"}, {"Name":"Spark", "Version": "2.4.4"}, {"Name":"Hive", "Version": "2.3.6"}]}
]
}
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index aba4c6a..7a07130 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -146,7 +146,7 @@ def put_resource_status(resource, status, dlab_path, os_user, hostname):
def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version, exploratory_name):
if not exists('/home/' + os_user + '/.ensure_dir/jupyter_ensured'):
try:
- sudo('pip2 install notebook=={} --no-cache-dir'.format(jupyter_version))
+ sudo('pip2 install notebook==5.7.8 --no-cache-dir')
sudo('pip2 install jupyter --no-cache-dir')
sudo('pip3.5 install notebook=={} --no-cache-dir'.format(jupyter_version))
sudo('pip3.5 install jupyter --no-cache-dir')
@@ -636,6 +636,8 @@ def configure_data_engine_service_pip(hostname, os_user, keyfile):
sudo('ln -s /usr/bin/pip-3.4 /usr/bin/pip3')
elif not exists('/usr/bin/pip3') and sudo("python3.5 -V 2>/dev/null | awk '{print $2}'"):
sudo('ln -s /usr/bin/pip-3.5 /usr/bin/pip3')
+ elif not exists('/usr/bin/pip3') and sudo("python3.6 -V 2>/dev/null | awk '{print $2}'"):
+ sudo('ln -s /usr/bin/pip-3.6 /usr/bin/pip3')
sudo('echo "export PATH=$PATH:/usr/local/bin" >> /etc/profile')
sudo('source /etc/profile')
run('source /etc/profile')
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py
index f011e6a..34755b5 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py
@@ -54,14 +54,12 @@ if __name__ == "__main__":
else:
endpoint = "https://s3-{}.amazonaws.com".format(args.region)
os.system('touch /tmp/python_version')
- python_ver = subprocess.check_output("python3.5 -V 2>/dev/null | awk '{print $2}'", shell=True)
- if python_ver != '':
- with open('/tmp/python_version', 'w') as outfile:
- outfile.write(python_ver)
- else:
- python_ver = subprocess.check_output("python3.4 -V 2>/dev/null | awk '{print $2}'", shell=True)
- with open('/tmp/python_version', 'w') as outfile:
- outfile.write(python_ver)
+ for v in range(4, 7):
+ python_ver_checker = "python3.{} -V 2>/dev/null".format(v) + " | awk '{print $2}'"
+ python_ver = subprocess.check_output(python_ver_checker, shell=True)
+ if python_ver != '':
+ with open('/tmp/python_version', 'w') as outfile:
+ outfile.write(python_ver)
os.system('/bin/tar -zhcvf /tmp/jars.tar.gz '
'--no-recursion '
'--absolute-names '
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org