You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by my...@apache.org on 2020/08/21 06:35:13 UTC
[incubator-dlab] 01/01: [DLAB-1985] - Got rid of previous logic for
allocation of Spark executor memory [DLAB-1979] - Deep Learning creation
fixed
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DLAB-1985
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit e6e7a15798beafcca7599c1d5639a955f5199f06
Author: bodnarmykola <bo...@gmail.com>
AuthorDate: Fri Aug 21 09:34:16 2020 +0300
[DLAB-1985] - Got rid of previous logic for allocation of Spark executor memory
[DLAB-1979] - Deep Learning creation fixed
---
infrastructure-provisioning/src/general/conf/dlab.ini | 4 ++--
infrastructure-provisioning/src/general/lib/aws/actions_lib.py | 4 ++--
infrastructure-provisioning/src/general/lib/azure/actions_lib.py | 4 ++--
infrastructure-provisioning/src/general/lib/gcp/actions_lib.py | 4 ++--
infrastructure-provisioning/src/general/lib/os/fab.py | 2 +-
.../src/general/scripts/aws/dataengine-service_create.py | 2 +-
.../src/general/scripts/aws/dataengine-service_prepare.py | 2 +-
.../src/general/scripts/os/jupyter_install_dataengine_kernels.py | 4 ++--
8 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini b/infrastructure-provisioning/src/general/conf/dlab.ini
index 597b260..ed6ff5c 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -262,7 +262,7 @@ cntk2_version = 2.3.1
### CNTK library version for DeepLearning notebook for python3
cntk_version = 2.7
### MXNet library version for DeepLearning notebook for python
-mxnet_version = 1.6.0
+mxnet_version = 1.6.0.post0
### Keras library version for Tensor/DeepLearning notebook
keras_version = 2.1.6
### Theano library version for Tensor/DeepLearning notebook
@@ -278,7 +278,7 @@ cudnn_file_name = cudnn-10.1-linux-x64-v7.6.5.32.tgz
### R enabled on Jupyter/Zeppelin notebook
r_enabled = true
### Temporary fixed python libraries due to dependencies
-tornado_version = 4.5.3
+tornado_version = 5.1.1
ipykernel_version = 4.8.2
### Version of ungit if previous needed. Use latest as default.
ungit_version = 1.4.36
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index 8b9ffab..6437a93 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -1649,8 +1649,8 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
'/tmp/notebook_spark-defaults_local.conf')
if not exists('/opt/spark/conf/spark-env.sh'):
sudo('mv /opt/spark/conf/spark-env.sh.template /opt/spark/conf/spark-env.sh')
- java_home = run("update-alternatives --query java | grep -o \'/.*/java-8.*/jre\'").splitlines()[0]
- sudo("echo 'export JAVA_HOME=\'{}\'' >> /opt/spark/conf/spark-env.sh".format(java_home))
+ java_home = run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").splitlines()[0]
+ sudo("echo 'export JAVA_HOME=\'{}\'' >> /opt/spark/conf/spark-env.sh".format(java_home))
if os.environ['application'] == 'zeppelin':
sudo('echo \"spark.jars $(ls -1 ' + jars_dir + '* | tr \'\\n\' \',\')\" >> '
'/tmp/notebook_spark-defaults_local.conf')
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index 761be9c..4f2057a 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -1128,8 +1128,8 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
spark_memory))
if not exists('/opt/spark/conf/spark-env.sh'):
sudo('mv /opt/spark/conf/spark-env.sh.template /opt/spark/conf/spark-env.sh')
- java_home = run("update-alternatives --query java | grep -o \'/.*/java-8.*/jre\'").splitlines()[0]
- sudo("echo 'export JAVA_HOME=\'{}\'' >> /opt/spark/conf/spark-env.sh".format(java_home))
+ java_home = run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").splitlines()[0]
+ sudo("echo 'export JAVA_HOME=\'{}\'' >> /opt/spark/conf/spark-env.sh".format(java_home))
if 'spark_configurations' in os.environ:
dlab_header = sudo('cat /tmp/notebook_spark-defaults_local.conf | grep "^#"')
spark_configurations = ast.literal_eval(os.environ['spark_configurations'])
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index e550d9d..50d0c4e 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -1410,8 +1410,8 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
spark_memory))
if not exists('/opt/spark/conf/spark-env.sh'):
sudo('mv /opt/spark/conf/spark-env.sh.template /opt/spark/conf/spark-env.sh')
- java_home = run("update-alternatives --query java | grep -o \'/.*/java-8.*/jre\'").splitlines()[0]
- sudo("echo 'export JAVA_HOME=\'{}\'' >> /opt/spark/conf/spark-env.sh".format(java_home))
+ java_home = run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").splitlines()[0]
+ sudo("echo 'export JAVA_HOME=\'{}\'' >> /opt/spark/conf/spark-env.sh".format(java_home))
if 'spark_configurations' in os.environ:
dlab_header = sudo('cat /tmp/notebook_spark-defaults_local.conf | grep "^#"')
spark_configurations = ast.literal_eval(os.environ['spark_configurations'])
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index df8cbe1..7563096 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -215,7 +215,7 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
# http_proxy))
#sudo('sed -i \'/\[Service\]/ a\Environment=\"HTTPS_PROXY={}\"\' /tmp/jupyter-notebook.service'.format(
# https_proxy))
- java_home = run("update-alternatives --query java | grep -o \'/.*/java-8.*/jre\'").splitlines()[0]
+ java_home = run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").splitlines()[0]
sudo('sed -i \'/\[Service\]/ a\Environment=\"JAVA_HOME={}\"\' /tmp/jupyter-notebook.service'.format(
java_home))
sudo('\cp /tmp/jupyter-notebook.service /etc/systemd/system/jupyter-notebook.service')
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py
index 6017968..0cdba1f 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py
@@ -40,7 +40,7 @@ parser = argparse.ArgumentParser()
parser.add_argument('--dry_run', action='store_true', help='Print all variables')
parser.add_argument('--name', type=str, default='', help='Name to be applied to Cluster ( MANDATORY !!! )')
parser.add_argument('--applications', type=str, default='',
- help='Set of applications to be installed on EMR (Default are: "Hadoop Hive Hue Spark")')
+ help='Set of applications to be installed on EMR (Default are: "Hadoop Hive Hue Spark Livy")')
parser.add_argument('--master_instance_type', type=str, default='', help='EC2 instance size for Master-Node (Default: m3.xlarge)')
parser.add_argument('--slave_instance_type', type=str, default='', help='EC2 instance size for Worker-Nodes (Default: m3.xlarge)')
parser.add_argument('--instance_count', type=int, default='',
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
index 7dd94d9..3a08ba4 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
@@ -78,7 +78,7 @@ if __name__ == "__main__":
emr_conf['computational_name'] = os.environ['computational_name']
else:
emr_conf['computational_name'] = ''
- emr_conf['apps'] = 'Hadoop Hive Hue Spark'
+ emr_conf['apps'] = 'Hadoop Hive Hue Spark Livy'
emr_conf['tag_name'] = '{0}-tag'.format(emr_conf['service_base_name'])
emr_conf['key_name'] = os.environ['conf_key_name']
emr_conf['endpoint_tag'] = emr_conf['endpoint_name']
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py
index 01afe7b..33e29a6 100644
--- a/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py
@@ -55,8 +55,8 @@ def configure_notebook(keyfile, hoststring):
#put(templates_dir + 'run_template.sh', '/tmp/{}/run_template.sh'.format(args.cluster_name))
put(templates_dir + 'notebook_spark-defaults_local.conf', '/tmp/{}/notebook_spark-defaults_local.conf'.format(args.cluster_name))
spark_master_ip = args.spark_master.split('//')[1].split(':')[0]
- spark_memory = get_spark_memory(True, args.os_user, spark_master_ip, keyfile)
- run('echo "spark.executor.memory {0}m" >> /tmp/{1}/notebook_spark-defaults_local.conf'.format(spark_memory, args.cluster_name))
+ #spark_memory = get_spark_memory(True, args.os_user, spark_master_ip, keyfile)
+ #run('echo "spark.executor.memory {0}m" >> /tmp/{1}/notebook_spark-defaults_local.conf'.format(spark_memory, args.cluster_name))
if not exists('/usr/local/bin/jupyter_dataengine_create_configs.py'):
put(scripts_dir + 'jupyter_dataengine_create_configs.py', '/usr/local/bin/jupyter_dataengine_create_configs.py', use_sudo=True)
sudo('chmod 755 /usr/local/bin/jupyter_dataengine_create_configs.py')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org