You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by my...@apache.org on 2020/08/21 06:35:12 UTC
[incubator-dlab] branch DLAB-1985 created (now e6e7a15)
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a change to branch DLAB-1985
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
at e6e7a15 [DLAB-1985] - Got rid of previous logic for allocation of Spark executor memory [DLAB-1979] - Deep Learning creation fixed
This branch includes the following new commits:
new e6e7a15 [DLAB-1985] - Got rid of previous logic for allocation of Spark executor memory [DLAB-1979] - Deep Learning creation fixed
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/01: [DLAB-1985] - Got rid of previous logic for
allocation of Spark executor memory [DLAB-1979] - Deep Learning creation
fixed
Posted by my...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DLAB-1985
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit e6e7a15798beafcca7599c1d5639a955f5199f06
Author: bodnarmykola <bo...@gmail.com>
AuthorDate: Fri Aug 21 09:34:16 2020 +0300
[DLAB-1985] - Got rid of previous logic for allocation of Spark executor memory
[DLAB-1979] - Deep Learning creation fixed
---
infrastructure-provisioning/src/general/conf/dlab.ini | 4 ++--
infrastructure-provisioning/src/general/lib/aws/actions_lib.py | 4 ++--
infrastructure-provisioning/src/general/lib/azure/actions_lib.py | 4 ++--
infrastructure-provisioning/src/general/lib/gcp/actions_lib.py | 4 ++--
infrastructure-provisioning/src/general/lib/os/fab.py | 2 +-
.../src/general/scripts/aws/dataengine-service_create.py | 2 +-
.../src/general/scripts/aws/dataengine-service_prepare.py | 2 +-
.../src/general/scripts/os/jupyter_install_dataengine_kernels.py | 4 ++--
8 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini b/infrastructure-provisioning/src/general/conf/dlab.ini
index 597b260..ed6ff5c 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -262,7 +262,7 @@ cntk2_version = 2.3.1
### CNTK library version for DeepLearning notebook for python3
cntk_version = 2.7
### MXNet library version for DeepLearning notebook for python
-mxnet_version = 1.6.0
+mxnet_version = 1.6.0.post0
### Keras library version for Tensor/DeepLearning notebook
keras_version = 2.1.6
### Theano library version for Tensor/DeepLearning notebook
@@ -278,7 +278,7 @@ cudnn_file_name = cudnn-10.1-linux-x64-v7.6.5.32.tgz
### R enabled on Jupyter/Zeppelin notebook
r_enabled = true
### Temporary fixed python libraries due to dependencies
-tornado_version = 4.5.3
+tornado_version = 5.1.1
ipykernel_version = 4.8.2
### Version of ungit if previous needed. Use latest as default.
ungit_version = 1.4.36
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index 8b9ffab..6437a93 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -1649,8 +1649,8 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
'/tmp/notebook_spark-defaults_local.conf')
if not exists('/opt/spark/conf/spark-env.sh'):
sudo('mv /opt/spark/conf/spark-env.sh.template /opt/spark/conf/spark-env.sh')
- java_home = run("update-alternatives --query java | grep -o \'/.*/java-8.*/jre\'").splitlines()[0]
- sudo("echo 'export JAVA_HOME=\'{}\'' >> /opt/spark/conf/spark-env.sh".format(java_home))
+ java_home = run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").splitlines()[0]
+ sudo("echo 'export JAVA_HOME=\'{}\'' >> /opt/spark/conf/spark-env.sh".format(java_home))
if os.environ['application'] == 'zeppelin':
sudo('echo \"spark.jars $(ls -1 ' + jars_dir + '* | tr \'\\n\' \',\')\" >> '
'/tmp/notebook_spark-defaults_local.conf')
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index 761be9c..4f2057a 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -1128,8 +1128,8 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
spark_memory))
if not exists('/opt/spark/conf/spark-env.sh'):
sudo('mv /opt/spark/conf/spark-env.sh.template /opt/spark/conf/spark-env.sh')
- java_home = run("update-alternatives --query java | grep -o \'/.*/java-8.*/jre\'").splitlines()[0]
- sudo("echo 'export JAVA_HOME=\'{}\'' >> /opt/spark/conf/spark-env.sh".format(java_home))
+ java_home = run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").splitlines()[0]
+ sudo("echo 'export JAVA_HOME=\'{}\'' >> /opt/spark/conf/spark-env.sh".format(java_home))
if 'spark_configurations' in os.environ:
dlab_header = sudo('cat /tmp/notebook_spark-defaults_local.conf | grep "^#"')
spark_configurations = ast.literal_eval(os.environ['spark_configurations'])
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index e550d9d..50d0c4e 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -1410,8 +1410,8 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
spark_memory))
if not exists('/opt/spark/conf/spark-env.sh'):
sudo('mv /opt/spark/conf/spark-env.sh.template /opt/spark/conf/spark-env.sh')
- java_home = run("update-alternatives --query java | grep -o \'/.*/java-8.*/jre\'").splitlines()[0]
- sudo("echo 'export JAVA_HOME=\'{}\'' >> /opt/spark/conf/spark-env.sh".format(java_home))
+ java_home = run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").splitlines()[0]
+ sudo("echo 'export JAVA_HOME=\'{}\'' >> /opt/spark/conf/spark-env.sh".format(java_home))
if 'spark_configurations' in os.environ:
dlab_header = sudo('cat /tmp/notebook_spark-defaults_local.conf | grep "^#"')
spark_configurations = ast.literal_eval(os.environ['spark_configurations'])
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index df8cbe1..7563096 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -215,7 +215,7 @@ def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version
# http_proxy))
#sudo('sed -i \'/\[Service\]/ a\Environment=\"HTTPS_PROXY={}\"\' /tmp/jupyter-notebook.service'.format(
# https_proxy))
- java_home = run("update-alternatives --query java | grep -o \'/.*/java-8.*/jre\'").splitlines()[0]
+ java_home = run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").splitlines()[0]
sudo('sed -i \'/\[Service\]/ a\Environment=\"JAVA_HOME={}\"\' /tmp/jupyter-notebook.service'.format(
java_home))
sudo('\cp /tmp/jupyter-notebook.service /etc/systemd/system/jupyter-notebook.service')
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py
index 6017968..0cdba1f 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py
@@ -40,7 +40,7 @@ parser = argparse.ArgumentParser()
parser.add_argument('--dry_run', action='store_true', help='Print all variables')
parser.add_argument('--name', type=str, default='', help='Name to be applied to Cluster ( MANDATORY !!! )')
parser.add_argument('--applications', type=str, default='',
- help='Set of applications to be installed on EMR (Default are: "Hadoop Hive Hue Spark")')
+ help='Set of applications to be installed on EMR (Default are: "Hadoop Hive Hue Spark Livy")')
parser.add_argument('--master_instance_type', type=str, default='', help='EC2 instance size for Master-Node (Default: m3.xlarge)')
parser.add_argument('--slave_instance_type', type=str, default='', help='EC2 instance size for Worker-Nodes (Default: m3.xlarge)')
parser.add_argument('--instance_count', type=int, default='',
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
index 7dd94d9..3a08ba4 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
@@ -78,7 +78,7 @@ if __name__ == "__main__":
emr_conf['computational_name'] = os.environ['computational_name']
else:
emr_conf['computational_name'] = ''
- emr_conf['apps'] = 'Hadoop Hive Hue Spark'
+ emr_conf['apps'] = 'Hadoop Hive Hue Spark Livy'
emr_conf['tag_name'] = '{0}-tag'.format(emr_conf['service_base_name'])
emr_conf['key_name'] = os.environ['conf_key_name']
emr_conf['endpoint_tag'] = emr_conf['endpoint_name']
diff --git a/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py b/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py
index 01afe7b..33e29a6 100644
--- a/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/os/jupyter_install_dataengine_kernels.py
@@ -55,8 +55,8 @@ def configure_notebook(keyfile, hoststring):
#put(templates_dir + 'run_template.sh', '/tmp/{}/run_template.sh'.format(args.cluster_name))
put(templates_dir + 'notebook_spark-defaults_local.conf', '/tmp/{}/notebook_spark-defaults_local.conf'.format(args.cluster_name))
spark_master_ip = args.spark_master.split('//')[1].split(':')[0]
- spark_memory = get_spark_memory(True, args.os_user, spark_master_ip, keyfile)
- run('echo "spark.executor.memory {0}m" >> /tmp/{1}/notebook_spark-defaults_local.conf'.format(spark_memory, args.cluster_name))
+ #spark_memory = get_spark_memory(True, args.os_user, spark_master_ip, keyfile)
+ #run('echo "spark.executor.memory {0}m" >> /tmp/{1}/notebook_spark-defaults_local.conf'.format(spark_memory, args.cluster_name))
if not exists('/usr/local/bin/jupyter_dataengine_create_configs.py'):
put(scripts_dir + 'jupyter_dataengine_create_configs.py', '/usr/local/bin/jupyter_dataengine_create_configs.py', use_sudo=True)
sudo('chmod 755 /usr/local/bin/jupyter_dataengine_create_configs.py')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org