You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/11/19 10:07:39 UTC
[incubator-datalab] branch DATALAB-2587 updated (3af65bd -> cc5c80b)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2587
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
from 3af65bd [DATALAB-2587]: fixed nod defined variable error
new 443db13 [DATALAB-2587]: decreased spark memory
new cc5c80b [DATALAB-2587]: added host ip to spark zeppelin configuration
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
infrastructure-provisioning/src/general/lib/os/fab.py | 2 +-
.../src/general/scripts/os/zeppelin_dataengine_create_configs.py | 5 +++--
.../src/general/templates/os/dataengine_interpreter_spark.json | 7 +++++++
3 files changed, 11 insertions(+), 3 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 02/02: [DATALAB-2587]: added host ip to spark zeppelin configuration
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2587
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit cc5c80ba8239d51f1f598fcc8b4009fe39d17d6e
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Fri Nov 19 12:07:15 2021 +0200
[DATALAB-2587]: added host ip to spark zeppelin configuration
---
.../src/general/scripts/os/zeppelin_dataengine_create_configs.py | 5 +++--
.../src/general/templates/os/dataengine_interpreter_spark.json | 7 +++++++
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py
index 1f2abcc..8f1a98b 100644
--- a/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py
@@ -56,7 +56,7 @@ spark_link = "https://archive.apache.org/dist/spark/spark-" + spark_version + "/
"-bin-hadoop" + hadoop_version + ".tgz"
-def configure_zeppelin_dataengine_interpreter(cluster_name, cluster_dir, os_user, multiple_clusters, spark_master, python_version):
+def configure_zeppelin_dataengine_interpreter(cluster_name, cluster_dir, os_user, multiple_clusters, spark_master, python_version, notebook_ip):
try:
port_number_found = False
zeppelin_restarted = False
@@ -127,6 +127,7 @@ def configure_zeppelin_dataengine_interpreter(cluster_name, cluster_dir, os_user
fr = open(template_file, 'r+')
text = fr.read()
text = text.replace('CLUSTERNAME', cluster_name)
+ text = text.replace('HOST_IP', notebook_ip)
text = text.replace('PYTHONVERSION', python_version[:3])
text = text.replace('PYTHONVER_FULL', python_version)
text = text.replace('SPARK_HOME', cluster_dir + 'spark/')
@@ -175,5 +176,5 @@ if __name__ == "__main__":
if args.multiple_clusters == 'true':
install_remote_livy(args)
configure_zeppelin_dataengine_interpreter(args.cluster_name, cluster_dir, args.os_user,
- args.multiple_clusters, args.spark_master, args.python_version)
+ args.multiple_clusters, args.spark_master, args.python_version, args.notebook_ip)
update_zeppelin_interpreters(args.multiple_clusters, args.r_enabled)
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/templates/os/dataengine_interpreter_spark.json b/infrastructure-provisioning/src/general/templates/os/dataengine_interpreter_spark.json
index ded07c1..eaa85e7 100644
--- a/infrastructure-provisioning/src/general/templates/os/dataengine_interpreter_spark.json
+++ b/infrastructure-provisioning/src/general/templates/os/dataengine_interpreter_spark.json
@@ -37,6 +37,13 @@
"description": "Spark master uri. ex) spark://masterhost:7077",
"type": "string"
},
+ "spark.driver.host":{
+ "envName": "spark.driver.host",
+ "propertyName": "spark.driver.host",
+ "value": "HOST_IP",
+ "description": "driver host ip (zeppelin)",
+ "type": "string"
+ },
"zeppelin.spark.concurrentSQL": {
"envName": "ZEPPELIN_SPARK_CONCURRENTSQL",
"propertyName": "zeppelin.spark.concurrentSQL",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/02: [DATALAB-2587]: decreased spark memory
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2587
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 443db130bf38a709995ef7493b371f67deac32d4
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Fri Nov 19 12:02:28 2021 +0200
[DATALAB-2587]: decreased spark memory
---
infrastructure-provisioning/src/general/lib/os/fab.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 722fb98..42318e2 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -788,7 +788,7 @@ def get_spark_memory(creds=False, os_user='', hostname='', keyfile=''):
else:
mem = conn.sudo('free -m | grep Mem | tr -s " " ":" | cut -f 2 -d ":"').stdout.replace('\n', '')
instance_memory = int(mem)
- spark_memory = round(instance_memory * 90 / 100)
+ spark_memory = round(instance_memory * 85 / 100)
return spark_memory
except Exception as err:
logging.error('Function install_inactivity_checker error:', str(err))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org