You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/11/19 10:07:41 UTC
[incubator-datalab] 02/02: [DATALAB-2587]: added host ip to spark zeppelin configuration
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2587
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit cc5c80ba8239d51f1f598fcc8b4009fe39d17d6e
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Fri Nov 19 12:07:15 2021 +0200
[DATALAB-2587]: added host ip to spark zeppelin configuration
---
.../src/general/scripts/os/zeppelin_dataengine_create_configs.py | 5 +++--
.../src/general/templates/os/dataengine_interpreter_spark.json | 7 +++++++
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py
index 1f2abcc..8f1a98b 100644
--- a/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py
@@ -56,7 +56,7 @@ spark_link = "https://archive.apache.org/dist/spark/spark-" + spark_version + "/
"-bin-hadoop" + hadoop_version + ".tgz"
-def configure_zeppelin_dataengine_interpreter(cluster_name, cluster_dir, os_user, multiple_clusters, spark_master, python_version):
+def configure_zeppelin_dataengine_interpreter(cluster_name, cluster_dir, os_user, multiple_clusters, spark_master, python_version, notebook_ip):
try:
port_number_found = False
zeppelin_restarted = False
@@ -127,6 +127,7 @@ def configure_zeppelin_dataengine_interpreter(cluster_name, cluster_dir, os_user
fr = open(template_file, 'r+')
text = fr.read()
text = text.replace('CLUSTERNAME', cluster_name)
+ text = text.replace('HOST_IP', notebook_ip)
text = text.replace('PYTHONVERSION', python_version[:3])
text = text.replace('PYTHONVER_FULL', python_version)
text = text.replace('SPARK_HOME', cluster_dir + 'spark/')
@@ -175,5 +176,5 @@ if __name__ == "__main__":
if args.multiple_clusters == 'true':
install_remote_livy(args)
configure_zeppelin_dataengine_interpreter(args.cluster_name, cluster_dir, args.os_user,
- args.multiple_clusters, args.spark_master, args.python_version)
+ args.multiple_clusters, args.spark_master, args.python_version, args.notebook_ip)
update_zeppelin_interpreters(args.multiple_clusters, args.r_enabled)
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/templates/os/dataengine_interpreter_spark.json b/infrastructure-provisioning/src/general/templates/os/dataengine_interpreter_spark.json
index ded07c1..eaa85e7 100644
--- a/infrastructure-provisioning/src/general/templates/os/dataengine_interpreter_spark.json
+++ b/infrastructure-provisioning/src/general/templates/os/dataengine_interpreter_spark.json
@@ -37,6 +37,13 @@
"description": "Spark master uri. ex) spark://masterhost:7077",
"type": "string"
},
+ "spark.driver.host":{
+ "envName": "spark.driver.host",
+ "propertyName": "spark.driver.host",
+ "value": "HOST_IP",
+ "description": "driver host ip (zeppelin)",
+ "type": "string"
+ },
"zeppelin.spark.concurrentSQL": {
"envName": "ZEPPELIN_SPARK_CONCURRENTSQL",
"propertyName": "zeppelin.spark.concurrentSQL",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org