You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/11/18 16:10:38 UTC
[incubator-datalab] branch DATALAB-2587 created (now f8e71e2)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2587
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
at f8e71e2 [DATALAB-2587]: fixed remote pyspark kernels for zeppelin
This branch includes the following new commits:
new f8e71e2 [DATALAB-2587]: fixed remote pyspark kernels for zeppelin
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/01: [DATALAB-2587]: fixed remote pyspark kernels for zeppelin
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2587
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit f8e71e25eb282ecb469bcee2a6fdfea73788f959
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Nov 18 18:10:25 2021 +0200
[DATALAB-2587]: fixed remote pyspark kernels for zeppelin
---
.../os/zeppelin_dataengine_create_configs.py | 42 +++++++++++-----------
.../templates/os/dataengine_interpreter_spark.json | 15 +++++---
2 files changed, 32 insertions(+), 25 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py b/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py
index 7f424eb..4f31d74 100644
--- a/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/os/zeppelin_dataengine_create_configs.py
@@ -123,27 +123,27 @@ def configure_zeppelin_dataengine_interpreter(cluster_name, cluster_dir, os_user
subprocess.run('sudo systemctl start livy-server-' + str(livy_port), shell=True, check=True)
else:
template_file = "/tmp/{}/dataengine_interpreter.json".format(args.cluster_name)
- p_versions = ["2", "3.8"]
- for p_version in p_versions:
- fr = open(template_file, 'r+')
- text = fr.read()
- text = text.replace('CLUSTERNAME', cluster_name)
- text = text.replace('PYTHONVERSION', p_version)
- text = text.replace('SPARK_HOME', cluster_dir + 'spark/')
- text = text.replace('PYTHONVER_SHORT', p_version[:1])
- text = text.replace('MASTER', str(spark_master))
- tmp_file = "/tmp/dataengine_spark_py" + p_version + "_interpreter.json"
- fw = open(tmp_file, 'w')
- fw.write(text)
- fw.close()
- for _ in range(5):
- try:
- subprocess.run("curl --noproxy localhost -H 'Content-Type: application/json' -X POST -d " +
- "@/tmp/dataengine_spark_py" + p_version +
- "_interpreter.json http://localhost:8080/api/interpreter/setting", shell=True, check=True)
- break
- except:
- subprocess.run('sleep 5', shell=True, check=True)
+ p_version = os.environ['notebook_python_venv_version']
+ fr = open(template_file, 'r+')
+ text = fr.read()
+ text = text.replace('CLUSTERNAME', cluster_name)
+ text = text.replace('PYTHONVERSION', p_version[:3])
+ text = text.replace('PYTHONVER_FULL', p_version)
+ text = text.replace('SPARK_HOME', cluster_dir + 'spark/')
+ text = text.replace('PYTHONVER_SHORT', p_version[:1])
+ text = text.replace('MASTER', str(spark_master))
+ tmp_file = "/tmp/dataengine_spark_py" + p_version + "_interpreter.json"
+ fw = open(tmp_file, 'w')
+ fw.write(text)
+ fw.close()
+ for _ in range(5):
+ try:
+ subprocess.run("curl --noproxy localhost -H 'Content-Type: application/json' -X POST -d " +
+ "@/tmp/dataengine_spark_py" + p_version +
+ "_interpreter.json http://localhost:8080/api/interpreter/setting", shell=True, check=True)
+ break
+ except:
+ subprocess.run('sleep 5', shell=True, check=True)
subprocess.run('touch /home/' + os_user + '/.ensure_dir/dataengine_' + cluster_name + '_interpreter_ensured', shell=True, check=True)
except Exception as err:
print('Error: {0}'.format(err))
diff --git a/infrastructure-provisioning/src/general/templates/os/dataengine_interpreter_spark.json b/infrastructure-provisioning/src/general/templates/os/dataengine_interpreter_spark.json
index f3bcebb..ded07c1 100644
--- a/infrastructure-provisioning/src/general/templates/os/dataengine_interpreter_spark.json
+++ b/infrastructure-provisioning/src/general/templates/os/dataengine_interpreter_spark.json
@@ -1,5 +1,5 @@
{
- "name": "CLUSTERNAME_pyPYTHONVER_SHORT",
+ "name": "CLUSTERNAME",
"group": "spark",
"properties": {
"zeppelin.spark.useHiveContext": {
@@ -58,10 +58,17 @@
"description": "",
"type": "checkbox"
},
- "zeppelin.pyspark.python": {
+ "PYSPARK_PYTHON": {
"envName": "ZEPPELIN_PYSPARK_PYTHON",
- "propertyName": "zeppelin.pyspark.python",
- "value": "pythonPYTHONVERSION",
+ "propertyName": "PYSPARK_PYTHON",
+ "value": "/opt/python/pythonPYTHONVER_FULL/bin/pythonPYTHONVERSION",
+ "description": "",
+ "type": "string"
+ },
+ "PYSPARK_DRIVER_PYTHON": {
+ "envName": "ZEPPELIN_PYSPARK_DRIVER_PYTHON",
+ "propertyName": "PYSPARK_DRIVER_PYTHON",
+ "value": "/opt/python/pythonPYTHONVER_FULL/bin/pythonPYTHONVERSION",
"description": "",
"type": "string"
},
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org