You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/02/17 14:48:31 UTC
[incubator-datalab] branch develop updated: Deleted python2, fixed python3 and deleted internal python interpreter in zeppelin
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/develop by this push:
new f9e27b1 Deleted python2, fixed python3 and deleted internal python interpreter in zeppelin
new 43ab2d5 Merge pull request #1425 from a1expol/DATALAB-2667
f9e27b1 is described below
commit f9e27b1e7257b655232c4152e6d3676ec18cf907
Author: Oleksandr Polishchuk2 <ol...@epam.com>
AuthorDate: Thu Feb 17 16:42:14 2022 +0200
Deleted python2, fixed python3 and deleted internal python interpreter in zeppelin
---
.../general/templates/aws/interpreter_spark.json | 121 ++-------------------
.../zeppelin/scripts/configure_zeppelin_node.py | 1 +
2 files changed, 8 insertions(+), 114 deletions(-)
diff --git a/infrastructure-provisioning/src/general/templates/aws/interpreter_spark.json b/infrastructure-provisioning/src/general/templates/aws/interpreter_spark.json
index 56eb105..1e503dd 100644
--- a/infrastructure-provisioning/src/general/templates/aws/interpreter_spark.json
+++ b/infrastructure-provisioning/src/general/templates/aws/interpreter_spark.json
@@ -1,119 +1,5 @@
{
"interpreterSettings": {
- "2C6RJRBD1": {
- "id": "2C6RJRBD1",
- "name": "local_interpreter_python2",
- "group": "spark",
- "properties": {
- "zeppelin.spark.printREPLOutput": {
- "propertyName": "zeppelin.spark.printREPLOutput",
- "value": "true",
- "description": "Print REPL output",
- "type": "checkbox"
- },
- "zeppelin.dep.additionalRemoteRepository": {
- "envName": "ZEPPELIN_DEP_ADDITIONALREMOTEREPOSITORY",
- "propertyName": "zeppelin.dep.additionalRemoteRepository",
- "value": "spark-packages,http://dl.bintray.com/spark-packages/maven,false;",
- "description": "",
- "type": "string"
- },
- "zeppelin.spark.sql.stacktrace": {
- "envName": "ZEPPELIN_SPARK_SQL_STACKTRACE",
- "propertyName": "zeppelin.spark.sql.stacktrace",
- "value": "false",
- "description": "",
- "type": "checkbox"
- },
- "zeppelin.spark.importImplicit":{
- "envName": "ZEPPELIN_SPARK_IMPORTIMPLICIT",
- "propertyName": "zeppelin.spark.importImplicit",
- "value": "true",
- "description": "",
- "type": "checkbox"
- },
- "zeppelin.spark.concurrentSQL": {
- "envName": "ZEPPELIN_SPARK_CONCURRENTSQL",
- "propertyName": "zeppelin.spark.concurrentSQL",
- "value": "false",
- "description": "",
- "type": "checkbox"
- },
- "zeppelin.spark.useHiveContext": {
- "envName": "ZEPPELIN_SPARK_USEHIVECONTEXT",
- "propertyName": "zeppelin.spark.useHiveContext",
- "value": "true",
- "description": "Use HiveContext instead of SQLContext if it is true.",
- "type": "checkbox"
- },
- "zeppelin.pyspark.python": {
- "envName": "ZEPPELIN_PYSPARK_PYTHON",
- "propertyName": "zeppelin.pyspark.python",
- "value": "python",
- "description": "",
- "type": "string"
- },
- "zeppelin.dep.localrepo": {
- "envName": "ZEPPELIN_DEP_LOCALREPO",
- "propertyName": "zeppelin.dep.localrepo",
- "value": "local-repo",
- "description": "",
- "type": "string"
- },
- "zeppelin.spark.maxResult": {
- "envName": "ZEPPELIN_SPARK_MAXRESULT",
- "propertyName": "zeppelin.spark.maxResult",
- "value": "1000",
- "description": "Max number of Spark SQL result to display.",
- "type": "number"
- },
- "master":{
- "envName": "Master",
- "propertyName": "spark.master",
- "value": "local[*]",
- "description": "Spark master uri. ex) spark://masterhost:7077",
- "type": "string"
- },
- "spark.app.name": {
- "envName": "SPARK_APP_NAME",
- "propertyName": "spark.app.name",
- "value": "Zeppelin",
- "description": "The name of spark application.",
- "type": "string"
- },
- "spark.hadoop.fs.s3a.endpoint": {
- "envName": "SPARK_HADOOP_FS_S3A_ENDPOINT",
- "propertyName": "spark.hadoop.fs.s3a.endpoint",
- "value": "ENDPOINTURL",
- "description": "",
- "type": "string"
- },
- "spark.driver.memory": {
- "envName": "MEMORY_DRIVER",
- "propertyName": "spark.driver.memory",
- "value": "DRIVER_MEMORY",
- "description": "",
- "type": "string"
- }
- },
- "interpreterGroup": [
- {
- "class": "org.apache.zeppelin.spark.SparkInterpreter",
- "name": "spark"
- },
- {
- "class": "org.apache.zeppelin.spark.PySparkInterpreter",
- "name": "pyspark"
- }
- ],
- "dependencies": [],
- "option": {
- "remote": true,
- "perNoteSession": false,
- "perNoteProcess": false,
- "isExistingProcess": false
- }
- },
"2C6RJRBD2": {
"id": "2C6RJRBD2",
"name": "local_interpreter_python3",
@@ -208,6 +94,13 @@
"value": "DRIVER_MEMORY",
"description": "",
"type": "string"
+ },
+ "spark.pyspark.python": {
+ "envName": "PYSPARK_PYTHON",
+ "propertyName": "spark.pyspark.python",
+ "value": "PYTHON_VENV_PATH",
+ "description": "",
+ "type": "string"
}
},
"interpreterGroup": [
diff --git a/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py b/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
index 868b289..a26273e 100644
--- a/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
+++ b/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
@@ -105,6 +105,7 @@ def configure_zeppelin(os_user):
conn.sudo('mkdir -p /opt/zeppelin/lib/interpreter/')
conn.sudo('cp /opt/zeppelin/interpreter/md/zeppelin-markdown-*.jar /opt/zeppelin/lib/interpreter/') # necessary when executing paragraph launches java process with "-cp :/opt/zeppelin/lib/interpreter/*:"
conn.sudo('cp /opt/zeppelin/interpreter/sh/zeppelin-shell-*.jar /opt/zeppelin/lib/interpreter/')
+ conn.sudo('rm -r /opt/zeppelin/interpreter/python/')
except Exception as err:
print('Error:', str(err))
sys.exit(1)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org