You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by jo...@apache.org on 2016/06/30 12:07:29 UTC
zeppelin git commit: [ZEPPELIN-1081] Extract spark.r setting from
interpreter-setting.json on Spark interpreter mudule
Repository: zeppelin
Updated Branches:
refs/heads/master c348161df -> 4c66689c2
[ZEPPELIN-1081] Extract spark.r setting from interpreter-setting.json on Spark interpreter mudule
### What is this PR for?
Avoid setting `spark.r` without `-Psparkr`. This enables to use `-Pr` correctly.
### What type of PR is it?
[Bug Fix]
### Todos
* [x] - Separate settings with profile
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-1081
### How should this be tested?
1. Build Zeppelin with `-Pr`
1. Run codes with R
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jongyoul Lee <jo...@gmail.com>
Closes #1108 from jongyoul/ZEPPELIN-1081 and squashes the following commits:
d7b7481 [Jongyoul Lee] Fixed not to include spark.r setting while not using -Psparkr
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/4c66689c
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/4c66689c
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/4c66689c
Branch: refs/heads/master
Commit: 4c66689c29e131a4f1d4735e8a099063d7102d16
Parents: c348161
Author: Jongyoul Lee <jo...@gmail.com>
Authored: Thu Jun 30 15:34:31 2016 +0900
Committer: Jongyoul Lee <jo...@apache.org>
Committed: Thu Jun 30 21:07:24 2016 +0900
----------------------------------------------------------------------
spark/pom.xml | 13 ++
.../src/main/resources/interpreter-setting.json | 31 ----
.../sparkr-resources/interpreter-setting.json | 153 +++++++++++++++++++
3 files changed, 166 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4c66689c/spark/pom.xml
----------------------------------------------------------------------
diff --git a/spark/pom.xml b/spark/pom.xml
index 6cffcf1..987308c 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -437,6 +437,19 @@
<!-- to deactivate 'exclude-sparkr' automatically when 'spark' is activated -->
<profile>
<id>sparkr</id>
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <excludes>
+ <exclude>interpreter-setting.json</exclude>
+ </excludes>
+ </resource>
+ <resource>
+ <directory>src/main/sparkr-resources</directory>
+ </resource>
+ </resources>
+ </build>
</profile>
<profile>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4c66689c/spark/src/main/resources/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/spark/src/main/resources/interpreter-setting.json b/spark/src/main/resources/interpreter-setting.json
index 4902baf..2343a0f 100644
--- a/spark/src/main/resources/interpreter-setting.json
+++ b/spark/src/main/resources/interpreter-setting.json
@@ -118,36 +118,5 @@
"description": "Python command to run pyspark with"
}
}
- },
- {
- "group": "spark",
- "name": "r",
- "className": "org.apache.zeppelin.spark.SparkRInterpreter",
- "properties": {
- "zeppelin.R.knitr": {
- "envName": "ZEPPELIN_R_KNITR",
- "propertyName": "zeppelin.R.knitr",
- "defaultValue": "true",
- "description": "whether use knitr or not"
- },
- "zeppelin.R.cmd": {
- "envName": "ZEPPELIN_R_CMD",
- "propertyName": "zeppelin.R.cmd",
- "defaultValue": "R",
- "description": "R repl path"
- },
- "zeppelin.R.image.width": {
- "envName": "ZEPPELIN_R_IMAGE_WIDTH",
- "propertyName": "zeppelin.R.image.width",
- "defaultValue": "100%",
- "description": ""
- },
- "zeppelin.R.render.options": {
- "envName": "ZEPPELIN_R_RENDER_OPTIONS",
- "propertyName": "zeppelin.R.render.options",
- "defaultValue": "out.format = 'html', comment = NA, echo = FALSE, results = 'asis', message = F, warning = F",
- "description": ""
- }
- }
}
]
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4c66689c/spark/src/main/sparkr-resources/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/spark/src/main/sparkr-resources/interpreter-setting.json b/spark/src/main/sparkr-resources/interpreter-setting.json
new file mode 100644
index 0000000..4902baf
--- /dev/null
+++ b/spark/src/main/sparkr-resources/interpreter-setting.json
@@ -0,0 +1,153 @@
+[
+ {
+ "group": "spark",
+ "name": "spark",
+ "className": "org.apache.zeppelin.spark.SparkInterpreter",
+ "defaultInterpreter": true,
+ "properties": {
+ "spark.executor.memory": {
+ "envName": null,
+ "propertyName": "spark.executor.memory",
+ "defaultValue": "",
+ "description": "Executor memory per worker instance. ex) 512m, 32g"
+ },
+ "args": {
+ "envName": null,
+ "propertyName": null,
+ "defaultValue": "",
+ "description": "spark commandline args"
+ },
+ "zeppelin.spark.useHiveContext": {
+ "envName": "ZEPPELIN_SPARK_USEHIVECONTEXT",
+ "propertyName": "zeppelin.spark.useHiveContext",
+ "defaultValue": "true",
+ "description": "Use HiveContext instead of SQLContext if it is true."
+ },
+ "spark.app.name": {
+ "envName": "SPARK_APP_NAME",
+
+ "propertyName": "spark.app.name",
+ "defaultValue": "Zeppelin",
+ "description": "The name of spark application."
+ },
+ "zeppelin.spark.printREPLOutput": {
+ "envName": null,
+ "propertyName": null,
+ "defaultValue": "true",
+ "description": "Print REPL output"
+ },
+ "spark.cores.max": {
+ "envName": null,
+ "propertyName": "spark.cores.max",
+ "defaultValue": "",
+ "description": "Total number of cores to use. Empty value uses all available core."
+ },
+ "zeppelin.spark.maxResult": {
+ "envName": "ZEPPELIN_SPARK_MAXRESULT",
+ "propertyName": "zeppelin.spark.maxResult",
+ "defaultValue": "1000",
+ "description": "Max number of SparkSQL result to display."
+ },
+ "master": {
+ "envName": "MASTER",
+ "propertyName": "spark.master",
+ "defaultValue": "local[*]",
+ "description": "Spark master uri. ex) spark://masterhost:7077"
+ }
+ }
+ },
+ {
+ "group": "spark",
+ "name": "sql",
+ "className": "org.apache.zeppelin.spark.SparkSqlInterpreter",
+ "properties": {
+ "zeppelin.spark.concurrentSQL": {
+ "envName": "ZEPPELIN_SPARK_CONCURRENTSQL",
+ "propertyName": "zeppelin.spark.concurrentSQL",
+ "defaultValue": "false",
+ "description": "Execute multiple SQL concurrently if set true."
+ },
+ "zeppelin.spark.sql.stacktrace": {
+ "envName": "ZEPPELIN_SPARK_SQL_STACKTRACE",
+ "propertyName": "zeppelin.spark.sql.stacktrace",
+ "defaultValue": "false",
+ "description": "Show full exception stacktrace for SQL queries if set to true."
+ },
+ "zeppelin.spark.maxResult": {
+ "envName": "ZEPPELIN_SPARK_MAXRESULT",
+ "propertyName": "zeppelin.spark.maxResult",
+ "defaultValue": "1000",
+ "description": "Max number of SparkSQL result to display."
+ },
+ "zeppelin.spark.importImplicit": {
+ "envName": "ZEPPELIN_SPARK_IMPORTIMPLICIT",
+ "propertyName": "zeppelin.spark.importImplicit",
+ "defaultValue": "true",
+ "description": "Import implicits, UDF collection, and sql if set true. true by default."
+ }
+ }
+ },
+ {
+ "group": "spark",
+ "name": "dep",
+ "className": "org.apache.zeppelin.spark.DepInterpreter",
+ "properties": {
+ "zeppelin.dep.localrepo": {
+ "envName": "ZEPPELIN_DEP_LOCALREPO",
+ "propertyName": null,
+ "defaultValue": "local-repo",
+ "description": "local repository for dependency loader"
+ },
+ "zeppelin.dep.additionalRemoteRepository": {
+ "envName": null,
+ "propertyName": null,
+ "defaultValue": "spark-packages,http://dl.bintray.com/spark-packages/maven,false;",
+ "description": "A list of 'id,remote-repository-URL,is-snapshot;' for each remote repository."
+ }
+ }
+ },
+ {
+ "group": "spark",
+ "name": "pyspark",
+ "className": "org.apache.zeppelin.spark.PySparkInterpreter",
+ "properties": {
+ "zeppelin.pyspark.python": {
+ "envName": "PYSPARK_PYTHON",
+ "propertyName": null,
+ "defaultValue": "python",
+ "description": "Python command to run pyspark with"
+ }
+ }
+ },
+ {
+ "group": "spark",
+ "name": "r",
+ "className": "org.apache.zeppelin.spark.SparkRInterpreter",
+ "properties": {
+ "zeppelin.R.knitr": {
+ "envName": "ZEPPELIN_R_KNITR",
+ "propertyName": "zeppelin.R.knitr",
+ "defaultValue": "true",
+ "description": "whether use knitr or not"
+ },
+ "zeppelin.R.cmd": {
+ "envName": "ZEPPELIN_R_CMD",
+ "propertyName": "zeppelin.R.cmd",
+ "defaultValue": "R",
+ "description": "R repl path"
+ },
+ "zeppelin.R.image.width": {
+ "envName": "ZEPPELIN_R_IMAGE_WIDTH",
+ "propertyName": "zeppelin.R.image.width",
+ "defaultValue": "100%",
+ "description": ""
+ },
+ "zeppelin.R.render.options": {
+ "envName": "ZEPPELIN_R_RENDER_OPTIONS",
+ "propertyName": "zeppelin.R.render.options",
+ "defaultValue": "out.format = 'html', comment = NA, echo = FALSE, results = 'asis', message = F, warning = F",
+ "description": ""
+ }
+ }
+ }
+]