You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by pr...@apache.org on 2016/07/02 14:22:19 UTC
[1/2] zeppelin git commit: Revert "merge master into branch-0.6"
Repository: zeppelin
Updated Branches:
refs/heads/branch-0.6 7db07f227 -> d962297c2
Revert "merge master into branch-0.6"
This reverts commit 7db07f22742896c6de0691cacbd37dbe5b0b493e.
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/bc75d6ba
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/bc75d6ba
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/bc75d6ba
Branch: refs/heads/branch-0.6
Commit: bc75d6bad9779ede8f03ab59efcbeaca614de07b
Parents: 7db07f2
Author: Prabhjyot Singh <pr...@gmail.com>
Authored: Sat Jul 2 19:43:35 2016 +0530
Committer: Prabhjyot Singh <pr...@gmail.com>
Committed: Sat Jul 2 19:43:35 2016 +0530
----------------------------------------------------------------------
.../org/apache/zeppelin/livy/LivyHelper.java | 4 +--
.../zeppelin/livy/LivySparkSQLInterpreter.java | 35 ++++++++++----------
.../src/main/resources/interpreter-setting.json | 5 ---
3 files changed, 19 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bc75d6ba/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java b/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
index ec77f1a..2c66fa9 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
@@ -96,7 +96,7 @@ public class LivyHelper {
}.getType());
if (jsonMap.get("state").equals("idle")) {
break;
- } else if (jsonMap.get("state").equals("error") || jsonMap.get("state").equals("dead")) {
+ } else if (jsonMap.get("state").equals("error")) {
json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/" +
sessionId + "/log",
"GET", null,
@@ -124,7 +124,7 @@ public class LivyHelper {
protected void initializeSpark(final InterpreterContext context,
final Map<String, Integer> userSessionMap) throws Exception {
- interpret("val sqlContext = new org.apache.spark.sql.SQLContext(sc)\n" +
+ interpret("val sqlContext= new org.apache.spark.sql.SQLContext(sc)\n" +
"import sqlContext.implicits._", context, userSessionMap);
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bc75d6ba/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
index 22773df..3c60204 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
@@ -35,6 +35,20 @@ import java.util.Properties;
public class LivySparkSQLInterpreter extends Interpreter {
Logger LOGGER = LoggerFactory.getLogger(LivySparkSQLInterpreter.class);
+ static String DEFAULT_MAX_RESULT = "1000";
+
+ static {
+ Interpreter.register(
+ "sql",
+ "livy",
+ LivySparkSQLInterpreter.class.getName(),
+ new InterpreterPropertyBuilder()
+ .add("zeppelin.livy.spark.maxResult",
+ DEFAULT_MAX_RESULT,
+ "Max number of SparkSQL result to display.")
+ .build()
+ );
+ }
protected Map<String, Integer> userSessionMap;
private LivyHelper livyHelper;
@@ -80,7 +94,7 @@ public class LivySparkSQLInterpreter extends Interpreter {
line.replaceAll("\"", "\\\\\"")
.replaceAll("\\n", " ")
+ "\").show(" +
- property.get("zeppelin.livy.spark.sql.maxResult") + ")",
+ property.get("zeppelin.livy.spark.maxResult") + ")",
interpreterContext, userSessionMap);
if (res.code() == InterpreterResult.Code.SUCCESS) {
@@ -123,10 +137,6 @@ public class LivySparkSQLInterpreter extends Interpreter {
}
}
- public boolean concurrentSQL() {
- return Boolean.parseBoolean(getProperty("zeppelin.livy.concurrentSQL"));
- }
-
@Override
public void cancel(InterpreterContext context) {
livyHelper.cancelHTTP(context.getParagraphId());
@@ -144,19 +154,8 @@ public class LivySparkSQLInterpreter extends Interpreter {
@Override
public Scheduler getScheduler() {
- if (concurrentSQL()) {
- int maxConcurrency = 10;
- return SchedulerFactory.singleton().createOrGetParallelScheduler(
- LivySparkInterpreter.class.getName() + this.hashCode(), maxConcurrency);
- } else {
- Interpreter intp =
- getInterpreterInTheSameSessionByClassName(LivySparkInterpreter.class.getName());
- if (intp != null) {
- return intp.getScheduler();
- } else {
- return null;
- }
- }
+ return SchedulerFactory.singleton().createOrGetFIFOScheduler(
+ LivySparkInterpreter.class.getName() + this.hashCode());
}
@Override
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bc75d6ba/livy/src/main/resources/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/livy/src/main/resources/interpreter-setting.json b/livy/src/main/resources/interpreter-setting.json
index e90216c..7ae435f 100644
--- a/livy/src/main/resources/interpreter-setting.json
+++ b/livy/src/main/resources/interpreter-setting.json
@@ -87,11 +87,6 @@
"propertyName": "zeppelin.livy.spark.sql.maxResult",
"defaultValue": "1000",
"description": "Max number of SparkSQL result to display."
- },
- "zeppelin.livy.concurrentSQL": {
- "propertyName": "zeppelin.livy.concurrentSQL",
- "defaultValue": "false",
- "description": "Execute multiple SQL concurrently if set true."
}
}
},
[2/2] zeppelin git commit: [Zeppelin-1090][Hot Fix]
LivySparkSQLInterpreter doesn't work in FIFO.
Posted by pr...@apache.org.
[Zeppelin-1090][Hot Fix] LivySparkSQLInterpreter doesn't work in FIFO.
LivySparkSQLInterpreter should work in FIFO with LivySparkInterpreter just like SparkSqlInterpreter works with SparkInterpreter
[Hot Fix]
* [x] - LivySparkSQLInterpreter should work in FIFO just like SparkSqlInterpreter
* [x] - add in property file zeppelin.livy.concurrentSQL
* [Zeppelin-1090](https://issues.apache.org/jira/browse/ZEPPELIN-1090)
In a notebook create 2 paragraph make content of first as
```
%livy
Thread.sleep(10000)
```
and other as
```
%livy.sql
show tables
```
The second paragraph should not get executed before first.
* Does the licenses files need update? n/a
* Is there breaking changes for older versions? n/a
* Does this needs documentation? n/a
Author: Prabhjyot Singh <pr...@gmail.com>
Closes #1109 from prabhjyotsingh/ZEPPELIN-1090 and squashes the following commits:
a2f863e [Prabhjyot Singh] check for both dead and error cases
fe2ec72 [Prabhjyot Singh] LivySparkSQLInterpreter should work in FIFO just like SparkSqlInterpreter
2f1c64c [Prabhjyot Singh] add in property file zeppelin.livy.concurrentSQL
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/d962297c
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/d962297c
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/d962297c
Branch: refs/heads/branch-0.6
Commit: d962297c26c5d70ed8e77696ca30117d6914b0a5
Parents: bc75d6b
Author: Prabhjyot Singh <pr...@gmail.com>
Authored: Thu Jun 30 15:41:01 2016 +0530
Committer: Prabhjyot Singh <pr...@gmail.com>
Committed: Sat Jul 2 19:45:44 2016 +0530
----------------------------------------------------------------------
.../org/apache/zeppelin/livy/LivyHelper.java | 4 +--
.../zeppelin/livy/LivySparkSQLInterpreter.java | 35 ++++++++++----------
.../src/main/resources/interpreter-setting.json | 5 +++
3 files changed, 25 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d962297c/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java b/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
index 2c66fa9..ec77f1a 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
@@ -96,7 +96,7 @@ public class LivyHelper {
}.getType());
if (jsonMap.get("state").equals("idle")) {
break;
- } else if (jsonMap.get("state").equals("error")) {
+ } else if (jsonMap.get("state").equals("error") || jsonMap.get("state").equals("dead")) {
json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/" +
sessionId + "/log",
"GET", null,
@@ -124,7 +124,7 @@ public class LivyHelper {
protected void initializeSpark(final InterpreterContext context,
final Map<String, Integer> userSessionMap) throws Exception {
- interpret("val sqlContext= new org.apache.spark.sql.SQLContext(sc)\n" +
+ interpret("val sqlContext = new org.apache.spark.sql.SQLContext(sc)\n" +
"import sqlContext.implicits._", context, userSessionMap);
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d962297c/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
index 3c60204..22773df 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
@@ -35,20 +35,6 @@ import java.util.Properties;
public class LivySparkSQLInterpreter extends Interpreter {
Logger LOGGER = LoggerFactory.getLogger(LivySparkSQLInterpreter.class);
- static String DEFAULT_MAX_RESULT = "1000";
-
- static {
- Interpreter.register(
- "sql",
- "livy",
- LivySparkSQLInterpreter.class.getName(),
- new InterpreterPropertyBuilder()
- .add("zeppelin.livy.spark.maxResult",
- DEFAULT_MAX_RESULT,
- "Max number of SparkSQL result to display.")
- .build()
- );
- }
protected Map<String, Integer> userSessionMap;
private LivyHelper livyHelper;
@@ -94,7 +80,7 @@ public class LivySparkSQLInterpreter extends Interpreter {
line.replaceAll("\"", "\\\\\"")
.replaceAll("\\n", " ")
+ "\").show(" +
- property.get("zeppelin.livy.spark.maxResult") + ")",
+ property.get("zeppelin.livy.spark.sql.maxResult") + ")",
interpreterContext, userSessionMap);
if (res.code() == InterpreterResult.Code.SUCCESS) {
@@ -137,6 +123,10 @@ public class LivySparkSQLInterpreter extends Interpreter {
}
}
+ public boolean concurrentSQL() {
+ return Boolean.parseBoolean(getProperty("zeppelin.livy.concurrentSQL"));
+ }
+
@Override
public void cancel(InterpreterContext context) {
livyHelper.cancelHTTP(context.getParagraphId());
@@ -154,8 +144,19 @@ public class LivySparkSQLInterpreter extends Interpreter {
@Override
public Scheduler getScheduler() {
- return SchedulerFactory.singleton().createOrGetFIFOScheduler(
- LivySparkInterpreter.class.getName() + this.hashCode());
+ if (concurrentSQL()) {
+ int maxConcurrency = 10;
+ return SchedulerFactory.singleton().createOrGetParallelScheduler(
+ LivySparkInterpreter.class.getName() + this.hashCode(), maxConcurrency);
+ } else {
+ Interpreter intp =
+ getInterpreterInTheSameSessionByClassName(LivySparkInterpreter.class.getName());
+ if (intp != null) {
+ return intp.getScheduler();
+ } else {
+ return null;
+ }
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d962297c/livy/src/main/resources/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/livy/src/main/resources/interpreter-setting.json b/livy/src/main/resources/interpreter-setting.json
index 7ae435f..e90216c 100644
--- a/livy/src/main/resources/interpreter-setting.json
+++ b/livy/src/main/resources/interpreter-setting.json
@@ -87,6 +87,11 @@
"propertyName": "zeppelin.livy.spark.sql.maxResult",
"defaultValue": "1000",
"description": "Max number of SparkSQL result to display."
+ },
+ "zeppelin.livy.concurrentSQL": {
+ "propertyName": "zeppelin.livy.concurrentSQL",
+ "defaultValue": "false",
+ "description": "Execute multiple SQL concurrently if set true."
}
}
},