You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by pr...@apache.org on 2016/07/02 14:22:19 UTC

[1/2] zeppelin git commit: Revert "merge master into branch-0.6"

Repository: zeppelin
Updated Branches:
  refs/heads/branch-0.6 7db07f227 -> d962297c2


Revert "merge master into branch-0.6"

This reverts commit 7db07f22742896c6de0691cacbd37dbe5b0b493e.


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/bc75d6ba
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/bc75d6ba
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/bc75d6ba

Branch: refs/heads/branch-0.6
Commit: bc75d6bad9779ede8f03ab59efcbeaca614de07b
Parents: 7db07f2
Author: Prabhjyot Singh <pr...@gmail.com>
Authored: Sat Jul 2 19:43:35 2016 +0530
Committer: Prabhjyot Singh <pr...@gmail.com>
Committed: Sat Jul 2 19:43:35 2016 +0530

----------------------------------------------------------------------
 .../org/apache/zeppelin/livy/LivyHelper.java    |  4 +--
 .../zeppelin/livy/LivySparkSQLInterpreter.java  | 35 ++++++++++----------
 .../src/main/resources/interpreter-setting.json |  5 ---
 3 files changed, 19 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bc75d6ba/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java b/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
index ec77f1a..2c66fa9 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
@@ -96,7 +96,7 @@ public class LivyHelper {
               }.getType());
           if (jsonMap.get("state").equals("idle")) {
             break;
-          } else if (jsonMap.get("state").equals("error") || jsonMap.get("state").equals("dead")) {
+          } else if (jsonMap.get("state").equals("error")) {
             json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/" +
                     sessionId + "/log",
                 "GET", null,
@@ -124,7 +124,7 @@ public class LivyHelper {
 
   protected void initializeSpark(final InterpreterContext context,
                                  final Map<String, Integer> userSessionMap) throws Exception {
-    interpret("val sqlContext = new org.apache.spark.sql.SQLContext(sc)\n" +
+    interpret("val sqlContext= new org.apache.spark.sql.SQLContext(sc)\n" +
         "import sqlContext.implicits._", context, userSessionMap);
   }
 

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bc75d6ba/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
index 22773df..3c60204 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
@@ -35,6 +35,20 @@ import java.util.Properties;
 public class LivySparkSQLInterpreter extends Interpreter {
 
   Logger LOGGER = LoggerFactory.getLogger(LivySparkSQLInterpreter.class);
+  static String DEFAULT_MAX_RESULT = "1000";
+
+  static {
+    Interpreter.register(
+        "sql",
+        "livy",
+        LivySparkSQLInterpreter.class.getName(),
+        new InterpreterPropertyBuilder()
+            .add("zeppelin.livy.spark.maxResult",
+                DEFAULT_MAX_RESULT,
+                "Max number of SparkSQL result to display.")
+            .build()
+    );
+  }
 
   protected Map<String, Integer> userSessionMap;
   private LivyHelper livyHelper;
@@ -80,7 +94,7 @@ public class LivySparkSQLInterpreter extends Interpreter {
               line.replaceAll("\"", "\\\\\"")
                   .replaceAll("\\n", " ")
               + "\").show(" +
-              property.get("zeppelin.livy.spark.sql.maxResult") + ")",
+              property.get("zeppelin.livy.spark.maxResult") + ")",
           interpreterContext, userSessionMap);
 
       if (res.code() == InterpreterResult.Code.SUCCESS) {
@@ -123,10 +137,6 @@ public class LivySparkSQLInterpreter extends Interpreter {
     }
   }
 
-  public boolean concurrentSQL() {
-    return Boolean.parseBoolean(getProperty("zeppelin.livy.concurrentSQL"));
-  }
-
   @Override
   public void cancel(InterpreterContext context) {
     livyHelper.cancelHTTP(context.getParagraphId());
@@ -144,19 +154,8 @@ public class LivySparkSQLInterpreter extends Interpreter {
 
   @Override
   public Scheduler getScheduler() {
-    if (concurrentSQL()) {
-      int maxConcurrency = 10;
-      return SchedulerFactory.singleton().createOrGetParallelScheduler(
-          LivySparkInterpreter.class.getName() + this.hashCode(), maxConcurrency);
-    } else {
-      Interpreter intp =
-          getInterpreterInTheSameSessionByClassName(LivySparkInterpreter.class.getName());
-      if (intp != null) {
-        return intp.getScheduler();
-      } else {
-        return null;
-      }
-    }
+    return SchedulerFactory.singleton().createOrGetFIFOScheduler(
+        LivySparkInterpreter.class.getName() + this.hashCode());
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bc75d6ba/livy/src/main/resources/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/livy/src/main/resources/interpreter-setting.json b/livy/src/main/resources/interpreter-setting.json
index e90216c..7ae435f 100644
--- a/livy/src/main/resources/interpreter-setting.json
+++ b/livy/src/main/resources/interpreter-setting.json
@@ -87,11 +87,6 @@
         "propertyName": "zeppelin.livy.spark.sql.maxResult",
         "defaultValue": "1000",
         "description": "Max number of SparkSQL result to display."
-      },
-      "zeppelin.livy.concurrentSQL": {
-        "propertyName": "zeppelin.livy.concurrentSQL",
-        "defaultValue": "false",
-        "description": "Execute multiple SQL concurrently if set true."
       }
     }
   },


[2/2] zeppelin git commit: [Zeppelin-1090][Hot Fix] LivySparkSQLInterpreter doesn't work in FIFO.

Posted by pr...@apache.org.
[Zeppelin-1090][Hot Fix] LivySparkSQLInterpreter doesn't work in FIFO.

LivySparkSQLInterpreter should work in FIFO with LivySparkInterpreter just like SparkSqlInterpreter works with SparkInterpreter

[Hot Fix]

* [x] - LivySparkSQLInterpreter should work in FIFO just like SparkSqlInterpreter
* [x] - add in property file zeppelin.livy.concurrentSQL

* [Zeppelin-1090](https://issues.apache.org/jira/browse/ZEPPELIN-1090)

In a notebook create 2 paragraph make content of first as

```
%livy
Thread.sleep(10000)
```

and other as

```
%livy.sql
show tables
```

The second paragraph should not get executed before first.

* Does the licenses files need update? n/a
* Is there breaking changes for older versions? n/a
* Does this needs documentation? n/a

Author: Prabhjyot Singh <pr...@gmail.com>

Closes #1109 from prabhjyotsingh/ZEPPELIN-1090 and squashes the following commits:

a2f863e [Prabhjyot Singh] check for both dead and error cases
fe2ec72 [Prabhjyot Singh] LivySparkSQLInterpreter should work in FIFO just like SparkSqlInterpreter
2f1c64c [Prabhjyot Singh] add in property file zeppelin.livy.concurrentSQL


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/d962297c
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/d962297c
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/d962297c

Branch: refs/heads/branch-0.6
Commit: d962297c26c5d70ed8e77696ca30117d6914b0a5
Parents: bc75d6b
Author: Prabhjyot Singh <pr...@gmail.com>
Authored: Thu Jun 30 15:41:01 2016 +0530
Committer: Prabhjyot Singh <pr...@gmail.com>
Committed: Sat Jul 2 19:45:44 2016 +0530

----------------------------------------------------------------------
 .../org/apache/zeppelin/livy/LivyHelper.java    |  4 +--
 .../zeppelin/livy/LivySparkSQLInterpreter.java  | 35 ++++++++++----------
 .../src/main/resources/interpreter-setting.json |  5 +++
 3 files changed, 25 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d962297c/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java b/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
index 2c66fa9..ec77f1a 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
@@ -96,7 +96,7 @@ public class LivyHelper {
               }.getType());
           if (jsonMap.get("state").equals("idle")) {
             break;
-          } else if (jsonMap.get("state").equals("error")) {
+          } else if (jsonMap.get("state").equals("error") || jsonMap.get("state").equals("dead")) {
             json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/" +
                     sessionId + "/log",
                 "GET", null,
@@ -124,7 +124,7 @@ public class LivyHelper {
 
   protected void initializeSpark(final InterpreterContext context,
                                  final Map<String, Integer> userSessionMap) throws Exception {
-    interpret("val sqlContext= new org.apache.spark.sql.SQLContext(sc)\n" +
+    interpret("val sqlContext = new org.apache.spark.sql.SQLContext(sc)\n" +
         "import sqlContext.implicits._", context, userSessionMap);
   }
 

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d962297c/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
index 3c60204..22773df 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
@@ -35,20 +35,6 @@ import java.util.Properties;
 public class LivySparkSQLInterpreter extends Interpreter {
 
   Logger LOGGER = LoggerFactory.getLogger(LivySparkSQLInterpreter.class);
-  static String DEFAULT_MAX_RESULT = "1000";
-
-  static {
-    Interpreter.register(
-        "sql",
-        "livy",
-        LivySparkSQLInterpreter.class.getName(),
-        new InterpreterPropertyBuilder()
-            .add("zeppelin.livy.spark.maxResult",
-                DEFAULT_MAX_RESULT,
-                "Max number of SparkSQL result to display.")
-            .build()
-    );
-  }
 
   protected Map<String, Integer> userSessionMap;
   private LivyHelper livyHelper;
@@ -94,7 +80,7 @@ public class LivySparkSQLInterpreter extends Interpreter {
               line.replaceAll("\"", "\\\\\"")
                   .replaceAll("\\n", " ")
               + "\").show(" +
-              property.get("zeppelin.livy.spark.maxResult") + ")",
+              property.get("zeppelin.livy.spark.sql.maxResult") + ")",
           interpreterContext, userSessionMap);
 
       if (res.code() == InterpreterResult.Code.SUCCESS) {
@@ -137,6 +123,10 @@ public class LivySparkSQLInterpreter extends Interpreter {
     }
   }
 
+  public boolean concurrentSQL() {
+    return Boolean.parseBoolean(getProperty("zeppelin.livy.concurrentSQL"));
+  }
+
   @Override
   public void cancel(InterpreterContext context) {
     livyHelper.cancelHTTP(context.getParagraphId());
@@ -154,8 +144,19 @@ public class LivySparkSQLInterpreter extends Interpreter {
 
   @Override
   public Scheduler getScheduler() {
-    return SchedulerFactory.singleton().createOrGetFIFOScheduler(
-        LivySparkInterpreter.class.getName() + this.hashCode());
+    if (concurrentSQL()) {
+      int maxConcurrency = 10;
+      return SchedulerFactory.singleton().createOrGetParallelScheduler(
+          LivySparkInterpreter.class.getName() + this.hashCode(), maxConcurrency);
+    } else {
+      Interpreter intp =
+          getInterpreterInTheSameSessionByClassName(LivySparkInterpreter.class.getName());
+      if (intp != null) {
+        return intp.getScheduler();
+      } else {
+        return null;
+      }
+    }
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d962297c/livy/src/main/resources/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/livy/src/main/resources/interpreter-setting.json b/livy/src/main/resources/interpreter-setting.json
index 7ae435f..e90216c 100644
--- a/livy/src/main/resources/interpreter-setting.json
+++ b/livy/src/main/resources/interpreter-setting.json
@@ -87,6 +87,11 @@
         "propertyName": "zeppelin.livy.spark.sql.maxResult",
         "defaultValue": "1000",
         "description": "Max number of SparkSQL result to display."
+      },
+      "zeppelin.livy.concurrentSQL": {
+        "propertyName": "zeppelin.livy.concurrentSQL",
+        "defaultValue": "false",
+        "description": "Execute multiple SQL concurrently if set true."
       }
     }
   },