You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mi...@apache.org on 2017/02/02 06:48:59 UTC
zeppelin git commit: [ZEPPELIN-2030] Select statement doesn't work in
LivySparkSQLInterpreter
Repository: zeppelin
Updated Branches:
refs/heads/master fe11b32e8 -> 8774f8073
[ZEPPELIN-2030] Select statement doesn't work in LivySparkSQLInterpreter
### What is this PR for?
I try to execute sql query by ``%livy.sql``, I got a error as below
```
ERROR [2017-01-30 11:15:57,060] ({pool-2-thread-6} LivySparkSQLInterpreter.java[interpret]:143) - Exception in LivySparkSQLInterpreter while interpret
java.lang.StringIndexOutOfBoundsException: String index out of range: 28
at java.lang.String.substring(String.java:1963)
at org.apache.zeppelin.livy.LivySparkSQLInterpreter.parseSQLOutput(LivySparkSQLInterpreter.java:177)
at org.apache.zeppelin.livy.LivySparkSQLInterpreter.interpret(LivySparkSQLInterpreter.java:128)
at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:94)
at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:489)
at org.apache.zeppelin.scheduler.Job.run(Job.java:175)
at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
```
I seem ``parseSQLOutput`` method try to filter out ``only showing top 1 row`` but it doesn't.
Therefore, I fix this issue by modifying condition
### What type of PR is it?
Bug Fix
### Todos
* [x] - none
### What is the Jira issue?
[ZEPPELIN-2030](https://issues.apache.org/jira/browse/ZEPPELIN-2030)
### How should this be tested?
try to execute ``%livy.sql SELECT * FROM bank`` on tutorial
### Screenshots (if appropriate)
Before
After
### Questions:
* Does the licenses files need update?
* Is there breaking changes for older versions?
* Does this needs documentation?
Author: Chin Tzulin <jp...@w022341412910m.local>
Closes #1960 from del680202/bug-2030 and squashes the following commits:
86ebc82 [Chin Tzulin] Fix test problem
d681100 [Chin Tzulin] Add test case into LivySQLInterpreterTest.java
8f499c4 [Chin Tzulin] [ZEPPELIN-2030] Select statement doesn't work in LivySparkSQLInterpreter
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/8774f807
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/8774f807
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/8774f807
Branch: refs/heads/master
Commit: 8774f80732927eaa62709ea0f790fb165847a389
Parents: fe11b32
Author: Chin Tzulin <jp...@w022341412910m.local>
Authored: Tue Jan 31 17:34:38 2017 +0900
Committer: Mina Lee <mi...@apache.org>
Committed: Thu Feb 2 15:48:38 2017 +0900
----------------------------------------------------------------------
.../zeppelin/livy/LivySparkSQLInterpreter.java | 3 ++-
.../zeppelin/livy/LivySQLInterpreterTest.java | 22 +++++++++++++++++++-
2 files changed, 23 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8774f807/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
index 9389d4d..48e4967 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
@@ -169,8 +169,9 @@ public class LivySparkSQLInterpreter extends BaseLivyInterprereter {
}
for (String line : lines) {
+ // Only match format "|....|"
// skip line like "+---+---+" and "only showing top 1 row"
- if (!line.matches("(\\+\\-+)+\\+") || line.contains("only showing")) {
+ if (line.matches("^\\|.*\\|$")) {
List<String> cells = new ArrayList<>();
for (Pair pair : pairs) {
// strip the blank space around the cell
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8774f807/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java
----------------------------------------------------------------------
diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java b/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java
index a764fba..9065f35 100644
--- a/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java
+++ b/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java
@@ -83,17 +83,37 @@ public class LivySQLInterpreterTest {
// | 2| 2b|
// | 3| 3c|
// +---+---+
+ // only showing top 3 rows
rows = sqlInterpreter.parseSQLOutput("+---+---+\n" +
"| a| b|\n" +
"+---+---+\n" +
"| 1| 1a|\n" +
"| 2| 2b|\n" +
"| 3| 3c|\n" +
- "+---+---+");
+ "+---+---+\n" +
+ "only showing top 3 rows");
assertEquals(4, rows.size());
assertEquals("a\tb", rows.get(0));
assertEquals("1\t1a", rows.get(1));
assertEquals("2\t2b", rows.get(2));
assertEquals("3\t3c", rows.get(3));
+
+
+ // sql output with 1 rows and showing "only showing top 1 rows"
+ // +---+
+ // | a|
+ // +---+
+ // | 1|
+ // +---+
+ // only showing top 1 rows
+ rows = sqlInterpreter.parseSQLOutput("+---+\n" +
+ "| a|\n" +
+ "+---+\n" +
+ "| 1|\n" +
+ "+---+\n" +
+ "only showing top 1 rows");
+ assertEquals(2, rows.size());
+ assertEquals("a", rows.get(0));
+ assertEquals("1", rows.get(1));
}
}