You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2017/11/02 07:47:44 UTC
zeppelin git commit: [ZEPPELIN-2965] Add code completion for livy
interpreter
Repository: zeppelin
Updated Branches:
refs/heads/master 13f6dc7de -> 02daea1c5
[ZEPPELIN-2965] Add code completion for livy interpreter
### What is this PR for?
This PR adds code autocompletion feature to LivyInterpreter.
Livy version 0.5 will have an auto completion API.
### What type of PR is it?
Feature
### Todos
* [ ] - Task
### What is the Jira issue?
[ZEPPELIN-2965] https://issues.apache.org/jira/browse/ZEPPELIN-2965
### How should this be tested?
Pulled out server calls to a separate class to support proper unit-testing with mockito.
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Pascal Pellmont <gi...@ppo2.ch>
Closes #2624 from pellmont/ZEPPELIN-2965 and squashes the following commits:
aeabf86 [Pascal Pellmont] ZEPPELIN-2965 code completion for livy - added missing commas in interpreter-settings.json
55515a7 [Pascal Pellmont] ZEPPELIN-2965 code completion for livy - fixed checkstyle violation
103681e [Pascal Pellmont] Merge branch 'master' of github.com:apache/zeppelin into ZEPPELIN-2965
d35a90f [Pascal Pellmont] ZEPPELIN-2965 code completion for livy - added missing commas in interpreter-settings.json
bbb48fb [Pascal Pellmont] ZEPPELIN-2965 code completion for livy - removed pointless try
2a21f3c [Pascal Pellmont] ZEPPELIN-2965 code completion for livy - no new session for code completion
a460ae7 [Pascal Pellmont] ZEPPELIN-2965 code completion for livy - fixed typo
9fac364 [Pascal Pellmont] ZEPPELIN-2965 code completion for livy - TAB as completion key
32b3c6b [Pascal Pellmont] ZEPPELIN-2965 code completion for livy
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/02daea1c
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/02daea1c
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/02daea1c
Branch: refs/heads/master
Commit: 02daea1c596939592d0d7899600c4243759d5fbf
Parents: 13f6dc7
Author: Pascal Pellmont <gi...@ppo2.ch>
Authored: Thu Nov 2 07:12:07 2017 +0100
Committer: Jeff Zhang <zj...@apache.org>
Committed: Thu Nov 2 15:47:39 2017 +0800
----------------------------------------------------------------------
.../zeppelin/livy/BaseLivyInterpreter.java | 107 +++++++++++++++----
.../src/main/resources/interpreter-setting.json | 15 ++-
2 files changed, 97 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/02daea1c/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
index 2122f53..0cdf464 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
@@ -17,10 +17,25 @@
package org.apache.zeppelin.livy;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.annotations.SerializedName;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.security.KeyStore;
+import java.security.Principal;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.net.ssl.SSLContext;
+
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.http.auth.AuthSchemeProvider;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;
@@ -36,14 +51,19 @@ import org.apache.http.impl.auth.SPNegoSchemeFactory;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResultMessage;
+import org.apache.zeppelin.interpreter.InterpreterUtils;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpEntity;
-import org.springframework.http.MediaType;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.security.kerberos.client.KerberosRestTemplate;
@@ -51,20 +71,10 @@ import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.HttpServerErrorException;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
-import javax.net.ssl.SSLContext;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.security.KeyStore;
-import java.security.Principal;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.annotations.SerializedName;
/**
@@ -205,6 +215,35 @@ public abstract class BaseLivyInterpreter extends Interpreter {
}
@Override
+ public List<InterpreterCompletion> completion(String buf, int cursor,
+ InterpreterContext interpreterContext) {
+ List<InterpreterCompletion> candidates = Collections.emptyList();
+ try {
+ candidates = callCompletion(new CompletionRequest(buf, getSessionKind(), cursor));
+ } catch (SessionNotFoundException e) {
+ LOGGER.warn("Livy session {} is expired. Will return empty list of candidates.",
+ sessionInfo.id);
+ } catch (LivyException le) {
+ logger.error("Failed to call code completions. Will return empty list of candidates", le);
+ }
+ return candidates;
+ }
+
+ private List<InterpreterCompletion> callCompletion(CompletionRequest req) throws LivyException {
+ List<InterpreterCompletion> candidates = new ArrayList<>();
+ try {
+ CompletionResponse resp = CompletionResponse.fromJson(
+ callRestAPI("/sessions/" + sessionInfo.id + "/completion", "POST", req.toJson()));
+ for (String candidate : resp.candidates) {
+ candidates.add(new InterpreterCompletion(candidate, candidate, StringUtils.EMPTY));
+ }
+ } catch (APINotFoundException e) {
+ logger.debug("completion api seems not to be available. (available from livy 0.5)", e);
+ }
+ return candidates;
+ }
+
+ @Override
public void cancel(InterpreterContext context) {
paragraphsToCancel.add(context.getParagraphId());
LOGGER.info("Added paragraph " + context.getParagraphId() + " for cancellation.");
@@ -774,6 +813,34 @@ public abstract class BaseLivyInterpreter extends Interpreter {
}
}
+ static class CompletionRequest {
+ public final String code;
+ public final String kind;
+ public final int cursor;
+
+ public CompletionRequest(String code, String kind, int cursor) {
+ this.code = code;
+ this.kind = kind;
+ this.cursor = cursor;
+ }
+
+ public String toJson() {
+ return gson.toJson(this);
+ }
+ }
+
+ static class CompletionResponse {
+ public final String[] candidates;
+
+ public CompletionResponse(String[] candidates) {
+ this.candidates = candidates;
+ }
+
+ public static CompletionResponse fromJson(String json) {
+ return gson.fromJson(json, CompletionResponse.class);
+ }
+ }
+
private static class LivyVersionResponse {
public String url;
public String branch;
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/02daea1c/livy/src/main/resources/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/livy/src/main/resources/interpreter-setting.json b/livy/src/main/resources/interpreter-setting.json
index ac213ba..2d72487 100644
--- a/livy/src/main/resources/interpreter-setting.json
+++ b/livy/src/main/resources/interpreter-setting.json
@@ -121,7 +121,8 @@
},
"editor": {
"language": "scala",
- "editOnDblClick": false
+ "editOnDblClick": false,
+ "completionKey": "TAB"
}
},
{
@@ -160,7 +161,8 @@
},
"editor": {
"language": "sql",
- "editOnDblClick": false
+ "editOnDblClick": false,
+ "completionKey": "TAB"
}
},
{
@@ -180,7 +182,8 @@
},
"editor": {
"language": "python",
- "editOnDblClick": false
+ "editOnDblClick": false,
+ "completionKey": "TAB"
}
},
{
@@ -200,7 +203,8 @@
},
"editor": {
"language": "python",
- "editOnDblClick": false
+ "editOnDblClick": false,
+ "completionKey": "TAB"
}
},
{
@@ -220,7 +224,8 @@
},
"editor": {
"language": "r",
- "editOnDblClick": false
+ "editOnDblClick": false,
+ "completionKey": "TAB"
}
}
]