You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2017/11/01 09:46:02 UTC
zeppelin git commit: ZEPPELIN-3013. Only parse paragraph text when
text is updated
Repository: zeppelin
Updated Branches:
refs/heads/master 24cc08ed9 -> afd2bca46
ZEPPELIN-3013. Only parse paragraph text when text is updated
### What is this PR for?
Before this PR, each time when paragraph use interpreter, it would parse the text to get the interpreter component which is inefficient. This PR would do it when `setText` is called.
### What type of PR is it?
[ Improvement | Refactoring]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-3013
### How should this be tested?
* Unit test is added
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #2635 from zjffdu/ZEPPELIN-3013 and squashes the following commits:
71c7edf [Jeff Zhang] minor change
c33d4ce [Jeff Zhang] ZEPPELIN-3013. Only parse paragraph text when text is updated
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/afd2bca4
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/afd2bca4
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/afd2bca4
Branch: refs/heads/master
Commit: afd2bca46c778e9e84deb7ba5e4f5b258735022b
Parents: 24cc08e
Author: Jeff Zhang <zj...@apache.org>
Authored: Sun Oct 29 20:41:52 2017 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Wed Nov 1 17:45:56 2017 +0800
----------------------------------------------------------------------
.../apache/zeppelin/socket/NotebookServer.java | 4 +-
.../zeppelin/socket/NotebookServerTest.java | 8 +-
.../java/org/apache/zeppelin/helium/Helium.java | 3 +-
.../helium/HeliumApplicationFactory.java | 8 +-
.../interpreter/InterpreterSetting.java | 7 +
.../interpreter/ManagedInterpreterGroup.java | 2 +-
.../interpreter/remote/RemoteInterpreter.java | 2 +-
.../java/org/apache/zeppelin/notebook/Note.java | 83 ++---
.../org/apache/zeppelin/notebook/Paragraph.java | 370 ++++++++-----------
.../zeppelin/notebook/repo/VFSNotebookRepo.java | 2 -
.../helium/HeliumApplicationFactoryTest.java | 2 +-
.../interpreter/AbstractInterpreterTest.java | 5 +
.../org/apache/zeppelin/notebook/NoteTest.java | 1 -
.../apache/zeppelin/notebook/NotebookTest.java | 12 +-
.../apache/zeppelin/notebook/ParagraphTest.java | 132 ++++---
.../notebook/repo/GitNotebookRepoTest.java | 11 +
16 files changed, 318 insertions(+), 334 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/afd2bca4/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
index aa5b0fb..e3fb004 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
@@ -1515,7 +1515,7 @@ public class NotebookServer extends WebSocketServlet
if (paragraph == null) {
throw new IllegalArgumentException("Unknown paragraph with id : " + paragraphId);
}
- return paragraph.getCurrentRepl().getInterpreterGroup();
+ return paragraph.getBindedInterpreter().getInterpreterGroup();
}
private void pushAngularObjectToRemoteRegistry(String noteId, String paragraphId, String varName,
@@ -1780,7 +1780,7 @@ public class NotebookServer extends WebSocketServlet
// if it's the last paragraph and not empty, let's add a new one
boolean isTheLastParagraph = note.isLastParagraph(p.getId());
if (!(Strings.isNullOrEmpty(p.getText()) ||
- p.getText().trim().equals(p.getMagic())) &&
+ Strings.isNullOrEmpty(p.getScriptText())) &&
isTheLastParagraph) {
Paragraph newPara = note.addNewParagraph(p.getAuthenticationInfo());
broadcastNewParagraph(note, newPara);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/afd2bca4/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
index 573ee56..a0cb587 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
@@ -208,7 +208,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
final InterpreterGroup mdGroup = new InterpreterGroup("mdGroup");
mdGroup.setAngularObjectRegistry(mdRegistry);
- when(paragraph.getCurrentRepl().getInterpreterGroup()).thenReturn(mdGroup);
+ when(paragraph.getBindedInterpreter().getInterpreterGroup()).thenReturn(mdGroup);
final AngularObject<String> ao1 = AngularObjectBuilder.build(varName, value, "noteId", "paragraphId");
@@ -256,7 +256,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
final InterpreterGroup mdGroup = new InterpreterGroup("mdGroup");
mdGroup.setAngularObjectRegistry(mdRegistry);
- when(paragraph.getCurrentRepl().getInterpreterGroup()).thenReturn(mdGroup);
+ when(paragraph.getBindedInterpreter().getInterpreterGroup()).thenReturn(mdGroup);
final AngularObject<String> ao1 = AngularObjectBuilder.build(varName, value, "noteId", "paragraphId");
@@ -302,7 +302,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
final InterpreterGroup mdGroup = new InterpreterGroup("mdGroup");
mdGroup.setAngularObjectRegistry(mdRegistry);
- when(paragraph.getCurrentRepl().getInterpreterGroup()).thenReturn(mdGroup);
+ when(paragraph.getBindedInterpreter().getInterpreterGroup()).thenReturn(mdGroup);
final AngularObject<String> ao1 = AngularObjectBuilder.build(varName, value, "noteId", "paragraphId");
when(mdRegistry.removeAndNotifyRemoteProcess(varName, "noteId", "paragraphId")).thenReturn(ao1);
@@ -347,7 +347,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
final InterpreterGroup mdGroup = new InterpreterGroup("mdGroup");
mdGroup.setAngularObjectRegistry(mdRegistry);
- when(paragraph.getCurrentRepl().getInterpreterGroup()).thenReturn(mdGroup);
+ when(paragraph.getBindedInterpreter().getInterpreterGroup()).thenReturn(mdGroup);
final AngularObject<String> ao1 = AngularObjectBuilder.build(varName, value, "noteId", "paragraphId");
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/afd2bca4/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
index e5f2e3b..0af15ab 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
@@ -20,7 +20,6 @@ import com.google.gson.Gson;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.ManagedInterpreterGroup;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
@@ -341,7 +340,7 @@ public class Helium {
public HeliumPackageSuggestion suggestApp(Paragraph paragraph) {
HeliumPackageSuggestion suggestion = new HeliumPackageSuggestion();
- Interpreter intp = paragraph.getCurrentRepl();
+ Interpreter intp = paragraph.getBindedInterpreter();
if (intp == null) {
return suggestion;
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/afd2bca4/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java
index f4dfae3..bdb3614 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java
@@ -78,7 +78,7 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
public void run() {
try {
// get interpreter process
- Interpreter intp = paragraph.getRepl(paragraph.getRequiredReplName());
+ Interpreter intp = paragraph.getBindedInterpreter();
ManagedInterpreterGroup intpGroup = (ManagedInterpreterGroup) intp.getInterpreterGroup();
RemoteInterpreterProcess intpProcess = intpGroup.getRemoteInterpreterProcess();
if (intpProcess == null) {
@@ -200,7 +200,7 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
"Can't unload application status " + appsToUnload.getStatus());
}
appStatusChange(paragraph, appsToUnload.getId(), ApplicationState.Status.UNLOADING);
- Interpreter intp = paragraph.getCurrentRepl();
+ Interpreter intp = paragraph.getBindedInterpreter();
if (intp == null) {
throw new ApplicationException("No interpreter found");
}
@@ -280,7 +280,7 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
"Can't run application status " + app.getStatus());
}
- Interpreter intp = paragraph.getCurrentRepl();
+ Interpreter intp = paragraph.getBindedInterpreter();
if (intp == null) {
throw new ApplicationException("No interpreter found");
}
@@ -417,7 +417,7 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
@Override
public void onUnbindInterpreter(Note note, InterpreterSetting setting) {
for (Paragraph p : note.getParagraphs()) {
- Interpreter currentInterpreter = p.getCurrentRepl();
+ Interpreter currentInterpreter = p.getBindedInterpreter();
List<InterpreterInfo> infos = setting.getInterpreterInfos();
for (InterpreterInfo info : infos) {
if (currentInterpreter != null &&
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/afd2bca4/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
index 5b88c12..0596cc5 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
@@ -841,4 +841,11 @@ public class InterpreterSetting {
}
throw new RuntimeException("Can not convert this type: " + properties.getClass());
}
+
+ public void waitForReady() throws InterruptedException {
+ while (getStatus().equals(
+ org.apache.zeppelin.interpreter.InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES)) {
+ Thread.sleep(200);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/afd2bca4/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java
index 2c1e631..219204f 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java
@@ -56,7 +56,7 @@ public class ManagedInterpreterGroup extends InterpreterGroup {
public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess() throws IOException {
if (remoteInterpreterProcess == null) {
- LOGGER.info("Create InterperterProcess for InterpreterGroup: " + getId());
+ LOGGER.info("Create InterpreterProcess for InterpreterGroup: " + getId());
remoteInterpreterProcess = interpreterSetting.createInterpreterProcess();
}
return remoteInterpreterProcess;
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/afd2bca4/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
index 1ab459e..a2e3e9b 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
@@ -162,7 +162,7 @@ public class RemoteInterpreter extends Interpreter {
private void internal_create() throws IOException {
synchronized (this) {
if (!isCreated) {
- RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
+ this.interpreterProcess = getOrCreateInterpreterProcess();
interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/afd2bca4/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
index 5124f5d..9fb0f0e 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
@@ -29,6 +29,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.gson.GsonBuilder;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.common.JsonSerializable;
+import org.apache.zeppelin.completer.CompletionType;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
@@ -224,7 +225,7 @@ public class Note implements ParagraphJobListener, JsonSerializable {
this.noteNameListener = listener;
}
- void setInterpreterFactory(InterpreterFactory factory) {
+ public void setInterpreterFactory(InterpreterFactory factory) {
this.factory = factory;
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
@@ -235,11 +236,6 @@ public class Note implements ParagraphJobListener, JsonSerializable {
void setInterpreterSettingManager(InterpreterSettingManager interpreterSettingManager) {
this.interpreterSettingManager = interpreterSettingManager;
- synchronized (paragraphs) {
- for (Paragraph p : paragraphs) {
- p.setInterpreterSettingManager(interpreterSettingManager);
- }
- }
}
public void initializeJobListenerForParagraph(Paragraph paragraph) {
@@ -305,8 +301,7 @@ public class Note implements ParagraphJobListener, JsonSerializable {
void addCloneParagraph(Paragraph srcParagraph) {
// Keep paragraph original ID
- final Paragraph newParagraph = new Paragraph(srcParagraph.getId(), this, this, factory,
- interpreterSettingManager);
+ final Paragraph newParagraph = new Paragraph(srcParagraph.getId(), this, this, factory);
Map<String, Object> config = new HashMap<>(srcParagraph.getConfig());
Map<String, Object> param = srcParagraph.settings.getParams();
@@ -349,7 +344,7 @@ public class Note implements ParagraphJobListener, JsonSerializable {
}
private Paragraph createParagraph(int index, AuthenticationInfo authenticationInfo) {
- Paragraph p = new Paragraph(this, this, factory, interpreterSettingManager);
+ Paragraph p = new Paragraph(this, this, factory);
p.setAuthenticationInfo(authenticationInfo);
setParagraphMagic(p, index);
return p;
@@ -570,14 +565,14 @@ public class Note implements ParagraphJobListener, JsonSerializable {
private void setParagraphMagic(Paragraph p, int index) {
if (paragraphs.size() > 0) {
- String magic;
+ String replName;
if (index == 0) {
- magic = paragraphs.get(0).getMagic();
+ replName = paragraphs.get(0).getIntpText();
} else {
- magic = paragraphs.get(index - 1).getMagic();
+ replName = paragraphs.get(index - 1).getIntpText();
}
- if (StringUtils.isNotEmpty(magic)) {
- p.setText(magic + "\n");
+ if (p.isValidInterpreter(replName) && StringUtils.isNotEmpty(replName)) {
+ p.setText("%" + replName + "\n");
}
}
}
@@ -620,44 +615,7 @@ public class Note implements ParagraphJobListener, JsonSerializable {
public boolean run(String paragraphId, boolean blocking) {
Paragraph p = getParagraph(paragraphId);
p.setListener(jobListenerFactory.getParagraphJobListener(this));
-
- if (p.isBlankParagraph()) {
- logger.info("skip to run blank paragraph. {}", p.getId());
- p.setStatus(Job.Status.FINISHED);
- return true;
- }
-
- p.clearRuntimeInfo(null);
- String requiredReplName = p.getRequiredReplName();
- Interpreter intp = factory.getInterpreter(p.getUser(), getId(), requiredReplName);
-
- if (intp == null) {
- String intpExceptionMsg =
- p.getJobName() + "'s Interpreter " + requiredReplName + " not found";
- RuntimeException intpException = new RuntimeException(intpExceptionMsg);
- InterpreterResult intpResult =
- new InterpreterResult(InterpreterResult.Code.ERROR, intpException.getMessage());
- p.setReturn(intpResult, intpException);
- p.setStatus(Job.Status.ERROR);
- throw intpException;
- }
- if (p.getConfig().get("enabled") == null || (Boolean) p.getConfig().get("enabled")) {
- p.setAuthenticationInfo(p.getAuthenticationInfo());
- intp.getScheduler().submit(p);
- }
-
- if (blocking) {
- while (!p.getStatus().isCompleted()) {
- try {
- Thread.sleep(100);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- return p.getStatus() == Status.FINISHED;
- } else {
- return true;
- }
+ return p.execute(blocking);
}
/**
@@ -690,6 +648,23 @@ public class Note implements ParagraphJobListener, JsonSerializable {
return p.completion(buffer, cursor);
}
+ public List<InterpreterCompletion> getInterpreterCompletion() {
+ List<InterpreterCompletion> completion = new LinkedList();
+ for (InterpreterSetting intp : interpreterSettingManager.getInterpreterSettings(getId())) {
+ List<InterpreterInfo> intInfo = intp.getInterpreterInfos();
+ if (intInfo.size() > 1) {
+ for (InterpreterInfo info : intInfo) {
+ String name = intp.getName() + "." + info.getName();
+ completion.add(new InterpreterCompletion(name, name, CompletionType.setting.name()));
+ }
+ } else {
+ completion.add(new InterpreterCompletion(intp.getName(), intp.getName(),
+ CompletionType.setting.name()));
+ }
+ }
+ return completion;
+ }
+
public List<Paragraph> getParagraphs() {
synchronized (paragraphs) {
return new LinkedList<>(paragraphs);
@@ -920,6 +895,10 @@ public class Note implements ParagraphJobListener, JsonSerializable {
this.noteEventListener = noteEventListener;
}
+ boolean hasInterpreterBinded() {
+ return !interpreterSettingManager.getInterpreterSettings(getId()).isEmpty();
+ }
+
public String toJson() {
return gson.toJson(this);
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/afd2bca4/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
index eaef494..e304e4a 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
@@ -28,10 +28,11 @@ import java.util.List;
import java.util.Map;
import java.security.SecureRandom;
import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.common.JsonSerializable;
-import org.apache.zeppelin.completer.CompletionType;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
@@ -43,7 +44,6 @@ import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterFactory;
-import org.apache.zeppelin.interpreter.InterpreterInfo;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import org.apache.zeppelin.interpreter.InterpreterOutputListener;
@@ -52,7 +52,7 @@ import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResultMessage;
import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput;
import org.apache.zeppelin.interpreter.InterpreterSetting;
-import org.apache.zeppelin.interpreter.InterpreterSettingManager;
+import org.apache.zeppelin.interpreter.ManagedInterpreterGroup;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.resource.ResourcePool;
import org.apache.zeppelin.scheduler.Job;
@@ -73,21 +73,24 @@ import com.google.common.collect.Maps;
*/
public class Paragraph extends Job implements Cloneable, JsonSerializable {
- private static final long serialVersionUID = -6328572073497992016L;
-
private static Logger logger = LoggerFactory.getLogger(Paragraph.class);
- private transient InterpreterFactory factory;
- private transient InterpreterSettingManager interpreterSettingManager;
+ private static Pattern REPL_PATTERN = Pattern.compile("(\\s*)%([\\w\\.]+).*", Pattern.DOTALL);
+
+ private transient InterpreterFactory interpreterFactory;
+ private transient Interpreter interpreter;
private transient Note note;
private transient AuthenticationInfo authenticationInfo;
private transient Map<String, Paragraph> userParagraphMap = Maps.newHashMap(); // personalized
- String title;
- String text;
- String user;
- Date dateUpdated;
- private Map<String, Object> config; // paragraph configs like isOpen, colWidth, etc
- public GUI settings; // form and parameter settings
+ private String title;
+ private String text; // text is composed of intpText and scriptText.
+ private transient String intpText;
+ private transient String scriptText;
+ private String user;
+ private Date dateUpdated;
+ // paragraph configs like isOpen, colWidth, etc
+ private Map<String, Object> config = new HashMap<>();
+ public GUI settings = new GUI(); // form and parameter settings
// since zeppelin-0.7.0, zeppelin stores multiple results of the paragraph
// see ZEPPELIN-212
@@ -105,37 +108,19 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
@VisibleForTesting
Paragraph() {
super(generateId(), null);
- config = new HashMap<>();
- settings = new GUI();
}
public Paragraph(String paragraphId, Note note, JobListener listener,
- InterpreterFactory factory, InterpreterSettingManager interpreterSettingManager) {
+ InterpreterFactory interpreterFactory) {
super(paragraphId, generateId(), listener);
this.note = note;
- this.factory = factory;
- this.interpreterSettingManager = interpreterSettingManager;
- title = null;
- text = null;
- authenticationInfo = null;
- user = null;
- dateUpdated = null;
- settings = new GUI();
- config = new HashMap<>();
- }
-
- public Paragraph(Note note, JobListener listener, InterpreterFactory factory,
- InterpreterSettingManager interpreterSettingManager) {
+ this.interpreterFactory = interpreterFactory;
+ }
+
+ public Paragraph(Note note, JobListener listener, InterpreterFactory interpreterFactory) {
super(generateId(), listener);
this.note = note;
- this.factory = factory;
- this.interpreterSettingManager = interpreterSettingManager;
- title = null;
- text = null;
- authenticationInfo = null;
- dateUpdated = null;
- settings = new GUI();
- config = new HashMap<>();
+ this.interpreterFactory = interpreterFactory;
}
private static String generateId() {
@@ -160,9 +145,14 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
public Paragraph cloneParagraphForUser(String user) {
Paragraph p = new Paragraph();
+ p.interpreterFactory = interpreterFactory;
+ p.note = note;
p.settings.setParams(Maps.newHashMap(settings.getParams()));
p.settings.setForms(Maps.newLinkedHashMap(settings.getForms()));
p.setConfig(Maps.newHashMap(config));
+ if (getAuthenticationInfo() != null) {
+ p.setAuthenticationInfo(getAuthenticationInfo());
+ }
p.setTitle(getTitle());
p.setText(getText());
p.setResult(getReturn());
@@ -189,8 +179,23 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
}
public void setText(String newText) {
+ // strip white space from the beginning
this.text = newText;
this.dateUpdated = new Date();
+ // parse text to get interpreter component
+ if (this.text != null) {
+ Matcher matcher = REPL_PATTERN.matcher(this.text);
+ if (matcher.matches()) {
+ String headingSpace = matcher.group(1);
+ this.intpText = matcher.group(2);
+ this.interpreter = interpreterFactory.getInterpreter(user, note.getId(), intpText);
+ this.scriptText = this.text.substring(headingSpace.length() + intpText.length() + 1).trim();
+ } else {
+ this.intpText = "";
+ this.interpreter = interpreterFactory.getInterpreter(user, note.getId(), "");
+ this.scriptText = this.text;
+ }
+ }
}
public AuthenticationInfo getAuthenticationInfo() {
@@ -210,6 +215,14 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
this.title = title;
}
+ public String getIntpText() {
+ return intpText;
+ }
+
+ public String getScriptText() {
+ return scriptText;
+ }
+
public void setNote(Note note) {
this.note = note;
}
@@ -223,100 +236,33 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
return enabled == null || enabled.booleanValue();
}
- public String getRequiredReplName() {
- return getRequiredReplName(text != null ? text.trim() : text);
+ public Interpreter getBindedInterpreter() {
+ return this.interpreterFactory.getInterpreter(user, note.getId(), intpText);
}
- public static String getRequiredReplName(String text) {
- if (text == null) {
- return null;
- }
-
- if (!text.startsWith("%")) {
- return null;
- }
-
- // get script head
- int scriptHeadIndex = 0;
- for (int i = 0; i < text.length(); i++) {
- char ch = text.charAt(i);
- if (Character.isWhitespace(ch) || ch == '(' || ch == '\n') {
- break;
- }
- scriptHeadIndex = i;
- }
- if (scriptHeadIndex < 1) {
- return null;
- }
- String head = text.substring(1, scriptHeadIndex + 1);
- return head;
- }
-
- public String getScriptBody() {
- return getScriptBody(text);
- }
-
- public static String getScriptBody(String text) {
- if (text == null) {
- return null;
- }
-
- String magic = getRequiredReplName(text);
- if (magic == null) {
- return text;
- }
-
- if (magic.length() + 1 >= text.length()) {
- return "";
- }
- return text.substring(magic.length() + 1).trim();
- }
-
- public Interpreter getRepl(String name) {
- return factory.getInterpreter(user, note.getId(), name);
- }
-
- public Interpreter getCurrentRepl() {
- return getRepl(getRequiredReplName());
- }
-
- public List<InterpreterCompletion> getInterpreterCompletion() {
- List<InterpreterCompletion> completion = new LinkedList();
- for (InterpreterSetting intp : interpreterSettingManager.getInterpreterSettings(note.getId())) {
- List<InterpreterInfo> intInfo = intp.getInterpreterInfos();
- if (intInfo.size() > 1) {
- for (InterpreterInfo info : intInfo) {
- String name = intp.getName() + "." + info.getName();
- completion.add(new InterpreterCompletion(name, name, CompletionType.setting.name()));
- }
- } else {
- completion.add(new InterpreterCompletion(intp.getName(), intp.getName(),
- CompletionType.setting.name()));
- }
- }
- return completion;
+ public void setInterpreter(Interpreter interpreter) {
+ this.interpreter = interpreter;
}
public List<InterpreterCompletion> completion(String buffer, int cursor) {
String lines[] = buffer.split(System.getProperty("line.separator"));
if (lines.length > 0 && lines[0].startsWith("%") && cursor <= lines[0].trim().length()) {
-
int idx = lines[0].indexOf(' ');
if (idx < 0 || (idx > 0 && cursor <= idx)) {
- return getInterpreterCompletion();
+ return note.getInterpreterCompletion();
}
}
String trimmedBuffer = buffer != null ? buffer.trim() : null;
cursor = calculateCursorPosition(buffer, trimmedBuffer, cursor);
- String replName = getRequiredReplName(trimmedBuffer);
-
- String body = getScriptBody(trimmedBuffer);
InterpreterContext interpreterContext = getInterpreterContextWithoutRunner(null);
try {
- Interpreter repl = getRepl(replName);
- return repl.completion(body, cursor, interpreterContext);
+ if (this.interpreter != null) {
+ return this.interpreter.completion(scriptText, cursor, interpreterContext);
+ } else {
+ return null;
+ }
} catch (InterpreterException e) {
throw new RuntimeException("Fail to get completion", e);
}
@@ -328,21 +274,25 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
cursor -= countWhitespacesAtStart;
}
- String replName = getRequiredReplName(trimmedBuffer);
- if (replName != null && cursor > replName.length()) {
- String body = trimmedBuffer.substring(replName.length() + 1);
- cursor -= replName.length() + 1 + body.indexOf(body.trim());
+ // parse text to get interpreter component
+ String repl = null;
+ if (trimmedBuffer != null) {
+ Matcher matcher = REPL_PATTERN.matcher(trimmedBuffer);
+ if (matcher.matches()) {
+ repl = matcher.group(2);
+ }
+ }
+
+ if (repl != null && cursor > repl.length()) {
+ String body = trimmedBuffer.substring(repl.length() + 1);
+ cursor -= repl.length() + 1 + body.indexOf(body.trim());
}
return cursor;
}
public void setInterpreterFactory(InterpreterFactory factory) {
- this.factory = factory;
- }
-
- public void setInterpreterSettingManager(InterpreterSettingManager interpreterSettingManager) {
- this.interpreterSettingManager = interpreterSettingManager;
+ this.interpreterFactory = factory;
}
public InterpreterResult getResult() {
@@ -360,14 +310,12 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
@Override
public int progress() {
- String replName = getRequiredReplName();
-
try {
- Interpreter repl = getRepl(replName);
- if (repl == null) {
+ if (this.interpreter != null) {
+ return this.interpreter.getProgress(getInterpreterContext(null));
+ } else {
return 0;
}
- return repl.getProgress(getInterpreterContext(null));
} catch (InterpreterException e) {
throw new RuntimeException("Fail to get progress", e);
}
@@ -388,31 +336,67 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
}
public boolean isBlankParagraph() {
- return Strings.isNullOrEmpty(getText()) || getText().trim().equals(getMagic());
+ return Strings.isNullOrEmpty(scriptText);
}
+ public boolean execute(boolean blocking) {
+ if (isBlankParagraph()) {
+ logger.info("skip to run blank paragraph. {}", getId());
+ setStatus(Job.Status.FINISHED);
+ return true;
+ }
+
+ clearRuntimeInfo(null);
+ this.interpreter = getBindedInterpreter();
+
+ if (interpreter == null) {
+ String intpExceptionMsg =
+ getJobName() + "'s Interpreter " + getIntpText() + " not found";
+ RuntimeException intpException = new RuntimeException(intpExceptionMsg);
+ InterpreterResult intpResult =
+ new InterpreterResult(InterpreterResult.Code.ERROR, intpException.getMessage());
+ setReturn(intpResult, intpException);
+ setStatus(Job.Status.ERROR);
+ throw intpException;
+ }
+ if (getConfig().get("enabled") == null || (Boolean) getConfig().get("enabled")) {
+ setAuthenticationInfo(getAuthenticationInfo());
+ interpreter.getScheduler().submit(this);
+ }
+
+ if (blocking) {
+ while (!getStatus().isCompleted()) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return getStatus() == Status.FINISHED;
+ } else {
+ return true;
+ }
+ }
@Override
protected Object jobRun() throws Throwable {
- String replName = getRequiredReplName();
- Interpreter repl = getRepl(replName);
- logger.info("run paragraph {} using {} " + repl, getId(), replName);
- if (repl == null) {
- logger.error("Can not find interpreter name " + repl);
- throw new RuntimeException("Can not find interpreter for " + getRequiredReplName());
- }
- //TODO(zjffdu) check interpreter setting status in interpreter setting itself
- InterpreterSetting intp = getInterpreterSettingById(repl.getInterpreterGroup().getId());
- while (intp.getStatus().equals(
- org.apache.zeppelin.interpreter.InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES)) {
- Thread.sleep(200);
- }
- if (this.noteHasUser() && this.noteHasInterpreters()) {
- if (intp != null && interpreterHasUser(intp)
- && isUserAuthorizedToAccessInterpreter(intp.getOption()) == false) {
- logger.error("{} has no permission for {} ", authenticationInfo.getUser(), repl);
+ logger.info("Run paragraph {} using {} ", getId(), intpText);
+ this.interpreter = getBindedInterpreter();
+ if (this.interpreter == null) {
+ logger.error("Can not find interpreter name " + intpText);
+ throw new RuntimeException("Can not find interpreter for " + intpText);
+ }
+ InterpreterSetting interpreterSetting = ((ManagedInterpreterGroup)
+ interpreter.getInterpreterGroup()).getInterpreterSetting();
+ if (interpreterSetting != null) {
+ interpreterSetting.waitForReady();
+ }
+ if (this.hasUser() && this.note.hasInterpreterBinded()) {
+ if (interpreterSetting != null && interpreterHasUser(interpreterSetting)
+ && isUserAuthorizedToAccessInterpreter(interpreterSetting.getOption()) == false) {
+ logger.error("{} has no permission for {} ", authenticationInfo.getUser(), intpText);
return new InterpreterResult(Code.ERROR,
- authenticationInfo.getUser() + " has no permission for " + getRequiredReplName());
+ authenticationInfo.getUser() + " has no permission for " + intpText);
}
}
@@ -420,20 +404,17 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
p.setText(getText());
}
- String script = getScriptBody();
// inject form
- if (repl.getFormType() == FormType.NATIVE) {
+ String script = this.scriptText;
+ if (interpreter.getFormType() == FormType.NATIVE) {
settings.clear();
- } else if (repl.getFormType() == FormType.SIMPLE) {
- String scriptBody = getScriptBody();
- // inputs will be built from script body
- LinkedHashMap<String, Input> inputs = Input.extractSimpleQueryForm(scriptBody);
-
+ } else if (interpreter.getFormType() == FormType.SIMPLE) {
+ // inputs will be built from script scriptText
+ LinkedHashMap<String, Input> inputs = Input.extractSimpleQueryForm(this.scriptText);
final AngularObjectRegistry angularRegistry =
- repl.getInterpreterGroup().getAngularObjectRegistry();
-
- scriptBody = extractVariablesFromAngularRegistry(scriptBody, inputs, angularRegistry);
-
+ interpreter.getInterpreterGroup().getAngularObjectRegistry();
+ String scriptBody = extractVariablesFromAngularRegistry(this.scriptText, inputs,
+ angularRegistry);
settings.setForms(inputs);
script = Input.getSimpleQuery(settings.getParams(), scriptBody);
}
@@ -441,7 +422,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
try {
InterpreterContext context = getInterpreterContext();
InterpreterContext.set(context);
- InterpreterResult ret = repl.interpret(script, context);
+ InterpreterResult ret = interpreter.interpret(script, context);
if (Code.KEEP_PREVIOUS_RESULT == ret.code()) {
return getReturn();
@@ -465,16 +446,13 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
}
}
- private boolean noteHasUser() {
+ private boolean hasUser() {
return this.user != null;
}
- private boolean noteHasInterpreters() {
- return !interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty();
- }
-
- private boolean interpreterHasUser(InterpreterSetting intp) {
- return intp.getOption().permissionIsSet() && intp.getOption().getOwners() != null;
+ private boolean interpreterHasUser(InterpreterSetting interpreterSetting) {
+ return interpreterSetting.getOption().permissionIsSet() &&
+ interpreterSetting.getOption().getOwners() != null;
}
private boolean isUserAuthorizedToAccessInterpreter(InterpreterOption intpOpt) {
@@ -482,24 +460,12 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
intpOpt.getOwners());
}
- private InterpreterSetting getInterpreterSettingById(String id) {
- InterpreterSetting setting = null;
- for (InterpreterSetting i : interpreterSettingManager.getInterpreterSettings(note.getId())) {
- if (id.startsWith(i.getId())) {
- setting = i;
- break;
- }
- }
- return setting;
- }
-
@Override
protected boolean jobAbort() {
- Interpreter repl = getRepl(getRequiredReplName());
- if (repl == null) {
+ if (interpreter == null) {
return true;
}
- Scheduler scheduler = repl.getScheduler();
+ Scheduler scheduler = interpreter.getScheduler();
if (scheduler == null) {
return true;
}
@@ -509,7 +475,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
job.setStatus(Status.ABORT);
} else {
try {
- repl.cancel(getInterpreterContextWithoutRunner(null));
+ interpreter.cancel(getInterpreterContextWithoutRunner(null));
} catch (InterpreterException e) {
throw new RuntimeException(e);
}
@@ -560,13 +526,9 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
AngularObjectRegistry registry = null;
ResourcePool resourcePool = null;
- if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) {
- InterpreterSetting intpGroup =
- interpreterSettingManager.getInterpreterSettings(note.getId()).get(0);
- registry = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
- .getAngularObjectRegistry();
- resourcePool = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
- .getResourcePool();
+ if (this.interpreter != null) {
+ registry = this.interpreter.getInterpreterGroup().getAngularObjectRegistry();
+ resourcePool = this.interpreter.getInterpreterGroup().getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<>();
@@ -583,7 +545,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
}
InterpreterContext interpreterContext =
- new InterpreterContext(note.getId(), getId(), getRequiredReplName(), this.getTitle(),
+ new InterpreterContext(note.getId(), getId(), intpText, this.getTitle(),
this.getText(), this.getAuthenticationInfo(), this.getConfig(), this.settings, registry,
resourcePool, runners, output);
return interpreterContext;
@@ -593,13 +555,9 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
AngularObjectRegistry registry = null;
ResourcePool resourcePool = null;
- if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) {
- InterpreterSetting intpGroup =
- interpreterSettingManager.getInterpreterSettings(note.getId()).get(0);
- registry = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
- .getAngularObjectRegistry();
- resourcePool = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
- .getResourcePool();
+ if (this.interpreter != null) {
+ registry = this.interpreter.getInterpreterGroup().getAngularObjectRegistry();
+ resourcePool = this.interpreter.getInterpreterGroup().getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<>();
@@ -617,7 +575,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
}
InterpreterContext interpreterContext =
- new InterpreterContext(note.getId(), getId(), getRequiredReplName(), this.getTitle(),
+ new InterpreterContext(note.getId(), getId(), intpText, this.getTitle(),
this.getText(), this.getAuthenticationInfo(), this.getConfig(), this.settings, registry,
resourcePool, runners, output);
return interpreterContext;
@@ -728,22 +686,8 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
return scriptBody;
}
- public String getMagic() {
- String magic = StringUtils.EMPTY;
- String text = getText();
- if (text != null && text.startsWith("%")) {
- magic = text.split("\\s+")[0];
- if (isValidInterpreter(magic.substring(1))) {
- return magic;
- } else {
- return StringUtils.EMPTY;
- }
- }
- return magic;
- }
-
- private boolean isValidInterpreter(String replName) {
- return factory.getInterpreter(user, note.getId(), replName) != null;
+ public boolean isValidInterpreter(String replName) {
+ return interpreterFactory.getInterpreter(user, note.getId(), replName) != null;
}
public void updateRuntimeInfos(String label, String tooltip, Map<String, String> infos,
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/afd2bca4/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
index 3fe5dab..63395f9 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
@@ -168,8 +168,6 @@ public class VFSNotebookRepo implements NotebookRepo {
ins.close();
Note note = Note.fromJson(json);
-// note.setReplLoader(replLoader);
-// note.jobListenerFactory = jobListenerFactory;
for (Paragraph p : note.getParagraphs()) {
if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/afd2bca4/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
index f23d433..0952de6 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
@@ -229,7 +229,7 @@ public class HeliumApplicationFactoryTest extends AbstractInterpreterTest implem
p1.setText("%fake ");
// make sure that p1's repl is null
- Interpreter intp = p1.getCurrentRepl();
+ Interpreter intp = p1.getBindedInterpreter();
assertEquals(intp, null);
// Unbind all interpreter from note
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/afd2bca4/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java
index 5438e93..9df402d 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java
@@ -5,6 +5,7 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
+import org.apache.zeppelin.notebook.Note;
import org.junit.After;
import org.junit.Before;
import org.slf4j.Logger;
@@ -69,4 +70,8 @@ public abstract class AbstractInterpreterTest {
FileUtils.deleteDirectory(confDir);
FileUtils.deleteDirectory(notebookDir);
}
+
+ protected Note createNote() {
+ return new Note(null, interpreterFactory, interpreterSettingManager, null, null, null, null);
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/afd2bca4/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java
index 69c1f86..73046ff 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java
@@ -247,7 +247,6 @@ public class NoteTest {
note.getInfo().put("info_1", "value_1");
String pText = "%spark sc.version";
Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
- p.dateUpdated = new Date();
p.setText(pText);
p.setResult("1.6.2");
p.settings.getForms().put("textbox_1", new TextBox("name", "default_name"));
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/afd2bca4/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
index af5971a..77eb4e5 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
@@ -144,7 +144,7 @@ public class NotebookTest extends AbstractInterpreterTest implements JobListener
// then interpreter factory should be injected into all the paragraphs
Note note = notebook.getAllNotes().get(0);
- assertNull(note.getParagraphs().get(0).getRepl(null));
+ assertNull(note.getParagraphs().get(0).getBindedInterpreter());
}
@Test
@@ -184,8 +184,8 @@ public class NotebookTest extends AbstractInterpreterTest implements JobListener
copiedNote.getParagraphs().get(0).getText());
assertEquals(notes.get(1).getParagraphs().get(0).settings,
copiedNote.getParagraphs().get(0).settings);
- assertEquals(notes.get(1).getParagraphs().get(0).title,
- copiedNote.getParagraphs().get(0).title);
+ assertEquals(notes.get(1).getParagraphs().get(0).getTitle(),
+ copiedNote.getParagraphs().get(0).getTitle());
// delete the notebook
for (String note : noteNames) {
@@ -470,7 +470,7 @@ public class NotebookTest extends AbstractInterpreterTest implements JobListener
// Test
assertEquals(p.getId(), p2.getId());
- assertEquals(p.text, p2.text);
+ assertEquals(p.getText(), p2.getText());
assertEquals(p.getResult().message().get(0).getData(), p2.getResult().message().get(0).getData());
// Verify import note with subject
@@ -503,7 +503,7 @@ public class NotebookTest extends AbstractInterpreterTest implements JobListener
// Keep same ParagraphId
assertEquals(cp.getId(), p.getId());
- assertEquals(cp.text, p.text);
+ assertEquals(cp.getText(), p.getText());
assertEquals(cp.getResult().message().get(0).getData(), p.getResult().message().get(0).getData());
// Verify clone note with subject
@@ -549,7 +549,7 @@ public class NotebookTest extends AbstractInterpreterTest implements JobListener
// Keep same ParagraphId
assertEquals(cp.getId(), p.getId());
- assertEquals(cp.text, p.text);
+ assertEquals(cp.getText(), p.getText());
assertNull(cp.getResult());
notebook.removeNote(note.getId(), anonymous);
notebook.removeNote(cloneNote.getId(), anonymous);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/afd2bca4/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
index 8a4bea0..9e9ce27 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
@@ -43,7 +43,6 @@ import org.apache.zeppelin.display.Input;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.Interpreter.FormType;
import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
@@ -58,55 +57,104 @@ import java.util.HashMap;
import java.util.Map;
import org.mockito.Mockito;
-public class ParagraphTest {
+public class ParagraphTest extends AbstractInterpreterTest {
+
@Test
public void scriptBodyWithReplName() {
- String text = "%spark(1234567";
- assertEquals("(1234567", Paragraph.getScriptBody(text));
-
- text = "%table 1234567";
- assertEquals("1234567", Paragraph.getScriptBody(text));
+ Note note = createNote();
+ Paragraph paragraph = new Paragraph(note, null, interpreterFactory);
+ paragraph.setText("%test(1234567");
+ assertEquals("test", paragraph.getIntpText());
+ assertEquals("(1234567", paragraph.getScriptText());
+
+ paragraph.setText("%test 1234567");
+ assertEquals("test", paragraph.getIntpText());
+ assertEquals("1234567", paragraph.getScriptText());
}
@Test
public void scriptBodyWithoutReplName() {
- String text = "12345678";
- assertEquals(text, Paragraph.getScriptBody(text));
+ Note note = createNote();
+ Paragraph paragraph = new Paragraph(note, null, interpreterFactory);
+ paragraph.setText("1234567");
+ assertEquals("", paragraph.getIntpText());
+ assertEquals("1234567", paragraph.getScriptText());
}
@Test
public void replNameAndNoBody() {
- String text = "%md";
- assertEquals("md", Paragraph.getRequiredReplName(text));
- assertEquals("", Paragraph.getScriptBody(text));
+ Note note = createNote();
+ Paragraph paragraph = new Paragraph(note, null, interpreterFactory);
+ paragraph.setText("%test");
+ assertEquals("test", paragraph.getIntpText());
+ assertEquals("", paragraph.getScriptText());
}
-
+
@Test
public void replSingleCharName() {
- String text = "%r a";
- assertEquals("r", Paragraph.getRequiredReplName(text));
- assertEquals("a", Paragraph.getScriptBody(text));
+ Note note = createNote();
+ Paragraph paragraph = new Paragraph(note, null, interpreterFactory);
+ paragraph.setText("%r a");
+ assertEquals("r", paragraph.getIntpText());
+ assertEquals("a", paragraph.getScriptText());
}
@Test
- public void replNameEndsWithWhitespace() {
- String text = "%md\r\n###Hello";
- assertEquals("md", Paragraph.getRequiredReplName(text));
-
- text = "%md\t###Hello";
- assertEquals("md", Paragraph.getRequiredReplName(text));
-
- text = "%md\u000b###Hello";
- assertEquals("md", Paragraph.getRequiredReplName(text));
-
- text = "%md\f###Hello";
- assertEquals("md", Paragraph.getRequiredReplName(text));
-
- text = "%md\n###Hello";
- assertEquals("md", Paragraph.getRequiredReplName(text));
+ public void replInvalid() {
+ Note note = createNote();
+ Paragraph paragraph = new Paragraph(note, null, interpreterFactory);
+ paragraph.setText("foo %r");
+ assertEquals("", paragraph.getIntpText());
+ assertEquals("foo %r", paragraph.getScriptText());
+
+ paragraph.setText("foo%r");
+ assertEquals("", paragraph.getIntpText());
+ assertEquals("foo%r", paragraph.getScriptText());
+
+ paragraph.setText("% foo");
+ assertEquals("", paragraph.getIntpText());
+ assertEquals("% foo", paragraph.getScriptText());
+ }
- text = "%md ###Hello";
- assertEquals("md", Paragraph.getRequiredReplName(text));
+ @Test
+ public void replNameEndsWithWhitespace() {
+ Note note = createNote();
+ Paragraph paragraph = new Paragraph(note, null, interpreterFactory);
+ paragraph.setText("%test\r\n###Hello");
+ assertEquals("test", paragraph.getIntpText());
+ assertEquals("###Hello", paragraph.getScriptText());
+
+ paragraph.setText("%test\t###Hello");
+ assertEquals("test", paragraph.getIntpText());
+ assertEquals("###Hello", paragraph.getScriptText());
+
+ paragraph.setText("%test\u000b###Hello");
+ assertEquals("test", paragraph.getIntpText());
+ assertEquals("###Hello", paragraph.getScriptText());
+
+ paragraph.setText("%test\f###Hello");
+ assertEquals("test", paragraph.getIntpText());
+ assertEquals("###Hello", paragraph.getScriptText());
+
+ paragraph.setText("%test\n###Hello");
+ assertEquals("test", paragraph.getIntpText());
+ assertEquals("###Hello", paragraph.getScriptText());
+
+ paragraph.setText("%test ###Hello");
+ assertEquals("test", paragraph.getIntpText());
+ assertEquals("###Hello", paragraph.getScriptText());
+
+ paragraph.setText(" %test ###Hello");
+ assertEquals("test", paragraph.getIntpText());
+ assertEquals("###Hello", paragraph.getScriptText());
+
+ paragraph.setText("\n\r%test ###Hello");
+ assertEquals("test", paragraph.getIntpText());
+ assertEquals("###Hello", paragraph.getScriptText());
+
+ paragraph.setText("%\r\n###Hello");
+ assertEquals("", paragraph.getIntpText());
+ assertEquals("%\r\n###Hello", paragraph.getScriptText());
}
@Test
@@ -124,7 +172,7 @@ public class ParagraphTest {
final String scriptBody = "My name is ${name} and I am ${age=20} years old. " +
"My occupation is ${ job = engineer | developer | artists}";
- final Paragraph paragraph = new Paragraph(note, null, null, null);
+ final Paragraph paragraph = new Paragraph(note, null, null);
final String paragraphId = paragraph.getId();
final AngularObject nameAO = AngularObjectBuilder.build("name", "DuyHai DOAN", noteId,
@@ -150,7 +198,7 @@ public class ParagraphTest {
@Test
public void returnDefaultParagraphWithNewUser() {
- Paragraph p = new Paragraph("para_1", null, null, null, null);
+ Paragraph p = new Paragraph("para_1", null, null, null);
Object defaultValue = "Default Value";
p.setResult(defaultValue);
Paragraph newUserParagraph = p.getUserParagraph("new_user");
@@ -160,16 +208,13 @@ public class ParagraphTest {
@Test
public void returnUnchangedResultsWithDifferentUser() throws Throwable {
- InterpreterSettingManager mockInterpreterSettingManager = mock(InterpreterSettingManager.class);
Note mockNote = mock(Note.class);
when(mockNote.getCredentials()).thenReturn(mock(Credentials.class));
- Paragraph spyParagraph = spy(new Paragraph("para_1", mockNote, null, null, mockInterpreterSettingManager));
-
- doReturn("spy").when(spyParagraph).getRequiredReplName();
-
+ Paragraph spyParagraph = spy(new Paragraph("para_1", mockNote, null, null));
Interpreter mockInterpreter = mock(Interpreter.class);
- doReturn(mockInterpreter).when(spyParagraph).getRepl(anyString());
+ spyParagraph.setInterpreter(mockInterpreter);
+ doReturn(mockInterpreter).when(spyParagraph).getBindedInterpreter();
ManagedInterpreterGroup mockInterpreterGroup = mock(ManagedInterpreterGroup.class);
when(mockInterpreter.getInterpreterGroup()).thenReturn(mockInterpreterGroup);
@@ -187,9 +232,6 @@ public class ParagraphTest {
when(mockInterpreterSetting.getOrCreateInterpreterGroup(anyString(), anyString())).thenReturn(mockInterpreterGroup);
spyInterpreterSettingList.add(mockInterpreterSetting);
when(mockNote.getId()).thenReturn("any_id");
- when(mockInterpreterSettingManager.getInterpreterSettings(anyString())).thenReturn(spyInterpreterSettingList);
-
- doReturn("spy script body").when(spyParagraph).getScriptBody();
when(mockInterpreter.getFormType()).thenReturn(FormType.NONE);
@@ -229,7 +271,7 @@ public class ParagraphTest {
@Test
public void testCursorPosition() {
Paragraph paragraph = spy(new Paragraph());
- doReturn(null).when(paragraph).getRepl(anyString());
+ doReturn(null).when(paragraph).getIntpText();
// left = buffer, middle = cursor position into source code, right = cursor position after parse
List<Triple<String, Integer, Integer>> dataSet = Arrays.asList(
Triple.of("%jdbc schema.", 13, 7),
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/afd2bca4/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java
index 954636c..2276c25 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java
@@ -18,6 +18,7 @@
package org.apache.zeppelin.notebook.repo;
import static com.google.common.truth.Truth.assertThat;
+import static org.mockito.Mockito.mock;
import java.io.File;
import java.io.IOException;
@@ -28,6 +29,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NoteInfo;
import org.apache.zeppelin.notebook.Paragraph;
@@ -141,6 +143,7 @@ public class GitNotebookRepoTest {
//modify, save and checkpoint first note
Note note = notebookRepo.get(TEST_NOTE_ID, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map<String, Object> config = p.getConfig();
config.put("enabled", true);
@@ -156,6 +159,7 @@ public class GitNotebookRepoTest {
//modify, save and checkpoint second note
note = notebookRepo.get(TEST_NOTE_ID2, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
config = p.getConfig();
config.put("enabled", false);
@@ -182,6 +186,7 @@ public class GitNotebookRepoTest {
// add changes to note
Note note = notebookRepo.get(TEST_NOTE_ID, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map<String, Object> config = p.getConfig();
config.put("enabled", true);
@@ -221,6 +226,7 @@ public class GitNotebookRepoTest {
// add paragraph and save
Note note = notebookRepo.get(TEST_NOTE_ID, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map<String, Object> config = p1.getConfig();
config.put("enabled", true);
@@ -240,6 +246,7 @@ public class GitNotebookRepoTest {
// get current note
note = notebookRepo.get(TEST_NOTE_ID, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
assertThat(note.getParagraphs().size()).isEqualTo(paragraphCount_2);
// add one more paragraph and save
@@ -249,6 +256,7 @@ public class GitNotebookRepoTest {
p2.setText("get revision when modified note test text");
notebookRepo.save(note, null);
note = notebookRepo.get(TEST_NOTE_ID, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
int paragraphCount_3 = note.getParagraphs().size();
assertThat(paragraphCount_3).isEqualTo(paragraphCount_2 + 1);
@@ -276,6 +284,7 @@ public class GitNotebookRepoTest {
// get current note
Note note = notebookRepo.get(TEST_NOTE_ID, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
assertThat(note.getParagraphs().size()).isEqualTo(paragraphCount_1);
// add one more paragraph and save
@@ -293,6 +302,7 @@ public class GitNotebookRepoTest {
// get current note
note = notebookRepo.get(TEST_NOTE_ID, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
assertThat(note.getParagraphs().size()).isEqualTo(paragraphCount_2);
// test for absent revision
@@ -311,6 +321,7 @@ public class GitNotebookRepoTest {
// get current note
Note note = notebookRepo.get(TEST_NOTE_ID, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
int paragraphCount_1 = note.getParagraphs().size();
LOG.info("initial paragraph count: {}", paragraphCount_1);