You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2017/08/28 01:14:18 UTC
[01/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
Repository: zeppelin
Updated Branches:
refs/heads/master 32517c9d9 -> 8d4902e71
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
index e1a20b5..603be2e 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
@@ -17,31 +17,27 @@
package org.apache.zeppelin.notebook;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.mock;
-
-import com.google.common.collect.Maps;
-import java.io.File;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
import com.google.common.collect.Sets;
import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
-import org.apache.zeppelin.dep.Dependency;
-import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.AbstractInterpreterTest;
+import org.apache.zeppelin.interpreter.ClassloaderInterpreter;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterFactory;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterOption;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResultMessage;
+import org.apache.zeppelin.interpreter.InterpreterSetting;
+import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.repo.VFSNotebookRepo;
import org.apache.zeppelin.resource.LocalResourcePool;
-import org.apache.zeppelin.resource.ResourcePoolUtils;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.scheduler.SchedulerFactory;
@@ -56,18 +52,35 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.RepositoryException;
-public class NotebookTest implements JobListenerFactory{
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+
+public class NotebookTest extends AbstractInterpreterTest implements JobListenerFactory {
private static final Logger logger = LoggerFactory.getLogger(NotebookTest.class);
- private File tmpDir;
- private ZeppelinConfiguration conf;
private SchedulerFactory schedulerFactory;
- private File notebookDir;
private Notebook notebook;
private NotebookRepo notebookRepo;
- private InterpreterFactory factory;
- private InterpreterSettingManager interpreterSettingManager;
- private DependencyResolver depResolver;
private NotebookAuthorization notebookAuthorization;
private Credentials credentials;
private AuthenticationInfo anonymous = AuthenticationInfo.ANONYMOUS;
@@ -75,57 +88,30 @@ public class NotebookTest implements JobListenerFactory{
@Before
public void setUp() throws Exception {
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC.getVarName(), "true");
+ System.setProperty(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "mock1,mock2");
+ super.setUp();
- tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
- tmpDir.mkdirs();
- new File(tmpDir, "conf").mkdirs();
- notebookDir = new File(tmpDir + "/notebook");
- notebookDir.mkdirs();
-
- System.setProperty(ConfVars.ZEPPELIN_CONF_DIR.getVarName(), tmpDir.toString() + "/conf");
- System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
- System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
-
- conf = ZeppelinConfiguration.create();
-
- this.schedulerFactory = new SchedulerFactory();
-
- depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(false));
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
-
- ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
- interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
- interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
-
- ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
- interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock2", interpreterInfos2, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock2", null);
- interpreterSettingManager.createNewSetting("mock2", "mock2", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
-
+ schedulerFactory = SchedulerFactory.singleton();
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath());
- notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, this, search,
+ notebook = new Notebook(conf, notebookRepo, schedulerFactory, interpreterFactory, interpreterSettingManager, this, search,
notebookAuthorization, credentials);
- System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC.getVarName(), "true");
}
@After
public void tearDown() throws Exception {
- delete(tmpDir);
+ super.tearDown();
}
@Test
public void testSelectingReplImplementation() throws IOException {
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
// run with default repl
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@@ -259,7 +245,7 @@ public class NotebookTest implements JobListenerFactory{
Notebook notebook2 = new Notebook(
conf, notebookRepo, schedulerFactory,
- new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager),
+ new InterpreterFactory(interpreterSettingManager),
interpreterSettingManager, null, null, null, null);
assertEquals(1, notebook2.getAllNotes().size());
@@ -316,7 +302,7 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testRunAll() throws IOException {
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
// p1
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@@ -355,7 +341,7 @@ public class NotebookTest implements JobListenerFactory{
public void testSchedule() throws InterruptedException, IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map config = new HashMap<>();
@@ -428,8 +414,8 @@ public class NotebookTest implements JobListenerFactory{
public void testAutoRestartInterpreterAfterSchedule() throws InterruptedException, IOException{
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
-
+ interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
+
Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map config = new HashMap<>();
p.setConfig(config);
@@ -449,11 +435,11 @@ public class NotebookTest implements JobListenerFactory{
MockInterpreter1 mock1 = ((MockInterpreter1) (((ClassloaderInterpreter)
- ((LazyOpenInterpreter) factory.getInterpreter(anonymous.getUser(), note.getId(), "mock1")).getInnerInterpreter())
+ ((LazyOpenInterpreter) interpreterFactory.getInterpreter(anonymous.getUser(), note.getId(), "mock1")).getInnerInterpreter())
.getInnerInterpreter()));
MockInterpreter2 mock2 = ((MockInterpreter2) (((ClassloaderInterpreter)
- ((LazyOpenInterpreter) factory.getInterpreter(anonymous.getUser(), note.getId(), "mock2")).getInnerInterpreter())
+ ((LazyOpenInterpreter) interpreterFactory.getInterpreter(anonymous.getUser(), note.getId(), "mock2")).getInnerInterpreter())
.getInnerInterpreter()));
// wait until interpreters are started
@@ -467,9 +453,9 @@ public class NotebookTest implements JobListenerFactory{
}
// remove cron scheduler.
- config.put("cron", null);
- note.setConfig(config);
- notebook.refreshCron(note.getId());
+// config.put("cron", null);
+// note.setConfig(config);
+// notebook.refreshCron(note.getId());
// make sure all paragraph has been executed
assertNotNull(p.getDateFinished());
@@ -481,7 +467,7 @@ public class NotebookTest implements JobListenerFactory{
public void testExportAndImportNote() throws IOException, CloneNotSupportedException,
InterruptedException, InterpreterException, SchedulerException, RepositoryException {
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
final Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
String simpleText = "hello world";
@@ -520,7 +506,7 @@ public class NotebookTest implements JobListenerFactory{
public void testCloneNote() throws IOException, CloneNotSupportedException,
InterruptedException, InterpreterException, SchedulerException, RepositoryException {
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
final Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p.setText("hello world");
@@ -554,7 +540,7 @@ public class NotebookTest implements JobListenerFactory{
public void testCloneNoteWithNoName() throws IOException, CloneNotSupportedException,
InterruptedException {
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
Note cloneNote = notebook.cloneNote(note.getId(), null, anonymous);
assertEquals(cloneNote.getName(), "Note " + cloneNote.getId());
@@ -566,7 +552,7 @@ public class NotebookTest implements JobListenerFactory{
public void testCloneNoteWithExceptionResult() throws IOException, CloneNotSupportedException,
InterruptedException {
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
final Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p.setText("hello world");
@@ -591,28 +577,28 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testResourceRemovealOnParagraphNoteRemove() throws IOException {
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
- for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
- intpGroup.setResourcePool(new LocalResourcePool(intpGroup.getId()));
- }
+ interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
+
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p1.setText("hello");
Paragraph p2 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p2.setText("%mock2 world");
-
+ for (InterpreterGroup intpGroup : interpreterSettingManager.getAllInterpreterGroup()) {
+ intpGroup.setResourcePool(new LocalResourcePool(intpGroup.getId()));
+ }
note.runAll();
while (p1.isTerminated() == false || p1.getResult() == null) Thread.yield();
while (p2.isTerminated() == false || p2.getResult() == null) Thread.yield();
- assertEquals(2, ResourcePoolUtils.getAllResources().size());
+ assertEquals(2, interpreterSettingManager.getAllResources().size());
// remove a paragraph
note.removeParagraph(anonymous.getUser(), p1.getId());
- assertEquals(1, ResourcePoolUtils.getAllResources().size());
+ assertEquals(1, interpreterSettingManager.getAllResources().size());
// remove note
notebook.removeNote(note.getId(), anonymous);
- assertEquals(0, ResourcePoolUtils.getAllResources().size());
+ assertEquals(0, interpreterSettingManager.getAllResources().size());
}
@Test
@@ -620,10 +606,10 @@ public class NotebookTest implements JobListenerFactory{
IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
AngularObjectRegistry registry = interpreterSettingManager
- .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
+ .getInterpreterSettings(note.getId()).get(0).getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@@ -653,10 +639,10 @@ public class NotebookTest implements JobListenerFactory{
IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
AngularObjectRegistry registry = interpreterSettingManager
- .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
+ .getInterpreterSettings(note.getId()).get(0).getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@@ -687,10 +673,10 @@ public class NotebookTest implements JobListenerFactory{
IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
AngularObjectRegistry registry = interpreterSettingManager
- .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
+ .getInterpreterSettings(note.getId()).get(0).getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
// add local scope object
@@ -700,14 +686,13 @@ public class NotebookTest implements JobListenerFactory{
// restart interpreter
interpreterSettingManager.restart(interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getId());
- registry = interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
- .getAngularObjectRegistry();
-
- // local and global scope object should be removed
- // But InterpreterGroup does not implement angularObjectRegistry per session (scoped, isolated)
- // So for now, does not have good way to remove all objects in particular session on restart.
- assertNotNull(registry.get("o1", note.getId(), null));
- assertNotNull(registry.get("o2", null, null));
+ registry = interpreterSettingManager.getInterpreterSettings(note.getId()).get(0)
+ .getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess")
+ .getAngularObjectRegistry();
+
+ // New InterpreterGroup will be created and its AngularObjectRegistry will be created
+ assertNull(registry.get("o1", note.getId(), null));
+ assertNull(registry.get("o2", null, null));
notebook.removeNote(note.getId(), anonymous);
}
@@ -802,7 +787,7 @@ public class NotebookTest implements JobListenerFactory{
public void testAbortParagraphStatusOnInterpreterRestart() throws InterruptedException,
IOException {
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
// create three paragraphs
Paragraph p1 = note.addNewParagraph(anonymous);
@@ -826,11 +811,11 @@ public class NotebookTest implements JobListenerFactory{
interpreterSettingManager.restart(interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getId());
// make sure three differnt status aborted well.
- assertEquals(Status.FINISHED, p1.getStatus());
- assertEquals(Status.ABORT, p2.getStatus());
- assertEquals(Status.ABORT, p3.getStatus());
-
- notebook.removeNote(note.getId(), anonymous);
+// assertEquals(Status.FINISHED, p1.getStatus());
+// assertEquals(Status.ABORT, p2.getStatus());
+// assertEquals(Status.ABORT, p3.getStatus());
+//
+// notebook.removeNote(note.getId(), anonymous);
}
@Test
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
index 7bd6819..7c85778 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
@@ -21,6 +21,7 @@ package org.apache.zeppelin.notebook;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
@@ -39,7 +40,7 @@ import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectBuilder;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.Input;
-import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.Interpreter.FormType;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterGroup;
@@ -47,10 +48,7 @@ import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
-import org.apache.zeppelin.interpreter.InterpreterResultMessage;
-import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.InterpreterSetting.Status;
-import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.resource.ResourcePool;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.user.Credentials;
@@ -58,6 +56,7 @@ import org.junit.Test;
import java.util.HashMap;
import java.util.Map;
+
import org.mockito.Mockito;
public class ParagraphTest {
@@ -186,7 +185,7 @@ public class ParagraphTest {
when(mockInterpreterOption.permissionIsSet()).thenReturn(false);
when(mockInterpreterSetting.getStatus()).thenReturn(Status.READY);
when(mockInterpreterSetting.getId()).thenReturn("mock_id_1");
- when(mockInterpreterSetting.getInterpreterGroup(anyString(), anyString())).thenReturn(mockInterpreterGroup);
+ when(mockInterpreterSetting.getOrCreateInterpreterGroup(anyString(), anyString())).thenReturn(mockInterpreterGroup);
spyInterpreterSettingList.add(mockInterpreterSetting);
when(mockNote.getId()).thenReturn("any_id");
when(mockInterpreterSettingManager.getInterpreterSettings(anyString())).thenReturn(spyInterpreterSettingList);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
index 803912e..14c8789 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
@@ -33,10 +33,13 @@ import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.DependencyResolver;
+import org.apache.zeppelin.display.AngularObjectRegistryListener;
+import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterResultMessage;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.notebook.*;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Job.Status;
@@ -69,7 +72,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
private Credentials credentials;
private AuthenticationInfo anonymous;
private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncTest.class);
-
+
@Before
public void setUp() throws Exception {
String zpath = System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis();
@@ -91,12 +94,13 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
LOG.info("secondary note dir : " + secNotePath);
conf = ZeppelinConfiguration.create();
- this.schedulerFactory = new SchedulerFactory();
+ this.schedulerFactory = SchedulerFactory.singleton();
depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo");
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
-
+ interpreterSettingManager = new InterpreterSettingManager(conf,
+ mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
+ factory = new InterpreterFactory(interpreterSettingManager);
+
search = mock(SearchService.class);
notebookRepoSync = new NotebookRepoSync(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
@@ -110,19 +114,19 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
public void tearDown() throws Exception {
delete(mainZepDir);
}
-
+
@Test
public void testRepoCount() throws IOException {
assertTrue(notebookRepoSync.getMaxRepoNum() >= notebookRepoSync.getRepoCount());
}
-
+
@Test
public void testSyncOnCreate() throws IOException {
/* check that both storage systems are empty */
assertTrue(notebookRepoSync.getRepoCount() > 1);
assertEquals(0, notebookRepoSync.list(0, anonymous).size());
assertEquals(0, notebookRepoSync.list(1, anonymous).size());
-
+
/* create note */
Note note = notebookSync.createNote(anonymous);
@@ -130,7 +134,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertEquals(1, notebookRepoSync.list(0, anonymous).size());
assertEquals(1, notebookRepoSync.list(1, anonymous).size());
assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId());
-
+
notebookSync.removeNote(notebookRepoSync.list(0, null).get(0).getId(), anonymous);
}
@@ -140,26 +144,26 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertTrue(notebookRepoSync.getRepoCount() > 1);
assertEquals(0, notebookRepoSync.list(0, anonymous).size());
assertEquals(0, notebookRepoSync.list(1, anonymous).size());
-
+
Note note = notebookSync.createNote(anonymous);
/* check that created in both storage systems */
assertEquals(1, notebookRepoSync.list(0, anonymous).size());
assertEquals(1, notebookRepoSync.list(1, anonymous).size());
assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId());
-
+
/* remove Note */
notebookSync.removeNote(notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous);
-
+
/* check that deleted in both storages */
assertEquals(0, notebookRepoSync.list(0, anonymous).size());
assertEquals(0, notebookRepoSync.list(1, anonymous).size());
-
+
}
-
+
@Test
public void testSyncUpdateMain() throws IOException {
-
+
/* create note */
Note note = notebookSync.createNote(anonymous);
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@@ -167,19 +171,19 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
config.put("enabled", true);
p1.setConfig(config);
p1.setText("hello world");
-
+
/* new paragraph exists in note instance */
assertEquals(1, note.getParagraphs().size());
-
+
/* new paragraph not yet saved into storages */
assertEquals(0, notebookRepoSync.get(0,
notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size());
assertEquals(0, notebookRepoSync.get(1,
notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size());
-
- /* save to storage under index 0 (first storage) */
+
+ /* save to storage under index 0 (first storage) */
notebookRepoSync.save(0, note, anonymous);
-
+
/* check paragraph saved to first storage */
assertEquals(1, notebookRepoSync.get(0,
notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size());
@@ -284,45 +288,45 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
// one git versioned storage initialized
assertThat(vRepoSync.getRepoCount()).isEqualTo(1);
assertThat(vRepoSync.getRepo(0)).isInstanceOf(GitNotebookRepo.class);
-
+
GitNotebookRepo gitRepo = (GitNotebookRepo) vRepoSync.getRepo(0);
-
+
// no notes
assertThat(vRepoSync.list(anonymous).size()).isEqualTo(0);
// create note
Note note = vNotebookSync.createNote(anonymous);
assertThat(vRepoSync.list(anonymous).size()).isEqualTo(1);
-
+
String noteId = vRepoSync.list(anonymous).get(0).getId();
// first checkpoint
vRepoSync.checkpoint(noteId, "checkpoint message", anonymous);
int vCount = gitRepo.revisionHistory(noteId, anonymous).size();
assertThat(vCount).isEqualTo(1);
-
+
Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map<String, Object> config = p.getConfig();
config.put("enabled", true);
p.setConfig(config);
p.setText("%md checkpoint test");
-
+
// save and checkpoint again
vRepoSync.save(note, anonymous);
vRepoSync.checkpoint(noteId, "checkpoint message 2", anonymous);
assertThat(gitRepo.revisionHistory(noteId, anonymous).size()).isEqualTo(vCount + 1);
notebookRepoSync.remove(note.getId(), anonymous);
}
-
+
@Test
public void testSyncWithAcl() throws IOException {
/* scenario 1 - note exists with acl on main storage */
AuthenticationInfo user1 = new AuthenticationInfo("user1");
Note note = notebookSync.createNote(user1);
assertEquals(0, note.getParagraphs().size());
-
+
// saved on both storages
assertEquals(1, notebookRepoSync.list(0, null).size());
assertEquals(1, notebookRepoSync.list(1, null).size());
-
+
/* check that user1 is the only owner */
NotebookAuthorization authInfo = NotebookAuthorization.getInstance();
Set<String> entity = new HashSet<String>();
@@ -331,23 +335,23 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertEquals(1, authInfo.getOwners(note.getId()).size());
assertEquals(0, authInfo.getReaders(note.getId()).size());
assertEquals(0, authInfo.getWriters(note.getId()).size());
-
+
/* update note and save on secondary storage */
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p1.setText("hello world");
assertEquals(1, note.getParagraphs().size());
notebookRepoSync.save(1, note, null);
-
+
/* check paragraph isn't saved into first storage */
assertEquals(0, notebookRepoSync.get(0,
notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size());
/* check paragraph is saved into second storage */
assertEquals(1, notebookRepoSync.get(1,
notebookRepoSync.list(1, null).get(0).getId(), null).getParagraphs().size());
-
+
/* now sync by user1 */
notebookRepoSync.sync(user1);
-
+
/* check that note updated and acl are same on main storage*/
assertEquals(1, notebookRepoSync.get(0,
notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size());
@@ -355,7 +359,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertEquals(1, authInfo.getOwners(note.getId()).size());
assertEquals(0, authInfo.getReaders(note.getId()).size());
assertEquals(0, authInfo.getWriters(note.getId()).size());
-
+
/* scenario 2 - note doesn't exist on main storage */
/* remove from main storage */
notebookRepoSync.remove(0, note.getId(), user1);
@@ -365,7 +369,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertEquals(0, authInfo.getOwners(note.getId()).size());
assertEquals(0, authInfo.getReaders(note.getId()).size());
assertEquals(0, authInfo.getWriters(note.getId()).size());
-
+
/* now sync - should bring note from secondary storage with added acl */
notebookRepoSync.sync(user1);
assertEquals(1, notebookRepoSync.list(0, null).size());
@@ -423,5 +427,5 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
}
};
}
-
+
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
index 6f85bf6..b393589 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
@@ -22,18 +22,15 @@ import static org.mockito.Mockito.mock;
import java.io.File;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Properties;
-import com.google.common.collect.Maps;
import org.apache.commons.io.FileUtils;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
+import org.apache.zeppelin.interpreter.AbstractInterpreterTest;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterInfo;
import org.apache.zeppelin.interpreter.InterpreterOption;
@@ -41,6 +38,7 @@ import org.apache.zeppelin.interpreter.DefaultInterpreterProperty;
import org.apache.zeppelin.interpreter.InterpreterProperty;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
+
import org.apache.zeppelin.notebook.JobListenerFactory;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Notebook;
@@ -58,59 +56,34 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableMap;
-public class VFSNotebookRepoTest implements JobListenerFactory {
+public class VFSNotebookRepoTest extends AbstractInterpreterTest implements JobListenerFactory {
+
private static final Logger LOG = LoggerFactory.getLogger(VFSNotebookRepoTest.class);
- private ZeppelinConfiguration conf;
+
private SchedulerFactory schedulerFactory;
private Notebook notebook;
private NotebookRepo notebookRepo;
- private InterpreterSettingManager interpreterSettingManager;
- private InterpreterFactory factory;
- private DependencyResolver depResolver;
private NotebookAuthorization notebookAuthorization;
- private File mainZepDir;
- private File mainNotebookDir;
-
@Before
public void setUp() throws Exception {
- String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis();
- mainZepDir = new File(zpath);
- mainZepDir.mkdirs();
- new File(mainZepDir, "conf").mkdirs();
- String mainNotePath = zpath + "/notebook";
- mainNotebookDir = new File(mainNotePath);
- mainNotebookDir.mkdirs();
-
- System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath());
- System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath());
+ System.setProperty(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "mock1,mock2");
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.VFSNotebookRepo");
- conf = ZeppelinConfiguration.create();
-
- this.schedulerFactory = new SchedulerFactory();
-
- this.schedulerFactory = new SchedulerFactory();
- depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo");
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
- ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
- interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
- interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
+ super.setUp();
+ this.schedulerFactory = SchedulerFactory.singleton();
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
- notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, this, search,
+ notebook = new Notebook(conf, notebookRepo, schedulerFactory, interpreterFactory, interpreterSettingManager, this, search,
notebookAuthorization, null);
}
@After
public void tearDown() throws Exception {
- if (!FileUtils.deleteQuietly(mainZepDir)) {
- LOG.error("Failed to delete {} ", mainZepDir.getName());
+ if (!FileUtils.deleteQuietly(testRootDir)) {
+ LOG.error("Failed to delete {} ", testRootDir.getName());
}
}
@@ -120,7 +93,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
int numNotes = notebookRepo.list(null).size();
// when create invalid json file
- File testNoteDir = new File(mainNotebookDir, "test");
+ File testNoteDir = new File(notebookDir, "test");
testNoteDir.mkdir();
FileUtils.writeStringToFile(new File(testNoteDir, "note.json"), "");
@@ -132,7 +105,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
public void testSaveNotebook() throws IOException, InterruptedException {
AuthenticationInfo anonymous = new AuthenticationInfo("anonymous");
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map<String, Object> config = p1.getConfig();
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java
index 46134e5..7223109 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java
@@ -17,35 +17,34 @@
package org.apache.zeppelin.resource;
import com.google.gson.Gson;
-import org.apache.zeppelin.user.AuthenticationInfo;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventPoller;
import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterResourcePool;
+import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
-import java.util.Properties;
+import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
/**
* Unittest for DistributedResourcePool
*/
-public class DistributedResourcePoolTest {
- private static final String INTERPRETER_SCRIPT =
- System.getProperty("os.name").startsWith("Windows") ?
- "../bin/interpreter.cmd" :
- "../bin/interpreter.sh";
- private InterpreterGroup intpGroup1;
- private InterpreterGroup intpGroup2;
- private HashMap<String, String> env;
+public class DistributedResourcePoolTest extends AbstractInterpreterTest {
+
private RemoteInterpreter intp1;
private RemoteInterpreter intp2;
private InterpreterContext context;
@@ -55,50 +54,10 @@ public class DistributedResourcePoolTest {
@Before
public void setUp() throws Exception {
- env = new HashMap<>();
- env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
-
- Properties p = new Properties();
-
- intp1 = new RemoteInterpreter(
- p,
- "note",
- MockInterpreterResourcePool.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- null,
- null,
- "anonymous",
- false
- );
-
- intpGroup1 = new InterpreterGroup("intpGroup1");
- intpGroup1.put("note", new LinkedList<Interpreter>());
- intpGroup1.get("note").add(intp1);
- intp1.setInterpreterGroup(intpGroup1);
-
- intp2 = new RemoteInterpreter(
- p,
- "note",
- MockInterpreterResourcePool.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- null,
- null,
- "anonymous",
- false
- );
-
- intpGroup2 = new InterpreterGroup("intpGroup2");
- intpGroup2.put("note", new LinkedList<Interpreter>());
- intpGroup2.get("note").add(intp2);
- intp2.setInterpreterGroup(intpGroup2);
+ super.setUp();
+ InterpreterSetting interpreterSetting = interpreterSettingManager.getByName("mock_resource_pool");
+ intp1 = (RemoteInterpreter) interpreterSetting.getInterpreter("user1", "note1", "mock_resource_pool");
+ intp2 = (RemoteInterpreter) interpreterSetting.getInterpreter("user2", "note1", "mock_resource_pool");
context = new InterpreterContext(
"note",
@@ -117,26 +76,13 @@ public class DistributedResourcePoolTest {
intp1.open();
intp2.open();
- eventPoller1 = new RemoteInterpreterEventPoller(null, null);
- eventPoller1.setInterpreterGroup(intpGroup1);
- eventPoller1.setInterpreterProcess(intpGroup1.getRemoteInterpreterProcess());
-
- eventPoller2 = new RemoteInterpreterEventPoller(null, null);
- eventPoller2.setInterpreterGroup(intpGroup2);
- eventPoller2.setInterpreterProcess(intpGroup2.getRemoteInterpreterProcess());
-
- eventPoller1.start();
- eventPoller2.start();
+ eventPoller1 = intp1.getInterpreterGroup().getRemoteInterpreterProcess().getRemoteInterpreterEventPoller();
+ eventPoller2 = intp1.getInterpreterGroup().getRemoteInterpreterProcess().getRemoteInterpreterEventPoller();
}
@After
public void tearDown() throws Exception {
- eventPoller1.shutdown();
- intp1.close();
- intpGroup1.close();
- eventPoller2.shutdown();
- intp2.close();
- intpGroup2.close();
+ interpreterSettingManager.close();
}
@Test
@@ -235,13 +181,13 @@ public class DistributedResourcePoolTest {
// then get all resources.
- assertEquals(4, ResourcePoolUtils.getAllResources().size());
+ assertEquals(4, interpreterSettingManager.getAllResources().size());
// when remove all resources from note1
- ResourcePoolUtils.removeResourcesBelongsToNote("note1");
+ interpreterSettingManager.removeResourcesBelongsToNote("note1");
// then resources should be removed.
- assertEquals(2, ResourcePoolUtils.getAllResources().size());
+ assertEquals(2, interpreterSettingManager.getAllResources().size());
assertEquals("", gson.fromJson(
intp1.interpret("get note1:paragraph1:key1", context).message().get(0).getData(),
String.class));
@@ -251,10 +197,10 @@ public class DistributedResourcePoolTest {
// when remove all resources from note2:paragraph1
- ResourcePoolUtils.removeResourcesBelongsToParagraph("note2", "paragraph1");
+ interpreterSettingManager.removeResourcesBelongsToParagraph("note2", "paragraph1");
// then 1
- assertEquals(1, ResourcePoolUtils.getAllResources().size());
+ assertEquals(1, interpreterSettingManager.getAllResources().size());
assertEquals("value2", gson.fromJson(
intp1.interpret("get note2:paragraph2:key2", context).message().get(0).getData(),
String.class));
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
deleted file mode 100644
index ebb5100..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
+++ /dev/null
@@ -1,364 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.scheduler;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.interpreter.*;
-import org.apache.zeppelin.user.AuthenticationInfo;
-import org.apache.zeppelin.display.GUI;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
-import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterA;
-import org.apache.zeppelin.resource.LocalResourcePool;
-import org.apache.zeppelin.scheduler.Job.Status;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
-
- private static final String INTERPRETER_SCRIPT =
- System.getProperty("os.name").startsWith("Windows") ?
- "../bin/interpreter.cmd" :
- "../bin/interpreter.sh";
- private SchedulerFactory schedulerSvc;
- private static final int TICK_WAIT = 100;
- private static final int MAX_WAIT_CYCLES = 100;
-
- @Before
- public void setUp() throws Exception{
- schedulerSvc = new SchedulerFactory();
- }
-
- @After
- public void tearDown(){
-
- }
-
- @Test
- public void test() throws Exception {
- Properties p = new Properties();
- final InterpreterGroup intpGroup = new InterpreterGroup();
- Map<String, String> env = new HashMap<>();
- env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
-
- final RemoteInterpreter intpA = new RemoteInterpreter(
- p,
- "note",
- MockInterpreterA.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- this,
- null,
- "anonymous",
- false);
-
- intpGroup.put("note", new LinkedList<Interpreter>());
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
- intpA.open();
-
- Scheduler scheduler = schedulerSvc.createOrGetRemoteScheduler("test", "note",
- intpA.getInterpreterProcess(),
- 10);
-
- Job job = new Job("jobId", "jobName", null, 200) {
- Object results;
- @Override
- public Object getReturn() {
- return results;
- }
-
- @Override
- public int progress() {
- return 0;
- }
-
- @Override
- public Map<String, Object> info() {
- return null;
- }
-
- @Override
- protected Object jobRun() throws Throwable {
- intpA.interpret("1000", new InterpreterContext(
- "note",
- "jobId",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
- return "1000";
- }
-
- @Override
- protected boolean jobAbort() {
- return false;
- }
-
- @Override
- public void setResult(Object results) {
- this.results = results;
- }
- };
- scheduler.submit(job);
-
- int cycles = 0;
- while (!job.isRunning() && cycles < MAX_WAIT_CYCLES) {
- Thread.sleep(TICK_WAIT);
- cycles++;
- }
- assertTrue(job.isRunning());
-
- Thread.sleep(5*TICK_WAIT);
- assertEquals(0, scheduler.getJobsWaiting().size());
- assertEquals(1, scheduler.getJobsRunning().size());
-
- cycles = 0;
- while (!job.isTerminated() && cycles < MAX_WAIT_CYCLES) {
- Thread.sleep(TICK_WAIT);
- cycles++;
- }
-
- assertTrue(job.isTerminated());
- assertEquals(0, scheduler.getJobsWaiting().size());
- assertEquals(0, scheduler.getJobsRunning().size());
-
- intpA.close();
- schedulerSvc.removeScheduler("test");
- }
-
- @Test
- public void testAbortOnPending() throws Exception {
- Properties p = new Properties();
- final InterpreterGroup intpGroup = new InterpreterGroup();
- Map<String, String> env = new HashMap<>();
- env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
-
- final RemoteInterpreter intpA = new RemoteInterpreter(
- p,
- "note",
- MockInterpreterA.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- this,
- null,
- "anonymous",
- false);
-
- intpGroup.put("note", new LinkedList<Interpreter>());
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
- intpA.open();
-
- Scheduler scheduler = schedulerSvc.createOrGetRemoteScheduler("test", "note",
- intpA.getInterpreterProcess(),
- 10);
-
- Job job1 = new Job("jobId1", "jobName1", null, 200) {
- Object results;
- InterpreterContext context = new InterpreterContext(
- "note",
- "jobId1",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null);
-
- @Override
- public Object getReturn() {
- return results;
- }
-
- @Override
- public int progress() {
- return 0;
- }
-
- @Override
- public Map<String, Object> info() {
- return null;
- }
-
- @Override
- protected Object jobRun() throws Throwable {
- intpA.interpret("1000", context);
- return "1000";
- }
-
- @Override
- protected boolean jobAbort() {
- if (isRunning()) {
- intpA.cancel(context);
- }
- return true;
- }
-
- @Override
- public void setResult(Object results) {
- this.results = results;
- }
- };
-
- Job job2 = new Job("jobId2", "jobName2", null, 200) {
- public Object results;
- InterpreterContext context = new InterpreterContext(
- "note",
- "jobId2",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null);
-
- @Override
- public Object getReturn() {
- return results;
- }
-
- @Override
- public int progress() {
- return 0;
- }
-
- @Override
- public Map<String, Object> info() {
- return null;
- }
-
- @Override
- protected Object jobRun() throws Throwable {
- intpA.interpret("1000", context);
- return "1000";
- }
-
- @Override
- protected boolean jobAbort() {
- if (isRunning()) {
- intpA.cancel(context);
- }
- return true;
- }
-
- @Override
- public void setResult(Object results) {
- this.results = results;
- }
- };
-
- job2.setResult("result2");
-
- scheduler.submit(job1);
- scheduler.submit(job2);
-
-
- int cycles = 0;
- while (!job1.isRunning() && cycles < MAX_WAIT_CYCLES) {
- Thread.sleep(TICK_WAIT);
- cycles++;
- }
- assertTrue(job1.isRunning());
- assertTrue(job2.getStatus() == Status.PENDING);
-
- job2.abort();
-
- cycles = 0;
- while (!job1.isTerminated() && cycles < MAX_WAIT_CYCLES) {
- Thread.sleep(TICK_WAIT);
- cycles++;
- }
-
- assertNotNull(job1.getDateFinished());
- assertTrue(job1.isTerminated());
- assertNull(job2.getDateFinished());
- assertTrue(job2.isTerminated());
- assertEquals("result2", job2.getReturn());
-
- intpA.close();
- schedulerSvc.removeScheduler("test");
- }
-
- @Override
- public void onOutputAppend(String noteId, String paragraphId, int index, String output) {
-
- }
-
- @Override
- public void onOutputUpdated(String noteId, String paragraphId, int index, InterpreterResult.Type type, String output) {
-
- }
-
- @Override
- public void onOutputClear(String noteId, String paragraphId) {
-
- }
-
- @Override
- public void onMetaInfosReceived(String settingId, Map<String, String> metaInfos) {
-
- }
-
- @Override
- public void onGetParagraphRunners(String noteId, String paragraphId, RemoteWorksEventListener callback) {
- if (callback != null) {
- callback.onFinished(new LinkedList<>());
- }
- }
-
- @Override
- public void onRemoteRunParagraph(String noteId, String PsaragraphID) throws Exception {
- }
-
- @Override
- public void onParaInfosReceived(String noteId, String paragraphId,
- String interpreterSettingId, Map<String, String> metaInfos) {
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/resources/conf/interpreter.json
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/resources/conf/interpreter.json b/zeppelin-zengine/src/test/resources/conf/interpreter.json
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/zeppelin-zengine/src/test/resources/conf/interpreter.json
@@ -0,0 +1 @@
+{}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/resources/interpreter/mock/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/resources/interpreter/mock/interpreter-setting.json b/zeppelin-zengine/src/test/resources/interpreter/mock/interpreter-setting.json
deleted file mode 100644
index 65568ef..0000000
--- a/zeppelin-zengine/src/test/resources/interpreter/mock/interpreter-setting.json
+++ /dev/null
@@ -1,12 +0,0 @@
-[
- {
- "group": "mock11",
- "name": "mock11",
- "className": "org.apache.zeppelin.interpreter.mock.MockInterpreter11",
- "properties": {
- },
- "editor": {
- "language": "java"
- }
- }
-]
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/resources/interpreter/mock1/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/resources/interpreter/mock1/interpreter-setting.json b/zeppelin-zengine/src/test/resources/interpreter/mock1/interpreter-setting.json
new file mode 100644
index 0000000..0e6fb21
--- /dev/null
+++ b/zeppelin-zengine/src/test/resources/interpreter/mock1/interpreter-setting.json
@@ -0,0 +1,19 @@
+[
+ {
+ "group": "mock1",
+ "name": "mock1",
+ "className": "org.apache.zeppelin.interpreter.mock.MockInterpreter1",
+ "properties": {
+ },
+ "option": {
+ "remote": false,
+ "port": -1,
+ "perNote": "shared",
+ "perUser": "shared",
+ "isExistingProcess": false,
+ "setPermission": false,
+ "users": [],
+ "isUserImpersonate": false
+ }
+ }
+]
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/resources/interpreter/mock2/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/resources/interpreter/mock2/interpreter-setting.json b/zeppelin-zengine/src/test/resources/interpreter/mock2/interpreter-setting.json
new file mode 100644
index 0000000..aca418a
--- /dev/null
+++ b/zeppelin-zengine/src/test/resources/interpreter/mock2/interpreter-setting.json
@@ -0,0 +1,19 @@
+[
+ {
+ "group": "mock2",
+ "name": "mock2",
+ "className": "org.apache.zeppelin.interpreter.mock.MockInterpreter2",
+ "properties": {
+ },
+ "option": {
+ "remote": false,
+ "port": -1,
+ "perNote": "shared",
+ "perUser": "isolated",
+ "isExistingProcess": false,
+ "setPermission": false,
+ "users": [],
+ "isUserImpersonate": false
+ }
+ }
+]
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/resources/interpreter/mock_resource_pool/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/resources/interpreter/mock_resource_pool/interpreter-setting.json b/zeppelin-zengine/src/test/resources/interpreter/mock_resource_pool/interpreter-setting.json
new file mode 100644
index 0000000..4dfe0a7
--- /dev/null
+++ b/zeppelin-zengine/src/test/resources/interpreter/mock_resource_pool/interpreter-setting.json
@@ -0,0 +1,19 @@
+[
+ {
+ "group": "mock_resource_pool",
+ "name": "mock_resource_pool",
+ "className": "org.apache.zeppelin.interpreter.remote.mock.MockInterpreterResourcePool",
+ "properties": {
+ },
+ "option": {
+ "remote": true,
+ "port": -1,
+ "perNote": "shared",
+ "perUser": "shared",
+ "isExistingProcess": false,
+ "setPermission": false,
+ "users": [],
+ "isUserImpersonate": false
+ }
+ }
+]
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/resources/log4j.properties b/zeppelin-zengine/src/test/resources/log4j.properties
index 001a222..74f619b 100644
--- a/zeppelin-zengine/src/test/resources/log4j.properties
+++ b/zeppelin-zengine/src/test/resources/log4j.properties
@@ -35,7 +35,6 @@ log4j.logger.org.apache.hadoop.mapred=WARN
log4j.logger.org.apache.hadoop.hive.ql=WARN
log4j.logger.org.apache.hadoop.hive.metastore=WARN
log4j.logger.org.apache.haadoop.hive.service.HiveServer=WARN
-log4j.logger.org.apache.zeppelin.scheduler=WARN
log4j.logger.org.quartz=WARN
log4j.logger.DataNucleus=WARN
@@ -45,4 +44,6 @@ log4j.logger.DataNucleus.Datastore=ERROR
# Log all JDBC parameters
log4j.logger.org.hibernate.type=ALL
+log4j.logger.org.apache.zeppelin.interpreter=DEBUG
+log4j.logger.org.apache.zeppelin.scheduler=DEBUG
[05/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
deleted file mode 100644
index 752b4e2..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
+++ /dev/null
@@ -1,459 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.zeppelin.dep.Dependency;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.annotations.SerializedName;
-import com.google.gson.internal.StringMap;
-
-import static org.apache.zeppelin.notebook.utility.IdHashes.generateId;
-
-/**
- * Interpreter settings
- */
-public class InterpreterSetting {
-
- private static final Logger logger = LoggerFactory.getLogger(InterpreterSetting.class);
- private static final String SHARED_PROCESS = "shared_process";
- private String id;
- private String name;
- // always be null in case of InterpreterSettingRef
- private String group;
- private transient Map<String, String> infos;
-
- // Map of the note and paragraphs which has runtime infos generated by this interpreter setting.
- // This map is used to clear the infos in paragraph when the interpretersetting is restarted
- private transient Map<String, Set<String>> runtimeInfosToBeCleared;
-
- /**
- * properties can be either Map<String, DefaultInterpreterProperty> or
- * Map<String, InterpreterProperty>
- * properties should be:
- * - Map<String, InterpreterProperty> when Interpreter instances are saved to
- * `conf/interpreter.json` file
- * - Map<String, DefaultInterpreterProperty> when Interpreters are registered
- * : this is needed after https://github.com/apache/zeppelin/pull/1145
- * which changed the way of getting default interpreter setting AKA interpreterSettingsRef
- */
- private Object properties;
- private Status status;
- private String errorReason;
-
- @SerializedName("interpreterGroup")
- private List<InterpreterInfo> interpreterInfos;
- private final transient Map<String, InterpreterGroup> interpreterGroupRef = new HashMap<>();
- private List<Dependency> dependencies = new LinkedList<>();
- private InterpreterOption option;
- private transient String path;
-
- @SerializedName("runner")
- private InterpreterRunner interpreterRunner;
-
- @Deprecated
- private transient InterpreterGroupFactory interpreterGroupFactory;
-
- private final transient ReentrantReadWriteLock.ReadLock interpreterGroupReadLock;
- private final transient ReentrantReadWriteLock.WriteLock interpreterGroupWriteLock;
-
- public InterpreterSetting() {
- ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
- interpreterGroupReadLock = lock.readLock();
- interpreterGroupWriteLock = lock.writeLock();
- }
-
- public InterpreterSetting(String id, String name, String group,
- List<InterpreterInfo> interpreterInfos, Object properties, List<Dependency> dependencies,
- InterpreterOption option, String path, InterpreterRunner runner) {
- this();
- this.id = id;
- this.name = name;
- this.group = group;
- this.interpreterInfos = interpreterInfos;
- this.properties = properties;
- this.dependencies = dependencies;
- this.option = option;
- this.path = path;
- this.status = Status.READY;
- this.interpreterRunner = runner;
- }
-
- public InterpreterSetting(String name, String group, List<InterpreterInfo> interpreterInfos,
- Object properties, List<Dependency> dependencies, InterpreterOption option, String path,
- InterpreterRunner runner) {
- this(generateId(), name, group, interpreterInfos, properties, dependencies, option, path,
- runner);
- }
-
- /**
- * Create interpreter from interpreterSettingRef
- *
- * @param o interpreterSetting from interpreterSettingRef
- */
- public InterpreterSetting(InterpreterSetting o) {
- this(generateId(), o.getName(), o.getGroup(), o.getInterpreterInfos(), o.getProperties(),
- o.getDependencies(), o.getOption(), o.getPath(), o.getInterpreterRunner());
- }
-
- public String getId() {
- return id;
- }
-
- public String getName() {
- return name;
- }
-
- public String getGroup() {
- return group;
- }
-
- private String getInterpreterProcessKey(String user, String noteId) {
- InterpreterOption option = getOption();
- String key;
- if (getOption().isExistingProcess) {
- key = Constants.EXISTING_PROCESS;
- } else if (getOption().isProcess()) {
- key = (option.perUserIsolated() ? user : "") + ":" + (option.perNoteIsolated() ? noteId : "");
- } else {
- key = SHARED_PROCESS;
- }
-
- //logger.debug("getInterpreterProcessKey: {} for InterpreterSetting Id: {}, Name: {}",
- // key, getId(), getName());
- return key;
- }
-
- private boolean isEqualInterpreterKeyProcessKey(String refKey, String processKey) {
- InterpreterOption option = getOption();
- int validCount = 0;
- if (getOption().isProcess()
- && !(option.perUserIsolated() == true && option.perNoteIsolated() == true)) {
-
- List<String> processList = Arrays.asList(processKey.split(":"));
- List<String> refList = Arrays.asList(refKey.split(":"));
-
- if (refList.size() <= 1 || processList.size() <= 1) {
- return refKey.equals(processKey);
- }
-
- if (processList.get(0).equals("") || processList.get(0).equals(refList.get(0))) {
- validCount = validCount + 1;
- }
-
- if (processList.get(1).equals("") || processList.get(1).equals(refList.get(1))) {
- validCount = validCount + 1;
- }
-
- return (validCount >= 2);
- } else {
- return refKey.equals(processKey);
- }
- }
-
- String getInterpreterSessionKey(String user, String noteId) {
- InterpreterOption option = getOption();
- String key;
- if (option.isExistingProcess()) {
- key = Constants.EXISTING_PROCESS;
- } else if (option.perNoteScoped() && option.perUserScoped()) {
- key = user + ":" + noteId;
- } else if (option.perUserScoped()) {
- key = user;
- } else if (option.perNoteScoped()) {
- key = noteId;
- } else {
- key = "shared_session";
- }
-
- logger.debug("Interpreter session key: {}, for note: {}, user: {}, InterpreterSetting Name: " +
- "{}", key, noteId, user, getName());
- return key;
- }
-
- public InterpreterGroup getInterpreterGroup(String user, String noteId) {
- String key = getInterpreterProcessKey(user, noteId);
- if (!interpreterGroupRef.containsKey(key)) {
- String interpreterGroupId = getId() + ":" + key;
- InterpreterGroup intpGroup =
- interpreterGroupFactory.createInterpreterGroup(interpreterGroupId, getOption());
-
- interpreterGroupWriteLock.lock();
- logger.debug("create interpreter group with groupId:" + interpreterGroupId);
- interpreterGroupRef.put(key, intpGroup);
- interpreterGroupWriteLock.unlock();
- }
- try {
- interpreterGroupReadLock.lock();
- return interpreterGroupRef.get(key);
- } finally {
- interpreterGroupReadLock.unlock();
- }
- }
-
- public Collection<InterpreterGroup> getAllInterpreterGroups() {
- try {
- interpreterGroupReadLock.lock();
- return new LinkedList<>(interpreterGroupRef.values());
- } finally {
- interpreterGroupReadLock.unlock();
- }
- }
-
- void closeAndRemoveInterpreterGroup(String noteId, String user) {
- if (user.equals("anonymous")) {
- user = "";
- }
- String processKey = getInterpreterProcessKey(user, noteId);
- String sessionKey = getInterpreterSessionKey(user, noteId);
- List<InterpreterGroup> groupToRemove = new LinkedList<>();
- InterpreterGroup groupItem;
- for (String intpKey : new HashSet<>(interpreterGroupRef.keySet())) {
- if (isEqualInterpreterKeyProcessKey(intpKey, processKey)) {
- interpreterGroupWriteLock.lock();
- // TODO(jl): interpreterGroup has two or more sessionKeys inside it. thus we should not
- // remove interpreterGroup if it has two or more values.
- groupItem = interpreterGroupRef.get(intpKey);
- interpreterGroupWriteLock.unlock();
- groupToRemove.add(groupItem);
- }
- for (InterpreterGroup groupToClose : groupToRemove) {
- // TODO(jl): Fix the logic removing session. Now, it's handled into groupToClose.clsose()
- groupToClose.close(interpreterGroupRef, intpKey, sessionKey);
- }
- groupToRemove.clear();
- }
-
- //Remove session because all interpreters in this session are closed
- //TODO(jl): Change all code to handle interpreter one by one or all at once
-
- }
-
- void closeAndRemoveAllInterpreterGroups() {
- for (String processKey : new HashSet<>(interpreterGroupRef.keySet())) {
- InterpreterGroup interpreterGroup = interpreterGroupRef.get(processKey);
- for (String sessionKey : new HashSet<>(interpreterGroup.keySet())) {
- interpreterGroup.close(interpreterGroupRef, processKey, sessionKey);
- }
- }
- }
-
- void shutdownAndRemoveAllInterpreterGroups() {
- for (InterpreterGroup interpreterGroup : interpreterGroupRef.values()) {
- interpreterGroup.shutdown();
- }
- }
-
- public Object getProperties() {
- return properties;
- }
-
- public Properties getFlatProperties() {
- Properties p = new Properties();
- if (properties != null) {
- Map<String, InterpreterProperty> propertyMap = (Map<String, InterpreterProperty>) properties;
- for (String key : propertyMap.keySet()) {
- InterpreterProperty tmp = propertyMap.get(key);
- p.put(tmp.getName() != null ? tmp.getName() : key,
- tmp.getValue() != null ? tmp.getValue().toString() : null);
- }
- }
- return p;
- }
-
- public List<Dependency> getDependencies() {
- if (dependencies == null) {
- return new LinkedList<>();
- }
- return dependencies;
- }
-
- public void setDependencies(List<Dependency> dependencies) {
- this.dependencies = dependencies;
- }
-
- public InterpreterOption getOption() {
- if (option == null) {
- option = new InterpreterOption();
- }
-
- return option;
- }
-
- public void setOption(InterpreterOption option) {
- this.option = option;
- }
-
- public String getPath() {
- return path;
- }
-
- public void setPath(String path) {
- this.path = path;
- }
-
- public List<InterpreterInfo> getInterpreterInfos() {
- return interpreterInfos;
- }
-
- void setInterpreterGroupFactory(InterpreterGroupFactory interpreterGroupFactory) {
- this.interpreterGroupFactory = interpreterGroupFactory;
- }
-
- void appendDependencies(List<Dependency> dependencies) {
- for (Dependency dependency : dependencies) {
- if (!this.dependencies.contains(dependency)) {
- this.dependencies.add(dependency);
- }
- }
- }
-
- void setInterpreterOption(InterpreterOption interpreterOption) {
- this.option = interpreterOption;
- }
-
- public void setProperties(Map<String, InterpreterProperty> p) {
- this.properties = p;
- }
-
- void setGroup(String group) {
- this.group = group;
- }
-
- void setName(String name) {
- this.name = name;
- }
-
- /***
- * Interpreter status
- */
- public enum Status {
- DOWNLOADING_DEPENDENCIES,
- ERROR,
- READY
- }
-
- public Status getStatus() {
- return status;
- }
-
- public void setStatus(Status status) {
- this.status = status;
- }
-
- public String getErrorReason() {
- return errorReason;
- }
-
- public void setErrorReason(String errorReason) {
- this.errorReason = errorReason;
- }
-
- public void setInfos(Map<String, String> infos) {
- this.infos = infos;
- }
-
- public Map<String, String> getInfos() {
- return infos;
- }
-
- public InterpreterRunner getInterpreterRunner() {
- return interpreterRunner;
- }
-
- public void setInterpreterRunner(InterpreterRunner interpreterRunner) {
- this.interpreterRunner = interpreterRunner;
- }
-
- public void addNoteToPara(String noteId, String paraId) {
- if (runtimeInfosToBeCleared == null) {
- runtimeInfosToBeCleared = new HashMap<>();
- }
- Set<String> paraIdSet = runtimeInfosToBeCleared.get(noteId);
- if (paraIdSet == null) {
- paraIdSet = new HashSet<>();
- runtimeInfosToBeCleared.put(noteId, paraIdSet);
- }
- paraIdSet.add(paraId);
- }
-
- public Map<String, Set<String>> getNoteIdAndParaMap() {
- return runtimeInfosToBeCleared;
- }
-
- public void clearNoteIdAndParaMap() {
- runtimeInfosToBeCleared = null;
- }
-
- // For backward compatibility of interpreter.json format after ZEPPELIN-2654
- public void convertPermissionsFromUsersToOwners(JsonObject jsonObject) {
- if (jsonObject != null) {
- JsonObject option = jsonObject.getAsJsonObject("option");
- if (option != null) {
- JsonArray users = option.getAsJsonArray("users");
- if (users != null) {
- if (this.option.getOwners() == null) {
- this.option.owners = new LinkedList<>();
- }
- for (JsonElement user : users) {
- this.option.getOwners().add(user.getAsString());
- }
- }
- }
- }
- }
-
- // For backward compatibility of interpreter.json format after ZEPPELIN-2403
- public void convertFlatPropertiesToPropertiesWithWidgets() {
- StringMap newProperties = new StringMap();
- if (properties != null && properties instanceof StringMap) {
- StringMap p = (StringMap) properties;
-
- for (Object o : p.entrySet()) {
- Map.Entry entry = (Map.Entry) o;
- if (!(entry.getValue() instanceof StringMap)) {
- StringMap newProperty = new StringMap();
- newProperty.put("name", entry.getKey());
- newProperty.put("value", entry.getValue());
- newProperty.put("type", InterpreterPropertyType.TEXTAREA.getValue());
- newProperties.put(entry.getKey().toString(), newProperty);
- } else {
- // already converted
- return;
- }
- }
-
- this.properties = newProperties;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
deleted file mode 100644
index 12545d6..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
+++ /dev/null
@@ -1,1136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.lang.reflect.Type;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.DirectoryStream.Filter;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.attribute.PosixFilePermission;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.EnumSet;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
-import org.apache.zeppelin.dep.Dependency;
-import org.apache.zeppelin.dep.DependencyResolver;
-import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter;
-import org.apache.zeppelin.scheduler.Job;
-import org.apache.zeppelin.scheduler.Job.Status;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.sonatype.aether.RepositoryException;
-import org.sonatype.aether.repository.Authentication;
-import org.sonatype.aether.repository.Proxy;
-import org.sonatype.aether.repository.RemoteRepository;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import com.google.gson.internal.StringMap;
-import com.google.gson.reflect.TypeToken;
-
-import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
-import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
-
-/**
- * TBD
- */
-public class InterpreterSettingManager {
-
- private static final Logger logger = LoggerFactory.getLogger(InterpreterSettingManager.class);
- private static final String SHARED_SESSION = "shared_session";
- private static final Map<String, Object> DEFAULT_EDITOR = ImmutableMap.of(
- "language", (Object) "text",
- "editOnDblClick", false);
-
- private final ZeppelinConfiguration zeppelinConfiguration;
- private final Path interpreterDirPath;
- private final Path interpreterBindingPath;
-
- /**
- * This is only references with default settings, name and properties
- * key: InterpreterSetting.name
- */
- private final Map<String, InterpreterSetting> interpreterSettingsRef;
- /**
- * This is used by creating and running Interpreters
- * key: InterpreterSetting.id <- This is becuase backward compatibility
- */
- private final Map<String, InterpreterSetting> interpreterSettings;
- private final Map<String, List<String>> interpreterBindings;
-
- private final DependencyResolver dependencyResolver;
- private final List<RemoteRepository> interpreterRepositories;
-
- private final InterpreterOption defaultOption;
-
- private final Map<String, URLClassLoader> cleanCl;
-
- @Deprecated
- private String[] interpreterClassList;
- private String[] interpreterGroupOrderList;
- private InterpreterGroupFactory interpreterGroupFactory;
-
- private final Gson gson;
-
- public InterpreterSettingManager(ZeppelinConfiguration zeppelinConfiguration,
- DependencyResolver dependencyResolver, InterpreterOption interpreterOption)
- throws IOException, RepositoryException {
- this.zeppelinConfiguration = zeppelinConfiguration;
- this.interpreterDirPath = Paths.get(zeppelinConfiguration.getInterpreterDir());
- logger.debug("InterpreterRootPath: {}", interpreterDirPath);
- this.interpreterBindingPath = Paths.get(zeppelinConfiguration.getInterpreterSettingPath());
- logger.debug("InterpreterBindingPath: {}", interpreterBindingPath);
-
- this.interpreterSettingsRef = Maps.newConcurrentMap();
- this.interpreterSettings = Maps.newConcurrentMap();
- this.interpreterBindings = Maps.newConcurrentMap();
-
- this.dependencyResolver = dependencyResolver;
- this.interpreterRepositories = dependencyResolver.getRepos();
-
- this.defaultOption = interpreterOption;
-
- this.cleanCl = Collections.synchronizedMap(new HashMap<String, URLClassLoader>());
-
- String replsConf = zeppelinConfiguration.getString(ConfVars.ZEPPELIN_INTERPRETERS);
- this.interpreterClassList = replsConf.split(",");
- String groupOrder = zeppelinConfiguration.getString(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER);
- this.interpreterGroupOrderList = groupOrder.split(",");
-
- GsonBuilder gsonBuilder = new GsonBuilder();
- gsonBuilder.setPrettyPrinting();
- this.gson = gsonBuilder.create();
-
- init();
- }
-
- /**
- * Remember this method doesn't keep current connections after being called
- */
- private void loadFromFile() {
- if (!Files.exists(interpreterBindingPath)) {
- // nothing to read
- return;
- }
- InterpreterInfoSaving infoSaving;
- try (BufferedReader jsonReader =
- Files.newBufferedReader(interpreterBindingPath, StandardCharsets.UTF_8)) {
- JsonParser jsonParser = new JsonParser();
- JsonObject jsonObject = jsonParser.parse(jsonReader).getAsJsonObject();
- infoSaving = gson.fromJson(jsonObject.toString(), InterpreterInfoSaving.class);
-
- for (String k : infoSaving.interpreterSettings.keySet()) {
- InterpreterSetting setting = infoSaving.interpreterSettings.get(k);
-
- setting.convertFlatPropertiesToPropertiesWithWidgets();
-
- List<InterpreterInfo> infos = setting.getInterpreterInfos();
-
- // Convert json StringMap to Properties
- StringMap<StringMap> p = (StringMap<StringMap>) setting.getProperties();
- Map<String, InterpreterProperty> properties = new HashMap();
- for (String key : p.keySet()) {
- StringMap<String> fields = (StringMap<String>) p.get(key);
- String type = InterpreterPropertyType.TEXTAREA.getValue();
- try {
- type = InterpreterPropertyType.byValue(fields.get("type")).getValue();
- } catch (Exception e) {
- logger.warn("Incorrect type of property {} in settings {}", key,
- setting.getId());
- }
- properties.put(key, new InterpreterProperty(key, fields.get("value"), type));
- }
- setting.setProperties(properties);
-
- // Always use separate interpreter process
- // While we decided to turn this feature on always (without providing
- // enable/disable option on GUI).
- // previously created setting should turn this feature on here.
- setting.getOption().setRemote(true);
-
- setting.convertPermissionsFromUsersToOwners(
- jsonObject.getAsJsonObject("interpreterSettings").getAsJsonObject(setting.getId()));
-
- // Update transient information from InterpreterSettingRef
- InterpreterSetting interpreterSettingObject =
- interpreterSettingsRef.get(setting.getGroup());
- if (interpreterSettingObject == null) {
- logger.warn("can't get InterpreterSetting " +
- "Information From loaded Interpreter Setting Ref - {} ", setting.getGroup());
- continue;
- }
- String depClassPath = interpreterSettingObject.getPath();
- setting.setPath(depClassPath);
-
- for (InterpreterInfo info : infos) {
- if (info.getEditor() == null) {
- Map<String, Object> editor = getEditorFromSettingByClassName(interpreterSettingObject,
- info.getClassName());
- info.setEditor(editor);
- }
- }
-
- setting.setInterpreterGroupFactory(interpreterGroupFactory);
-
- loadInterpreterDependencies(setting);
- interpreterSettings.put(k, setting);
- }
-
- interpreterBindings.putAll(infoSaving.interpreterBindings);
-
- if (infoSaving.interpreterRepositories != null) {
- for (RemoteRepository repo : infoSaving.interpreterRepositories) {
- if (!dependencyResolver.getRepos().contains(repo)) {
- this.interpreterRepositories.add(repo);
- }
- }
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public void saveToFile() throws IOException {
- String jsonString;
-
- synchronized (interpreterSettings) {
- InterpreterInfoSaving info = new InterpreterInfoSaving();
- info.interpreterBindings = interpreterBindings;
- info.interpreterSettings = interpreterSettings;
- info.interpreterRepositories = interpreterRepositories;
-
- jsonString = info.toJson();
- }
-
- if (!Files.exists(interpreterBindingPath)) {
- Files.createFile(interpreterBindingPath);
-
- try {
- Set<PosixFilePermission> permissions = EnumSet.of(OWNER_READ, OWNER_WRITE);
- Files.setPosixFilePermissions(interpreterBindingPath, permissions);
- } catch (UnsupportedOperationException e) {
- // File system does not support Posix file permissions (likely windows) - continue anyway.
- logger.warn("unable to setPosixFilePermissions on '{}'.", interpreterBindingPath);
- }
- }
-
- FileOutputStream fos = new FileOutputStream(interpreterBindingPath.toFile(), false);
- OutputStreamWriter out = new OutputStreamWriter(fos);
- out.append(jsonString);
- out.close();
- fos.close();
- }
-
- //TODO(jl): Fix it to remove InterpreterGroupFactory
- public void setInterpreterGroupFactory(InterpreterGroupFactory interpreterGroupFactory) {
- for (InterpreterSetting setting : interpreterSettings.values()) {
- setting.setInterpreterGroupFactory(interpreterGroupFactory);
- }
- this.interpreterGroupFactory = interpreterGroupFactory;
- }
-
- private void init() throws InterpreterException, IOException, RepositoryException {
- String interpreterJson = zeppelinConfiguration.getInterpreterJson();
- ClassLoader cl = Thread.currentThread().getContextClassLoader();
-
- if (Files.exists(interpreterDirPath)) {
- for (Path interpreterDir : Files
- .newDirectoryStream(interpreterDirPath, new Filter<Path>() {
- @Override
- public boolean accept(Path entry) throws IOException {
- return Files.exists(entry) && Files.isDirectory(entry);
- }
- })) {
- String interpreterDirString = interpreterDir.toString();
-
- /**
- * Register interpreter by the following ordering
- * 1. Register it from path {ZEPPELIN_HOME}/interpreter/{interpreter_name}/
- * interpreter-setting.json
- * 2. Register it from interpreter-setting.json in classpath
- * {ZEPPELIN_HOME}/interpreter/{interpreter_name}
- * 3. Register it by Interpreter.register
- */
- if (!registerInterpreterFromPath(interpreterDirString, interpreterJson)) {
- if (!registerInterpreterFromResource(cl, interpreterDirString, interpreterJson)) {
- /*
- * TODO(jongyoul)
- * - Remove these codes below because of legacy code
- * - Support ThreadInterpreter
- */
- URLClassLoader ccl = new URLClassLoader(
- recursiveBuildLibList(interpreterDir.toFile()), cl);
- for (String className : interpreterClassList) {
- try {
- // Load classes
- Class.forName(className, true, ccl);
- Set<String> interpreterKeys = Interpreter.registeredInterpreters.keySet();
- for (String interpreterKey : interpreterKeys) {
- if (className
- .equals(Interpreter.registeredInterpreters.get(interpreterKey)
- .getClassName())) {
- Interpreter.registeredInterpreters.get(interpreterKey)
- .setPath(interpreterDirString);
- logger.info("Interpreter " + interpreterKey + " found. class=" + className);
- cleanCl.put(interpreterDirString, ccl);
- }
- }
- } catch (Throwable t) {
- // nothing to do
- }
- }
- }
- }
- }
- }
-
- for (RegisteredInterpreter registeredInterpreter : Interpreter.registeredInterpreters
- .values()) {
- logger
- .debug("Registered: {} -> {}. Properties: {}", registeredInterpreter.getInterpreterKey(),
- registeredInterpreter.getClassName(), registeredInterpreter.getProperties());
- }
-
- // RegisteredInterpreters -> interpreterSettingRef
- InterpreterInfo interpreterInfo;
- for (RegisteredInterpreter r : Interpreter.registeredInterpreters.values()) {
- interpreterInfo =
- new InterpreterInfo(r.getClassName(), r.getName(), r.isDefaultInterpreter(),
- r.getEditor());
- add(r.getGroup(), interpreterInfo, r.getProperties(), defaultOption, r.getPath(),
- r.getRunner());
- }
-
- for (String settingId : interpreterSettingsRef.keySet()) {
- InterpreterSetting setting = interpreterSettingsRef.get(settingId);
- logger.info("InterpreterSettingRef name {}", setting.getName());
- }
-
- loadFromFile();
-
- // if no interpreter settings are loaded, create default set
- if (0 == interpreterSettings.size()) {
- Map<String, InterpreterSetting> temp = new HashMap<>();
- InterpreterSetting interpreterSetting;
- for (InterpreterSetting setting : interpreterSettingsRef.values()) {
- interpreterSetting = createFromInterpreterSettingRef(setting);
- temp.put(setting.getName(), interpreterSetting);
- }
-
- for (String group : interpreterGroupOrderList) {
- if (null != (interpreterSetting = temp.remove(group))) {
- interpreterSettings.put(interpreterSetting.getId(), interpreterSetting);
- }
- }
-
- for (InterpreterSetting setting : temp.values()) {
- interpreterSettings.put(setting.getId(), setting);
- }
-
- saveToFile();
- }
-
- for (String settingId : interpreterSettings.keySet()) {
- InterpreterSetting setting = interpreterSettings.get(settingId);
- logger.info("InterpreterSetting group {} : id={}, name={}", setting.getGroup(), settingId,
- setting.getName());
- }
- }
-
- private boolean registerInterpreterFromResource(ClassLoader cl, String interpreterDir,
- String interpreterJson) throws IOException, RepositoryException {
- URL[] urls = recursiveBuildLibList(new File(interpreterDir));
- ClassLoader tempClassLoader = new URLClassLoader(urls, cl);
-
- Enumeration<URL> interpreterSettings = tempClassLoader.getResources(interpreterJson);
- if (!interpreterSettings.hasMoreElements()) {
- return false;
- }
- for (URL url : Collections.list(interpreterSettings)) {
- try (InputStream inputStream = url.openStream()) {
- logger.debug("Reading {} from {}", interpreterJson, url);
- List<RegisteredInterpreter> registeredInterpreterList =
- getInterpreterListFromJson(inputStream);
- registerInterpreters(registeredInterpreterList, interpreterDir);
- }
- }
- return true;
- }
-
- private boolean registerInterpreterFromPath(String interpreterDir, String interpreterJson)
- throws IOException, RepositoryException {
-
- Path interpreterJsonPath = Paths.get(interpreterDir, interpreterJson);
- if (Files.exists(interpreterJsonPath)) {
- logger.debug("Reading {}", interpreterJsonPath);
- List<RegisteredInterpreter> registeredInterpreterList =
- getInterpreterListFromJson(interpreterJsonPath);
- registerInterpreters(registeredInterpreterList, interpreterDir);
- return true;
- }
- return false;
- }
-
- private List<RegisteredInterpreter> getInterpreterListFromJson(Path filename)
- throws FileNotFoundException {
- return getInterpreterListFromJson(new FileInputStream(filename.toFile()));
- }
-
- private List<RegisteredInterpreter> getInterpreterListFromJson(InputStream stream) {
- Type registeredInterpreterListType = new TypeToken<List<RegisteredInterpreter>>() {
- }.getType();
- return gson.fromJson(new InputStreamReader(stream), registeredInterpreterListType);
- }
-
- private void registerInterpreters(List<RegisteredInterpreter> registeredInterpreters,
- String absolutePath) throws IOException, RepositoryException {
-
- for (RegisteredInterpreter registeredInterpreter : registeredInterpreters) {
- InterpreterInfo interpreterInfo =
- new InterpreterInfo(registeredInterpreter.getClassName(), registeredInterpreter.getName(),
- registeredInterpreter.isDefaultInterpreter(), registeredInterpreter.getEditor());
- // use defaultOption if it is not specified in interpreter-setting.json
- InterpreterOption option = registeredInterpreter.getOption() == null ? defaultOption :
- registeredInterpreter.getOption();
- add(registeredInterpreter.getGroup(), interpreterInfo, registeredInterpreter.getProperties(),
- option, absolutePath, registeredInterpreter.getRunner());
- }
-
- }
-
- public InterpreterSetting getDefaultInterpreterSetting(List<InterpreterSetting> settings) {
- if (settings == null || settings.isEmpty()) {
- return null;
- }
- return settings.get(0);
- }
-
- public InterpreterSetting getDefaultInterpreterSetting(String noteId) {
- return getDefaultInterpreterSetting(getInterpreterSettings(noteId));
- }
-
- public List<InterpreterSetting> getInterpreterSettings(String noteId) {
- List<String> interpreterSettingIds = getNoteInterpreterSettingBinding(noteId);
- LinkedList<InterpreterSetting> settings = new LinkedList<>();
-
- Iterator<String> iter = interpreterSettingIds.iterator();
- while (iter.hasNext()) {
- String id = iter.next();
- InterpreterSetting setting = get(id);
- if (setting == null) {
- // interpreter setting is removed from factory. remove id from here, too
- iter.remove();
- } else {
- settings.add(setting);
- }
- }
- return settings;
- }
-
- private List<String> getNoteInterpreterSettingBinding(String noteId) {
- LinkedList<String> bindings = new LinkedList<>();
- List<String> settingIds = interpreterBindings.get(noteId);
- if (settingIds != null) {
- bindings.addAll(settingIds);
- }
- return bindings;
- }
-
- private InterpreterSetting createFromInterpreterSettingRef(String name) {
- Preconditions.checkNotNull(name, "reference name should be not null");
- InterpreterSetting settingRef = interpreterSettingsRef.get(name);
- return createFromInterpreterSettingRef(settingRef);
- }
-
- private InterpreterSetting createFromInterpreterSettingRef(InterpreterSetting o) {
- // should return immutable objects
- List<InterpreterInfo> infos = (null == o.getInterpreterInfos()) ?
- new ArrayList<InterpreterInfo>() : new ArrayList<>(o.getInterpreterInfos());
- List<Dependency> deps = (null == o.getDependencies()) ?
- new ArrayList<Dependency>() : new ArrayList<>(o.getDependencies());
- Map<String, InterpreterProperty> props =
- convertInterpreterProperties((Map<String, DefaultInterpreterProperty>) o.getProperties());
- InterpreterOption option = InterpreterOption.fromInterpreterOption(o.getOption());
-
- InterpreterSetting setting = new InterpreterSetting(o.getName(), o.getName(),
- infos, props, deps, option, o.getPath(), o.getInterpreterRunner());
- setting.setInterpreterGroupFactory(interpreterGroupFactory);
- return setting;
- }
-
- private Map<String, InterpreterProperty> convertInterpreterProperties(
- Map<String, DefaultInterpreterProperty> defaultProperties) {
- Map<String, InterpreterProperty> properties = new HashMap<>();
-
- for (String key : defaultProperties.keySet()) {
- DefaultInterpreterProperty defaultInterpreterProperty = defaultProperties.get(key);
- properties.put(key, new InterpreterProperty(key, defaultInterpreterProperty.getValue(),
- defaultInterpreterProperty.getType()));
- }
- return properties;
- }
-
- public Map<String, Object> getEditorSetting(Interpreter interpreter, String user, String noteId,
- String replName) {
- Map<String, Object> editor = DEFAULT_EDITOR;
- String group = StringUtils.EMPTY;
- try {
- String defaultSettingName = getDefaultInterpreterSetting(noteId).getName();
- List<InterpreterSetting> intpSettings = getInterpreterSettings(noteId);
- for (InterpreterSetting intpSetting : intpSettings) {
- String[] replNameSplit = replName.split("\\.");
- if (replNameSplit.length == 2) {
- group = replNameSplit[0];
- }
- // when replName is 'name' of interpreter
- if (defaultSettingName.equals(intpSetting.getName())) {
- editor = getEditorFromSettingByClassName(intpSetting, interpreter.getClassName());
- }
- // when replName is 'alias name' of interpreter or 'group' of interpreter
- if (replName.equals(intpSetting.getName()) || group.equals(intpSetting.getName())) {
- editor = getEditorFromSettingByClassName(intpSetting, interpreter.getClassName());
- break;
- }
- }
- } catch (NullPointerException e) {
- // Use `debug` level because this log occurs frequently
- logger.debug("Couldn't get interpreter editor setting");
- }
- return editor;
- }
-
- public Map<String, Object> getEditorFromSettingByClassName(InterpreterSetting intpSetting,
- String className) {
- List<InterpreterInfo> intpInfos = intpSetting.getInterpreterInfos();
- for (InterpreterInfo intpInfo : intpInfos) {
-
- if (className.equals(intpInfo.getClassName())) {
- if (intpInfo.getEditor() == null) {
- break;
- }
- return intpInfo.getEditor();
- }
- }
- return DEFAULT_EDITOR;
- }
-
- private void loadInterpreterDependencies(final InterpreterSetting setting) {
- setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
- setting.setErrorReason(null);
- interpreterSettings.put(setting.getId(), setting);
- synchronized (interpreterSettings) {
- final Thread t = new Thread() {
- public void run() {
- try {
- // dependencies to prevent library conflict
- File localRepoDir = new File(zeppelinConfiguration.getInterpreterLocalRepoPath() + "/" +
- setting.getId());
- if (localRepoDir.exists()) {
- try {
- FileUtils.forceDelete(localRepoDir);
- } catch (FileNotFoundException e) {
- logger.info("A file that does not exist cannot be deleted, nothing to worry", e);
- }
- }
-
- // load dependencies
- List<Dependency> deps = setting.getDependencies();
- if (deps != null) {
- for (Dependency d : deps) {
- File destDir = new File(
- zeppelinConfiguration.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
-
- if (d.getExclusions() != null) {
- dependencyResolver.load(d.getGroupArtifactVersion(), d.getExclusions(),
- new File(destDir, setting.getId()));
- } else {
- dependencyResolver
- .load(d.getGroupArtifactVersion(), new File(destDir, setting.getId()));
- }
- }
- }
-
- setting.setStatus(InterpreterSetting.Status.READY);
- setting.setErrorReason(null);
- } catch (Exception e) {
- logger.error(String.format("Error while downloading repos for interpreter group : %s," +
- " go to interpreter setting page click on edit and save it again to make " +
- "this interpreter work properly. : %s",
- setting.getGroup(), e.getLocalizedMessage()), e);
- setting.setErrorReason(e.getLocalizedMessage());
- setting.setStatus(InterpreterSetting.Status.ERROR);
- } finally {
- interpreterSettings.put(setting.getId(), setting);
- }
- }
- };
- t.start();
- }
- }
-
- /**
- * Overwrite dependency jar under local-repo/{interpreterId}
- * if jar file in original path is changed
- */
- private void copyDependenciesFromLocalPath(final InterpreterSetting setting) {
- setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
- interpreterSettings.put(setting.getId(), setting);
- synchronized (interpreterSettings) {
- final Thread t = new Thread() {
- public void run() {
- try {
- List<Dependency> deps = setting.getDependencies();
- if (deps != null) {
- for (Dependency d : deps) {
- File destDir = new File(
- zeppelinConfiguration.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
-
- int numSplits = d.getGroupArtifactVersion().split(":").length;
- if (!(numSplits >= 3 && numSplits <= 6)) {
- dependencyResolver.copyLocalDependency(d.getGroupArtifactVersion(),
- new File(destDir, setting.getId()));
- }
- }
- }
- setting.setStatus(InterpreterSetting.Status.READY);
- } catch (Exception e) {
- logger.error(String.format("Error while copying deps for interpreter group : %s," +
- " go to interpreter setting page click on edit and save it again to make " +
- "this interpreter work properly.",
- setting.getGroup()), e);
- setting.setErrorReason(e.getLocalizedMessage());
- setting.setStatus(InterpreterSetting.Status.ERROR);
- } finally {
- interpreterSettings.put(setting.getId(), setting);
- }
- }
- };
- t.start();
- }
- }
-
- /**
- * Return ordered interpreter setting list.
- * The list does not contain more than one setting from the same interpreter class.
- * Order by InterpreterClass (order defined by ZEPPELIN_INTERPRETERS), Interpreter setting name
- */
- public List<String> getDefaultInterpreterSettingList() {
- // this list will contain default interpreter setting list
- List<String> defaultSettings = new LinkedList<>();
-
- // to ignore the same interpreter group
- Map<String, Boolean> interpreterGroupCheck = new HashMap<>();
-
- List<InterpreterSetting> sortedSettings = get();
-
- for (InterpreterSetting setting : sortedSettings) {
- if (defaultSettings.contains(setting.getId())) {
- continue;
- }
-
- if (!interpreterGroupCheck.containsKey(setting.getName())) {
- defaultSettings.add(setting.getId());
- interpreterGroupCheck.put(setting.getName(), true);
- }
- }
- return defaultSettings;
- }
-
- List<RegisteredInterpreter> getRegisteredInterpreterList() {
- return new ArrayList<>(Interpreter.registeredInterpreters.values());
- }
-
-
- private boolean findDefaultInterpreter(List<InterpreterInfo> infos) {
- for (InterpreterInfo interpreterInfo : infos) {
- if (interpreterInfo.isDefaultInterpreter()) {
- return true;
- }
- }
- return false;
- }
-
- public InterpreterSetting createNewSetting(String name, String group,
- List<Dependency> dependencies, InterpreterOption option, Map<String, InterpreterProperty> p)
- throws IOException {
- if (name.indexOf(".") >= 0) {
- throw new IOException("'.' is invalid for InterpreterSetting name.");
- }
- InterpreterSetting setting = createFromInterpreterSettingRef(group);
- setting.setName(name);
- setting.setGroup(group);
- setting.appendDependencies(dependencies);
- setting.setInterpreterOption(option);
- setting.setProperties(p);
- setting.setInterpreterGroupFactory(interpreterGroupFactory);
- interpreterSettings.put(setting.getId(), setting);
- loadInterpreterDependencies(setting);
- saveToFile();
- return setting;
- }
-
- private InterpreterSetting add(String group, InterpreterInfo interpreterInfo,
- Map<String, DefaultInterpreterProperty> interpreterProperties, InterpreterOption option,
- String path, InterpreterRunner runner)
- throws InterpreterException, IOException, RepositoryException {
- ArrayList<InterpreterInfo> infos = new ArrayList<>();
- infos.add(interpreterInfo);
- return add(group, infos, new ArrayList<Dependency>(), option, interpreterProperties, path,
- runner);
- }
-
- /**
- * @param group InterpreterSetting reference name
- */
- public InterpreterSetting add(String group, ArrayList<InterpreterInfo> interpreterInfos,
- List<Dependency> dependencies, InterpreterOption option,
- Map<String, DefaultInterpreterProperty> interpreterProperties, String path,
- InterpreterRunner runner) {
- Preconditions.checkNotNull(group, "name should not be null");
- Preconditions.checkNotNull(interpreterInfos, "interpreterInfos should not be null");
- Preconditions.checkNotNull(dependencies, "dependencies should not be null");
- Preconditions.checkNotNull(option, "option should not be null");
- Preconditions.checkNotNull(interpreterProperties, "properties should not be null");
-
- InterpreterSetting interpreterSetting;
-
- synchronized (interpreterSettingsRef) {
- if (interpreterSettingsRef.containsKey(group)) {
- interpreterSetting = interpreterSettingsRef.get(group);
-
- // Append InterpreterInfo
- List<InterpreterInfo> infos = interpreterSetting.getInterpreterInfos();
- boolean hasDefaultInterpreter = findDefaultInterpreter(infos);
- for (InterpreterInfo interpreterInfo : interpreterInfos) {
- if (!infos.contains(interpreterInfo)) {
- if (!hasDefaultInterpreter && interpreterInfo.isDefaultInterpreter()) {
- hasDefaultInterpreter = true;
- infos.add(0, interpreterInfo);
- } else {
- infos.add(interpreterInfo);
- }
- }
- }
-
- // Append dependencies
- List<Dependency> dependencyList = interpreterSetting.getDependencies();
- for (Dependency dependency : dependencies) {
- if (!dependencyList.contains(dependency)) {
- dependencyList.add(dependency);
- }
- }
-
- // Append properties
- Map<String, DefaultInterpreterProperty> properties =
- (Map<String, DefaultInterpreterProperty>) interpreterSetting.getProperties();
- for (String key : interpreterProperties.keySet()) {
- if (!properties.containsKey(key)) {
- properties.put(key, interpreterProperties.get(key));
- }
- }
-
- } else {
- interpreterSetting =
- new InterpreterSetting(group, null, interpreterInfos, interpreterProperties,
- dependencies, option, path, runner);
- interpreterSettingsRef.put(group, interpreterSetting);
- }
- }
-
- if (dependencies.size() > 0) {
- loadInterpreterDependencies(interpreterSetting);
- }
-
- interpreterSetting.setInterpreterGroupFactory(interpreterGroupFactory);
- return interpreterSetting;
- }
-
- /**
- * map interpreter ids into noteId
- *
- * @param noteId note id
- * @param ids InterpreterSetting id list
- */
- public void setInterpreters(String user, String noteId, List<String> ids) throws IOException {
- putNoteInterpreterSettingBinding(user, noteId, ids);
- }
-
- private void putNoteInterpreterSettingBinding(String user, String noteId,
- List<String> settingList) throws IOException {
- List<String> unBindedSettings = new LinkedList<>();
-
- synchronized (interpreterSettings) {
- List<String> oldSettings = interpreterBindings.get(noteId);
- if (oldSettings != null) {
- for (String oldSettingId : oldSettings) {
- if (!settingList.contains(oldSettingId)) {
- unBindedSettings.add(oldSettingId);
- }
- }
- }
- interpreterBindings.put(noteId, settingList);
- saveToFile();
-
- for (String settingId : unBindedSettings) {
- InterpreterSetting setting = get(settingId);
- removeInterpretersForNote(setting, user, noteId);
- }
- }
- }
-
- public void removeInterpretersForNote(InterpreterSetting interpreterSetting, String user,
- String noteId) {
- //TODO(jl): This is only for hotfix. You should fix it as a beautiful way
- InterpreterOption interpreterOption = interpreterSetting.getOption();
- if (!(InterpreterOption.SHARED.equals(interpreterOption.perNote)
- && InterpreterOption.SHARED.equals(interpreterOption.perUser))) {
- interpreterSetting.closeAndRemoveInterpreterGroup(noteId, "");
- }
- }
-
- public String getInterpreterSessionKey(String user, String noteId, InterpreterSetting setting) {
- InterpreterOption option = setting.getOption();
- String key;
- if (option.isExistingProcess()) {
- key = Constants.EXISTING_PROCESS;
- } else if (option.perNoteScoped() && option.perUserScoped()) {
- key = user + ":" + noteId;
- } else if (option.perUserScoped()) {
- key = user;
- } else if (option.perNoteScoped()) {
- key = noteId;
- } else {
- key = SHARED_SESSION;
- }
-
- logger.debug("Interpreter session key: {}, for note: {}, user: {}, InterpreterSetting Name: " +
- "{}", key, noteId, user, setting.getName());
- return key;
- }
-
-
- public List<String> getInterpreters(String noteId) {
- return getNoteInterpreterSettingBinding(noteId);
- }
-
- public void closeNote(String user, String noteId) {
- // close interpreters in this note session
- List<InterpreterSetting> settings = getInterpreterSettings(noteId);
- if (settings == null || settings.size() == 0) {
- return;
- }
-
- logger.info("closeNote: {}", noteId);
- for (InterpreterSetting setting : settings) {
- removeInterpretersForNote(setting, user, noteId);
- }
- }
-
- public Map<String, InterpreterSetting> getAvailableInterpreterSettings() {
- return interpreterSettingsRef;
- }
-
- private URL[] recursiveBuildLibList(File path) throws MalformedURLException {
- URL[] urls = new URL[0];
- if (path == null || !path.exists()) {
- return urls;
- } else if (path.getName().startsWith(".")) {
- return urls;
- } else if (path.isDirectory()) {
- File[] files = path.listFiles();
- if (files != null) {
- for (File f : files) {
- urls = (URL[]) ArrayUtils.addAll(urls, recursiveBuildLibList(f));
- }
- }
- return urls;
- } else {
- return new URL[]{path.toURI().toURL()};
- }
- }
-
- public List<RemoteRepository> getRepositories() {
- return this.interpreterRepositories;
- }
-
- public void addRepository(String id, String url, boolean snapshot, Authentication auth,
- Proxy proxy) throws IOException {
- dependencyResolver.addRepo(id, url, snapshot, auth, proxy);
- saveToFile();
- }
-
- public void removeRepository(String id) throws IOException {
- dependencyResolver.delRepo(id);
- saveToFile();
- }
-
- public void removeNoteInterpreterSettingBinding(String user, String noteId) throws IOException {
- List<String> settingIds = interpreterBindings.remove(noteId);
- if (settingIds != null) {
- for (String settingId : settingIds) {
- InterpreterSetting setting = get(settingId);
- if (setting != null) {
- this.removeInterpretersForNote(setting, user, noteId);
- }
- }
- }
- saveToFile();
- }
-
- /**
- * Change interpreter property and restart
- */
- public void setPropertyAndRestart(String id, InterpreterOption option,
- Map<String, InterpreterProperty> properties,
- List<Dependency> dependencies) throws IOException {
- synchronized (interpreterSettings) {
- InterpreterSetting intpSetting = interpreterSettings.get(id);
- if (intpSetting != null) {
- try {
- stopJobAllInterpreter(intpSetting);
-
- intpSetting.closeAndRemoveAllInterpreterGroups();
- intpSetting.setOption(option);
- intpSetting.setProperties(properties);
- intpSetting.setDependencies(dependencies);
- loadInterpreterDependencies(intpSetting);
-
- saveToFile();
- } catch (Exception e) {
- loadFromFile();
- throw e;
- }
- } else {
- throw new InterpreterException("Interpreter setting id " + id + " not found");
- }
- }
- }
-
- public void restart(String settingId, String noteId, String user) {
- InterpreterSetting intpSetting = interpreterSettings.get(settingId);
- Preconditions.checkNotNull(intpSetting);
- synchronized (interpreterSettings) {
- intpSetting = interpreterSettings.get(settingId);
- // Check if dependency in specified path is changed
- // If it did, overwrite old dependency jar with new one
- if (intpSetting != null) {
- //clean up metaInfos
- intpSetting.setInfos(null);
- copyDependenciesFromLocalPath(intpSetting);
-
- stopJobAllInterpreter(intpSetting);
- if (user.equals("anonymous")) {
- intpSetting.closeAndRemoveAllInterpreterGroups();
- } else {
- intpSetting.closeAndRemoveInterpreterGroup(noteId, user);
- }
-
- } else {
- throw new InterpreterException("Interpreter setting id " + settingId + " not found");
- }
- }
- }
-
- public void restart(String id) {
- restart(id, "", "anonymous");
- }
-
- private void stopJobAllInterpreter(InterpreterSetting intpSetting) {
- if (intpSetting != null) {
- for (InterpreterGroup intpGroup : intpSetting.getAllInterpreterGroups()) {
- for (List<Interpreter> interpreters : intpGroup.values()) {
- for (Interpreter intp : interpreters) {
- for (Job job : intp.getScheduler().getJobsRunning()) {
- job.abort();
- job.setStatus(Status.ABORT);
- logger.info("Job " + job.getJobName() + " aborted ");
- }
- for (Job job : intp.getScheduler().getJobsWaiting()) {
- job.abort();
- job.setStatus(Status.ABORT);
- logger.info("Job " + job.getJobName() + " aborted ");
- }
- }
- }
- }
- }
- }
-
- public InterpreterSetting get(String name) {
- synchronized (interpreterSettings) {
- return interpreterSettings.get(name);
- }
- }
-
- public void remove(String id) throws IOException {
- synchronized (interpreterSettings) {
- if (interpreterSettings.containsKey(id)) {
- InterpreterSetting intp = interpreterSettings.get(id);
- intp.closeAndRemoveAllInterpreterGroups();
-
- interpreterSettings.remove(id);
- for (List<String> settings : interpreterBindings.values()) {
- Iterator<String> it = settings.iterator();
- while (it.hasNext()) {
- String settingId = it.next();
- if (settingId.equals(id)) {
- it.remove();
- }
- }
- }
- saveToFile();
- }
- }
-
- File localRepoDir = new File(zeppelinConfiguration.getInterpreterLocalRepoPath() + "/" + id);
- FileUtils.deleteDirectory(localRepoDir);
- }
-
- /**
- * Get interpreter settings
- */
- public List<InterpreterSetting> get() {
- synchronized (interpreterSettings) {
- List<InterpreterSetting> orderedSettings = new LinkedList<>();
-
- Map<String, List<InterpreterSetting>> nameInterpreterSettingMap = new HashMap<>();
- for (InterpreterSetting interpreterSetting : interpreterSettings.values()) {
- String group = interpreterSetting.getGroup();
- if (!nameInterpreterSettingMap.containsKey(group)) {
- nameInterpreterSettingMap.put(group, new ArrayList<InterpreterSetting>());
- }
- nameInterpreterSettingMap.get(group).add(interpreterSetting);
- }
-
- for (String groupName : interpreterGroupOrderList) {
- List<InterpreterSetting> interpreterSettingList =
- nameInterpreterSettingMap.remove(groupName);
- if (null != interpreterSettingList) {
- for (InterpreterSetting interpreterSetting : interpreterSettingList) {
- orderedSettings.add(interpreterSetting);
- }
- }
- }
-
- List<InterpreterSetting> settings = new ArrayList<>();
-
- for (List<InterpreterSetting> interpreterSettingList : nameInterpreterSettingMap.values()) {
- for (InterpreterSetting interpreterSetting : interpreterSettingList) {
- settings.add(interpreterSetting);
- }
- }
-
- Collections.sort(settings, new Comparator<InterpreterSetting>() {
- @Override
- public int compare(InterpreterSetting o1, InterpreterSetting o2) {
- return o1.getName().compareTo(o2.getName());
- }
- });
-
- orderedSettings.addAll(settings);
-
- return orderedSettings;
- }
- }
-
- public void close(InterpreterSetting interpreterSetting) {
- interpreterSetting.closeAndRemoveAllInterpreterGroups();
- }
-
- public void close() {
- List<Thread> closeThreads = new LinkedList<>();
- synchronized (interpreterSettings) {
- Collection<InterpreterSetting> intpSettings = interpreterSettings.values();
- for (final InterpreterSetting intpSetting : intpSettings) {
- Thread t = new Thread() {
- public void run() {
- intpSetting.closeAndRemoveAllInterpreterGroups();
- }
- };
- t.start();
- closeThreads.add(t);
- }
- }
-
- for (Thread t : closeThreads) {
- try {
- t.join();
- } catch (InterruptedException e) {
- logger.error("Can't close interpreterGroup", e);
- }
- }
- }
-
- public void shutdown() {
- List<Thread> closeThreads = new LinkedList<>();
- synchronized (interpreterSettings) {
- Collection<InterpreterSetting> intpSettings = interpreterSettings.values();
- for (final InterpreterSetting intpSetting : intpSettings) {
- Thread t = new Thread() {
- public void run() {
- intpSetting.shutdownAndRemoveAllInterpreterGroups();
- }
- };
- t.start();
- closeThreads.add(t);
- }
- }
-
- for (Thread t : closeThreads) {
- try {
- t.join();
- } catch (InterruptedException e) {
- logger.error("Can't close interpreterGroup", e);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java
deleted file mode 100644
index 3838f63..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.zeppelin.interpreter.install;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.log4j.ConsoleAppender;
-import org.apache.log4j.Logger;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.dep.DependencyResolver;
-import org.apache.zeppelin.util.Util;
-import org.sonatype.aether.RepositoryException;
-import java.io.File;
-import java.io.IOException;
-import java.net.URL;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Commandline utility to install interpreter from maven repository
- */
-public class InstallInterpreter {
- private final File interpreterListFile;
- private final File interpreterBaseDir;
- private final List<AvailableInterpreterInfo> availableInterpreters;
- private final String localRepoDir;
- private URL proxyUrl;
- private String proxyUser;
- private String proxyPassword;
-
- /**
- *
- * @param interpreterListFile
- * @param interpreterBaseDir interpreter directory for installing binaries
- * @throws IOException
- */
- public InstallInterpreter(File interpreterListFile, File interpreterBaseDir, String localRepoDir)
- throws IOException {
- this.interpreterListFile = interpreterListFile;
- this.interpreterBaseDir = interpreterBaseDir;
- this.localRepoDir = localRepoDir;
- availableInterpreters = new LinkedList<>();
- readAvailableInterpreters();
- }
-
-
- /**
- * Information for available informations
- */
- private static class AvailableInterpreterInfo {
- public final String name;
- public final String artifact;
- public final String description;
-
- public AvailableInterpreterInfo(String name, String artifact, String description) {
- this.name = name;
- this.artifact = artifact;
- this.description = description;
- }
- }
-
- private void readAvailableInterpreters() throws IOException {
- if (!interpreterListFile.isFile()) {
- System.err.println("Can't find interpreter list " + interpreterListFile.getAbsolutePath());
- return;
- }
- String text = FileUtils.readFileToString(interpreterListFile);
- String[] lines = text.split("\n");
-
- Pattern pattern = Pattern.compile("(\\S+)\\s+(\\S+)\\s+(.*)");
-
- int lineNo = 0;
- for (String line : lines) {
- lineNo++;
- if (line == null || line.length() == 0 || line.startsWith("#")) {
- continue;
- }
-
- Matcher match = pattern.matcher(line);
- if (match.groupCount() != 3) {
- System.err.println("Error on line " + lineNo + ", " + line);
- continue;
- }
-
- match.find();
-
- String name = match.group(1);
- String artifact = match.group(2);
- String description = match.group(3);
-
- availableInterpreters.add(new AvailableInterpreterInfo(name, artifact, description));
- }
- }
-
- public List<AvailableInterpreterInfo> list() {
- for (AvailableInterpreterInfo info : availableInterpreters) {
- System.out.println(info.name + "\t\t\t" + info.description);
- }
-
- return availableInterpreters;
- }
-
- public void installAll() {
- for (AvailableInterpreterInfo info : availableInterpreters) {
- install(info.name, info.artifact);
- }
- }
-
- public void install(String [] names) {
- for (String name : names) {
- install(name);
- }
- }
-
- public void install(String name) {
- // find artifact name
- for (AvailableInterpreterInfo info : availableInterpreters) {
- if (name.equals(info.name)) {
- install(name, info.artifact);
- return;
- }
- }
-
- throw new RuntimeException("Can't find interpreter '" + name + "'");
- }
-
- public void install(String [] names, String [] artifacts) {
- if (names.length != artifacts.length) {
- throw new RuntimeException("Length of given names and artifacts are different");
- }
-
- for (int i = 0; i < names.length; i++) {
- install(names[i], artifacts[i]);
- }
- }
-
- public void install(String name, String artifact) {
- DependencyResolver depResolver = new DependencyResolver(localRepoDir);
- if (proxyUrl != null) {
- depResolver.setProxy(proxyUrl, proxyUser, proxyPassword);
- }
-
- File installDir = new File(interpreterBaseDir, name);
- if (installDir.exists()) {
- System.err.println("Directory " + installDir.getAbsolutePath()
- + " already exists"
- + "\n\nSkipped");
- return;
- }
-
- System.out.println("Install " + name + "(" + artifact + ") to "
- + installDir.getAbsolutePath() + " ... ");
-
- try {
- depResolver.load(artifact, installDir);
- System.out.println("Interpreter " + name + " installed under " +
- installDir.getAbsolutePath() + ".");
- startTip();
- } catch (RepositoryException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public void setProxy(URL proxyUrl, String proxyUser, String proxyPassword) {
- this.proxyUrl = proxyUrl;
- this.proxyUser = proxyUser;
- this.proxyPassword = proxyPassword;
- }
-
- public static void usage() {
- System.out.println("Options");
- System.out.println(" -l, --list List available interpreters");
- System.out.println(" -a, --all Install all available interpreters");
- System.out.println(" -n, --name [NAMES] Install interpreters (comma separated " +
- "list)" +
- "e.g. md,shell,jdbc,python,angular");
- System.out.println(" -t, --artifact [ARTIFACTS] (Optional with -n) custom artifact names" +
- ". " +
- "(comma separated list correspond to --name) " +
- "e.g. customGroup:customArtifact:customVersion");
- System.out.println(" --proxy-url [url] (Optional) proxy url. http(s)://host:port");
- System.out.println(" --proxy-user [user] (Optional) proxy user");
- System.out.println(" --proxy-password [password] (Optional) proxy password");
- }
-
- public static void main(String [] args) throws IOException {
- if (args.length == 0) {
- usage();
- return;
- }
-
- ZeppelinConfiguration conf = ZeppelinConfiguration.create();
- InstallInterpreter installer = new InstallInterpreter(
- new File(conf.getInterpreterListPath()),
- new File(conf.getInterpreterDir()),
- conf.getInterpreterLocalRepoPath());
-
- String names = null;
- String artifacts = null;
- URL proxyUrl = null;
- String proxyUser = null;
- String proxyPassword = null;
- boolean all = false;
-
- for (int i = 0; i < args.length; i++) {
- String arg = args[i].toLowerCase(Locale.US);
- switch (arg) {
- case "--list":
- case "-l":
- installer.list();
- System.exit(0);
- break;
- case "--all":
- case "-a":
- all = true;
- break;
- case "--name":
- case "-n":
- names = args[++i];
- break;
- case "--artifact":
- case "-t":
- artifacts = args[++i];
- break;
- case "--version":
- case "-v":
- Util.getVersion();
- break;
- case "--proxy-url":
- proxyUrl = new URL(args[++i]);
- break;
- case "--proxy-user":
- proxyUser = args[++i];
- break;
- case "--proxy-password":
- proxyPassword = args[++i];
- break;
- case "--help":
- case "-h":
- usage();
- System.exit(0);
- break;
- default:
- System.out.println("Unknown option " + arg);
- }
- }
-
- if (proxyUrl != null) {
- installer.setProxy(proxyUrl, proxyUser, proxyPassword);
- }
-
- if (all) {
- installer.installAll();
- System.exit(0);
- }
-
- if (names != null) {
- if (artifacts != null) {
- installer.install(names.split(","), artifacts.split(","));
- } else {
- installer.install(names.split(","));
- }
- }
- }
-
- private static void startTip() {
- System.out.println("\n1. Restart Zeppelin"
- + "\n2. Create interpreter setting in 'Interpreter' menu on Zeppelin GUI"
- + "\n3. Then you can bind the interpreter on your note");
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java
deleted file mode 100644
index 0ac7116..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote;
-
-import java.util.List;
-
-import org.apache.thrift.TException;
-import org.apache.zeppelin.display.AngularObject;
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.display.AngularObjectRegistryListener;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.gson.Gson;
-
-/**
- * Proxy for AngularObjectRegistry that exists in remote interpreter process
- */
-public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
- Logger logger = LoggerFactory.getLogger(RemoteAngularObjectRegistry.class);
- private InterpreterGroup interpreterGroup;
-
- public RemoteAngularObjectRegistry(String interpreterId,
- AngularObjectRegistryListener listener,
- InterpreterGroup interpreterGroup) {
- super(interpreterId, listener);
- this.interpreterGroup = interpreterGroup;
- }
-
- private RemoteInterpreterProcess getRemoteInterpreterProcess() {
- return interpreterGroup.getRemoteInterpreterProcess();
- }
-
- /**
- * When ZeppelinServer side code want to add angularObject to the registry,
- * this method should be used instead of add()
- * @param name
- * @param o
- * @param noteId
- * @return
- */
- public AngularObject addAndNotifyRemoteProcess(String name, Object o, String noteId, String
- paragraphId) {
- Gson gson = new Gson();
- RemoteInterpreterProcess remoteInterpreterProcess = getRemoteInterpreterProcess();
- if (!remoteInterpreterProcess.isRunning()) {
- return super.add(name, o, noteId, paragraphId, true);
- }
-
- Client client = null;
- boolean broken = false;
- try {
- client = remoteInterpreterProcess.getClient();
- client.angularObjectAdd(name, noteId, paragraphId, gson.toJson(o));
- return super.add(name, o, noteId, paragraphId, true);
- } catch (TException e) {
- broken = true;
- logger.error("Error", e);
- } catch (Exception e) {
- logger.error("Error", e);
- } finally {
- if (client != null) {
- remoteInterpreterProcess.releaseClient(client, broken);
- }
- }
- return null;
- }
-
- /**
- * When ZeppelinServer side code want to remove angularObject from the registry,
- * this method should be used instead of remove()
- * @param name
- * @param noteId
- * @param paragraphId
- * @return
- */
- public AngularObject removeAndNotifyRemoteProcess(String name, String noteId, String
- paragraphId) {
- RemoteInterpreterProcess remoteInterpreterProcess = getRemoteInterpreterProcess();
- if (remoteInterpreterProcess == null || !remoteInterpreterProcess.isRunning()) {
- return super.remove(name, noteId, paragraphId);
- }
-
- Client client = null;
- boolean broken = false;
- try {
- client = remoteInterpreterProcess.getClient();
- client.angularObjectRemove(name, noteId, paragraphId);
- return super.remove(name, noteId, paragraphId);
- } catch (TException e) {
- broken = true;
- logger.error("Error", e);
- } catch (Exception e) {
- logger.error("Error", e);
- } finally {
- if (client != null) {
- remoteInterpreterProcess.releaseClient(client, broken);
- }
- }
- return null;
- }
-
- public void removeAllAndNotifyRemoteProcess(String noteId, String paragraphId) {
- List<AngularObject> all = getAll(noteId, paragraphId);
- for (AngularObject ao : all) {
- removeAndNotifyRemoteProcess(ao.getName(), noteId, paragraphId);
- }
- }
-
- @Override
- protected AngularObject createNewAngularObject(String name, Object o, String noteId, String
- paragraphId) {
- return new RemoteAngularObject(name, o, noteId, paragraphId, interpreterGroup,
- getAngularObjectListener());
- }
-}
[06/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
index 3ddeec0..102ca1a 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
@@ -41,12 +41,7 @@ import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.display.Input;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.helium.HeliumPackage;
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterContextRunner;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.InterpreterResultMessage;
-import org.apache.zeppelin.interpreter.InterpreterSetting;
+import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
@@ -463,7 +458,8 @@ public class NotebookServer extends WebSocketServlet
Notebook notebook = notebook();
List<Note> notes = notebook.getAllNotes();
for (Note note : notes) {
- List<String> ids = notebook.getInterpreterSettingManager().getInterpreters(note.getId());
+ List<String> ids = notebook.getInterpreterSettingManager()
+ .getInterpreterBinding(note.getId());
for (String id : ids) {
if (id.equals(interpreterGroupId)) {
broadcast(note.getId(), m);
@@ -1003,7 +999,7 @@ public class NotebookServer extends WebSocketServlet
List<String> interpreterSettingIds = new LinkedList<>();
interpreterSettingIds.add(defaultInterpreterId);
for (String interpreterSettingId : notebook.getInterpreterSettingManager().
- getDefaultInterpreterSettingList()) {
+ getInterpreterSettingIds()) {
if (!interpreterSettingId.equals(defaultInterpreterId)) {
interpreterSettingIds.add(interpreterSettingId);
}
@@ -1363,12 +1359,13 @@ public class NotebookServer extends WebSocketServlet
List<InterpreterSetting> settings =
notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
- if (setting.getInterpreterGroup(user, note.getId()) == null) {
+ if (setting.getOrCreateInterpreterGroup(user, note.getId()) == null) {
continue;
}
- if (interpreterGroupId.equals(setting.getInterpreterGroup(user, note.getId()).getId())) {
+ if (interpreterGroupId.equals(setting.getOrCreateInterpreterGroup(user, note.getId())
+ .getId())) {
AngularObjectRegistry angularObjectRegistry =
- setting.getInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
+ setting.getOrCreateInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
// first trying to get local registry
ao = angularObjectRegistry.get(varName, noteId, paragraphId);
@@ -1405,12 +1402,13 @@ public class NotebookServer extends WebSocketServlet
List<InterpreterSetting> settings =
notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
- if (setting.getInterpreterGroup(user, n.getId()) == null) {
+ if (setting.getOrCreateInterpreterGroup(user, n.getId()) == null) {
continue;
}
- if (interpreterGroupId.equals(setting.getInterpreterGroup(user, n.getId()).getId())) {
+ if (interpreterGroupId.equals(setting.getOrCreateInterpreterGroup(user, n.getId())
+ .getId())) {
AngularObjectRegistry angularObjectRegistry =
- setting.getInterpreterGroup(user, n.getId()).getAngularObjectRegistry();
+ setting.getOrCreateInterpreterGroup(user, n.getId()).getAngularObjectRegistry();
this.broadcastExcept(n.getId(),
new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao)
.put("interpreterGroupId", interpreterGroupId).put("noteId", n.getId())
@@ -2283,13 +2281,13 @@ public class NotebookServer extends WebSocketServlet
for (InterpreterSetting intpSetting : settings) {
AngularObjectRegistry registry =
- intpSetting.getInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
+ intpSetting.getOrCreateInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
List<AngularObject> objects = registry.getAllWithGlobal(note.getId());
for (AngularObject object : objects) {
conn.send(serializeMessage(
new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", object)
.put("interpreterGroupId",
- intpSetting.getInterpreterGroup(user, note.getId()).getId())
+ intpSetting.getOrCreateInterpreterGroup(user, note.getId()).getId())
.put("noteId", note.getId()).put("paragraphId", object.getParagraphId())));
}
}
@@ -2335,7 +2333,7 @@ public class NotebookServer extends WebSocketServlet
}
List<String> settingIds =
- notebook.getInterpreterSettingManager().getInterpreters(note.getId());
+ notebook.getInterpreterSettingManager().getInterpreterBinding(note.getId());
for (String id : settingIds) {
if (interpreterGroupId.contains(id)) {
broadcast(note.getId(),
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-server/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java b/zeppelin-server/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
deleted file mode 100644
index 1b1306a..0000000
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.zeppelin.interpreter.mock;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
-import org.apache.zeppelin.scheduler.Scheduler;
-import org.apache.zeppelin.scheduler.SchedulerFactory;
-
-public class MockInterpreter1 extends Interpreter{
- Map<String, Object> vars = new HashMap<>();
-
- public MockInterpreter1(Properties property) {
- super(property);
- }
-
- @Override
- public void open() {
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public InterpreterResult interpret(String st, InterpreterContext context) {
- return new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl1: "+st);
- }
-
- @Override
- public void cancel(InterpreterContext context) {
- }
-
- @Override
- public FormType getFormType() {
- return FormType.SIMPLE;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-
- @Override
- public Scheduler getScheduler() {
- return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_"+this.hashCode());
- }
-
- @Override
- public List<InterpreterCompletion> completion(String buf, int cursor,
- InterpreterContext interpreterContext) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
index a7907db..e2f171f 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
@@ -307,10 +307,9 @@ public abstract class AbstractTestRestApi {
protected static void shutDown() throws Exception {
if (!wasRunning) {
// restart interpreter to stop all interpreter processes
- List<String> settingList = ZeppelinServer.notebook.getInterpreterSettingManager()
- .getDefaultInterpreterSettingList();
- for (String setting : settingList) {
- ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting);
+ List<InterpreterSetting> settingList = ZeppelinServer.notebook.getInterpreterSettingManager().get();
+ for (InterpreterSetting setting : settingList) {
+ ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting.getId());
}
if (shiroIni != null) {
FileUtils.deleteQuietly(shiroIni);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
index 28541bd..72dd8a7 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
@@ -80,7 +80,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
// then
assertThat(get, isAllowed());
- assertEquals(ZeppelinServer.notebook.getInterpreterSettingManager().getAvailableInterpreterSettings().size(),
+ assertEquals(ZeppelinServer.notebook.getInterpreterSettingManager().getInterpreterSettingTemplates().size(),
body.entrySet().size());
get.releaseConnection();
}
@@ -110,7 +110,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
@Test
public void testSettingsCRUD() throws IOException {
// when: call create setting API
- String rawRequest = "{\"name\":\"md2\",\"group\":\"md\"," +
+ String rawRequest = "{\"name\":\"md3\",\"group\":\"md\"," +
"\"properties\":{\"propname\": {\"value\": \"propvalue\", \"name\": \"propname\", \"type\": \"textarea\"}}," +
"\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
"\"dependencies\":[]," +
@@ -367,7 +367,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
@Test
public void testGetMetadataInfo() throws IOException {
- String jsonRequest = "{\"name\":\"spark\",\"group\":\"spark\"," +
+ String jsonRequest = "{\"name\":\"spark_new\",\"group\":\"spark\"," +
"\"properties\":{\"propname\": {\"value\": \"propvalue\", \"name\": \"propname\", \"type\": \"textarea\"}}," +
"\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
"\"dependencies\":[]," +
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
index 8da36a6..10d77b2 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
@@ -30,6 +30,7 @@ import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.notebook.socket.Message;
import org.apache.zeppelin.notebook.socket.Message.OP;
import org.apache.zeppelin.rest.AbstractTestRestApi;
+import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.server.ZeppelinServer;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.AfterClass;
@@ -95,7 +96,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
}
@Test
- public void testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent() throws IOException {
+ public void testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent() throws IOException, InterruptedException {
// create a notebook
Note note1 = notebook.createNote(anonymous);
@@ -104,7 +105,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
List<InterpreterSetting> settings = notebook.getInterpreterSettingManager().getInterpreterSettings(note1.getId());
for (InterpreterSetting setting : settings) {
if (setting.getName().equals("md")) {
- interpreterGroup = setting.getInterpreterGroup("anonymous", "sharedProcess");
+ interpreterGroup = setting.getOrCreateInterpreterGroup("anonymous", "sharedProcess");
break;
}
}
@@ -115,6 +116,14 @@ public class NotebookServerTest extends AbstractTestRestApi {
p1.setAuthenticationInfo(anonymous);
note1.run(p1.getId());
+ // wait for paragraph finished
+ while(true) {
+ if (p1.getStatus() == Job.Status.FINISHED) {
+ break;
+ }
+ Thread.sleep(100);
+ }
+
// add angularObject
interpreterGroup.getAngularObjectRegistry().add("object1", "value1", note1.getId(), null);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-server/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/resources/log4j.properties b/zeppelin-server/src/test/resources/log4j.properties
index b0d1067..8368993 100644
--- a/zeppelin-server/src/test/resources/log4j.properties
+++ b/zeppelin-server/src/test/resources/log4j.properties
@@ -33,7 +33,6 @@ log4j.logger.org.apache.hadoop.mapred=WARN
log4j.logger.org.apache.hadoop.hive.ql=WARN
log4j.logger.org.apache.hadoop.hive.metastore=WARN
log4j.logger.org.apache.haadoop.hive.service.HiveServer=WARN
-log4j.logger.org.apache.zeppelin.scheduler=WARN
log4j.logger.org.quartz=WARN
log4j.logger.DataNucleus=WARN
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/pom.xml
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml
index b3d5c63..3ae382a 100644
--- a/zeppelin-zengine/pom.xml
+++ b/zeppelin-zengine/pom.xml
@@ -61,18 +61,21 @@
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>zeppelin-interpreter</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
+ <artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
deleted file mode 100644
index f00fe93..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ /dev/null
@@ -1,835 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.conf;
-
-import java.io.File;
-import java.net.URL;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.XMLConfiguration;
-import org.apache.commons.configuration.tree.ConfigurationNode;
-import org.apache.commons.lang.StringUtils;
-import org.apache.zeppelin.notebook.repo.GitNotebookRepo;
-import org.apache.zeppelin.util.Util;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Zeppelin configuration.
- *
- */
-public class ZeppelinConfiguration extends XMLConfiguration {
- private static final String ZEPPELIN_SITE_XML = "zeppelin-site.xml";
- private static final long serialVersionUID = 4749305895693848035L;
- private static final Logger LOG = LoggerFactory.getLogger(ZeppelinConfiguration.class);
-
- private static final String HELIUM_PACKAGE_DEFAULT_URL =
- "https://s3.amazonaws.com/helium-package/helium.json";
- private static ZeppelinConfiguration conf;
-
- public ZeppelinConfiguration(URL url) throws ConfigurationException {
- setDelimiterParsingDisabled(true);
- load(url);
- }
-
- public ZeppelinConfiguration() {
- ConfVars[] vars = ConfVars.values();
- for (ConfVars v : vars) {
- if (v.getType() == ConfVars.VarType.BOOLEAN) {
- this.setProperty(v.getVarName(), v.getBooleanValue());
- } else if (v.getType() == ConfVars.VarType.LONG) {
- this.setProperty(v.getVarName(), v.getLongValue());
- } else if (v.getType() == ConfVars.VarType.INT) {
- this.setProperty(v.getVarName(), v.getIntValue());
- } else if (v.getType() == ConfVars.VarType.FLOAT) {
- this.setProperty(v.getVarName(), v.getFloatValue());
- } else if (v.getType() == ConfVars.VarType.STRING) {
- this.setProperty(v.getVarName(), v.getStringValue());
- } else {
- throw new RuntimeException("Unsupported VarType");
- }
- }
-
- }
-
-
- /**
- * Load from resource.
- *url = ZeppelinConfiguration.class.getResource(ZEPPELIN_SITE_XML);
- * @throws ConfigurationException
- */
- public static synchronized ZeppelinConfiguration create() {
- if (conf != null) {
- return conf;
- }
-
- ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
- URL url;
-
- url = ZeppelinConfiguration.class.getResource(ZEPPELIN_SITE_XML);
- if (url == null) {
- ClassLoader cl = ZeppelinConfiguration.class.getClassLoader();
- if (cl != null) {
- url = cl.getResource(ZEPPELIN_SITE_XML);
- }
- }
- if (url == null) {
- url = classLoader.getResource(ZEPPELIN_SITE_XML);
- }
-
- if (url == null) {
- LOG.warn("Failed to load configuration, proceeding with a default");
- conf = new ZeppelinConfiguration();
- } else {
- try {
- LOG.info("Load configuration from " + url);
- conf = new ZeppelinConfiguration(url);
- } catch (ConfigurationException e) {
- LOG.warn("Failed to load configuration from " + url + " proceeding with a default", e);
- conf = new ZeppelinConfiguration();
- }
- }
-
- LOG.info("Server Host: " + conf.getServerAddress());
- if (conf.useSsl() == false) {
- LOG.info("Server Port: " + conf.getServerPort());
- } else {
- LOG.info("Server SSL Port: " + conf.getServerSslPort());
- }
- LOG.info("Context Path: " + conf.getServerContextPath());
- LOG.info("Zeppelin Version: " + Util.getVersion());
-
- return conf;
- }
-
-
- private String getStringValue(String name, String d) {
- List<ConfigurationNode> properties = getRootNode().getChildren();
- if (properties == null || properties.isEmpty()) {
- return d;
- }
- for (ConfigurationNode p : properties) {
- if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
- && name.equals(p.getChildren("name").get(0).getValue())) {
- return (String) p.getChildren("value").get(0).getValue();
- }
- }
- return d;
- }
-
- private int getIntValue(String name, int d) {
- List<ConfigurationNode> properties = getRootNode().getChildren();
- if (properties == null || properties.isEmpty()) {
- return d;
- }
- for (ConfigurationNode p : properties) {
- if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
- && name.equals(p.getChildren("name").get(0).getValue())) {
- return Integer.parseInt((String) p.getChildren("value").get(0).getValue());
- }
- }
- return d;
- }
-
- private long getLongValue(String name, long d) {
- List<ConfigurationNode> properties = getRootNode().getChildren();
- if (properties == null || properties.isEmpty()) {
- return d;
- }
- for (ConfigurationNode p : properties) {
- if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
- && name.equals(p.getChildren("name").get(0).getValue())) {
- return Long.parseLong((String) p.getChildren("value").get(0).getValue());
- }
- }
- return d;
- }
-
- private float getFloatValue(String name, float d) {
- List<ConfigurationNode> properties = getRootNode().getChildren();
- if (properties == null || properties.isEmpty()) {
- return d;
- }
- for (ConfigurationNode p : properties) {
- if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
- && name.equals(p.getChildren("name").get(0).getValue())) {
- return Float.parseFloat((String) p.getChildren("value").get(0).getValue());
- }
- }
- return d;
- }
-
- private boolean getBooleanValue(String name, boolean d) {
- List<ConfigurationNode> properties = getRootNode().getChildren();
- if (properties == null || properties.isEmpty()) {
- return d;
- }
- for (ConfigurationNode p : properties) {
- if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
- && name.equals(p.getChildren("name").get(0).getValue())) {
- return Boolean.parseBoolean((String) p.getChildren("value").get(0).getValue());
- }
- }
- return d;
- }
-
- public String getString(ConfVars c) {
- return getString(c.name(), c.getVarName(), c.getStringValue());
- }
-
- public String getString(String envName, String propertyName, String defaultValue) {
- if (System.getenv(envName) != null) {
- return System.getenv(envName);
- }
- if (System.getProperty(propertyName) != null) {
- return System.getProperty(propertyName);
- }
-
- return getStringValue(propertyName, defaultValue);
- }
-
- public int getInt(ConfVars c) {
- return getInt(c.name(), c.getVarName(), c.getIntValue());
- }
-
- public int getInt(String envName, String propertyName, int defaultValue) {
- if (System.getenv(envName) != null) {
- return Integer.parseInt(System.getenv(envName));
- }
-
- if (System.getProperty(propertyName) != null) {
- return Integer.parseInt(System.getProperty(propertyName));
- }
- return getIntValue(propertyName, defaultValue);
- }
-
- public long getLong(ConfVars c) {
- return getLong(c.name(), c.getVarName(), c.getLongValue());
- }
-
- public long getLong(String envName, String propertyName, long defaultValue) {
- if (System.getenv(envName) != null) {
- return Long.parseLong(System.getenv(envName));
- }
-
- if (System.getProperty(propertyName) != null) {
- return Long.parseLong(System.getProperty(propertyName));
- }
- return getLongValue(propertyName, defaultValue);
- }
-
- public float getFloat(ConfVars c) {
- return getFloat(c.name(), c.getVarName(), c.getFloatValue());
- }
-
- public float getFloat(String envName, String propertyName, float defaultValue) {
- if (System.getenv(envName) != null) {
- return Float.parseFloat(System.getenv(envName));
- }
- if (System.getProperty(propertyName) != null) {
- return Float.parseFloat(System.getProperty(propertyName));
- }
- return getFloatValue(propertyName, defaultValue);
- }
-
- public boolean getBoolean(ConfVars c) {
- return getBoolean(c.name(), c.getVarName(), c.getBooleanValue());
- }
-
- public boolean getBoolean(String envName, String propertyName, boolean defaultValue) {
- if (System.getenv(envName) != null) {
- return Boolean.parseBoolean(System.getenv(envName));
- }
-
- if (System.getProperty(propertyName) != null) {
- return Boolean.parseBoolean(System.getProperty(propertyName));
- }
- return getBooleanValue(propertyName, defaultValue);
- }
-
- public boolean useSsl() {
- return getBoolean(ConfVars.ZEPPELIN_SSL);
- }
-
- public int getServerSslPort() {
- return getInt(ConfVars.ZEPPELIN_SSL_PORT);
- }
-
- public boolean useClientAuth() {
- return getBoolean(ConfVars.ZEPPELIN_SSL_CLIENT_AUTH);
- }
-
- public String getServerAddress() {
- return getString(ConfVars.ZEPPELIN_ADDR);
- }
-
- public int getServerPort() {
- return getInt(ConfVars.ZEPPELIN_PORT);
- }
-
- public String getServerContextPath() {
- return getString(ConfVars.ZEPPELIN_SERVER_CONTEXT_PATH);
- }
-
- public String getKeyStorePath() {
- String path = getString(ConfVars.ZEPPELIN_SSL_KEYSTORE_PATH);
- if (path != null && path.startsWith("/") || isWindowsPath(path)) {
- return path;
- } else {
- return getRelativeDir(
- String.format("%s/%s",
- getConfDir(),
- path));
- }
- }
-
- public String getKeyStoreType() {
- return getString(ConfVars.ZEPPELIN_SSL_KEYSTORE_TYPE);
- }
-
- public String getKeyStorePassword() {
- return getString(ConfVars.ZEPPELIN_SSL_KEYSTORE_PASSWORD);
- }
-
- public String getKeyManagerPassword() {
- String password = getString(ConfVars.ZEPPELIN_SSL_KEY_MANAGER_PASSWORD);
- if (password == null) {
- return getKeyStorePassword();
- } else {
- return password;
- }
- }
-
- public String getTrustStorePath() {
- String path = getString(ConfVars.ZEPPELIN_SSL_TRUSTSTORE_PATH);
- if (path == null) {
- path = getKeyStorePath();
- }
- if (path != null && path.startsWith("/") || isWindowsPath(path)) {
- return path;
- } else {
- return getRelativeDir(
- String.format("%s/%s",
- getConfDir(),
- path));
- }
- }
-
- public String getTrustStoreType() {
- String type = getString(ConfVars.ZEPPELIN_SSL_TRUSTSTORE_TYPE);
- if (type == null) {
- return getKeyStoreType();
- } else {
- return type;
- }
- }
-
- public String getTrustStorePassword() {
- String password = getString(ConfVars.ZEPPELIN_SSL_TRUSTSTORE_PASSWORD);
- if (password == null) {
- return getKeyStorePassword();
- } else {
- return password;
- }
- }
-
- public String getNotebookDir() {
- return getString(ConfVars.ZEPPELIN_NOTEBOOK_DIR);
- }
-
- public String getUser() {
- return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_USER);
- }
-
- public String getBucketName() {
- return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_BUCKET);
- }
-
- public String getEndpoint() {
- return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_ENDPOINT);
- }
-
- public String getS3KMSKeyID() {
- return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID);
- }
-
- public String getS3KMSKeyRegion() {
- return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION);
- }
-
- public String getS3EncryptionMaterialsProviderClass() {
- return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_EMP);
- }
-
- public boolean isS3ServerSideEncryption() {
- return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_S3_SSE);
- }
-
- public String getMongoUri() {
- return getString(ConfVars.ZEPPELIN_NOTEBOOK_MONGO_URI);
- }
-
- public String getMongoDatabase() {
- return getString(ConfVars.ZEPPELIN_NOTEBOOK_MONGO_DATABASE);
- }
-
- public String getMongoCollection() {
- return getString(ConfVars.ZEPPELIN_NOTEBOOK_MONGO_COLLECTION);
- }
-
- public boolean getMongoAutoimport() {
- return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT);
- }
-
- public String getInterpreterListPath() {
- return getRelativeDir(String.format("%s/interpreter-list", getConfDir()));
- }
-
- public String getInterpreterDir() {
- return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_DIR);
- }
-
- public String getInterpreterJson() {
- return getString(ConfVars.ZEPPELIN_INTERPRETER_JSON);
- }
-
- public String getInterpreterSettingPath() {
- return getRelativeDir(String.format("%s/interpreter.json", getConfDir()));
- }
-
- public String getHeliumConfPath() {
- return getRelativeDir(String.format("%s/helium.json", getConfDir()));
- }
-
- public String getHeliumRegistry() {
- return getRelativeDir(ConfVars.ZEPPELIN_HELIUM_REGISTRY);
- }
-
- public String getHeliumNodeInstallerUrl() {
- return getString(ConfVars.ZEPPELIN_HELIUM_NODE_INSTALLER_URL);
- }
-
- public String getHeliumNpmInstallerUrl() {
- return getString(ConfVars.ZEPPELIN_HELIUM_NPM_INSTALLER_URL);
- }
-
- public String getHeliumYarnInstallerUrl() {
- return getString(ConfVars.ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL);
- }
-
- public String getNotebookAuthorizationPath() {
- return getRelativeDir(String.format("%s/notebook-authorization.json", getConfDir()));
- }
-
- public Boolean credentialsPersist() {
- return getBoolean(ConfVars.ZEPPELIN_CREDENTIALS_PERSIST);
- }
-
- public String getCredentialsPath() {
- return getRelativeDir(String.format("%s/credentials.json", getConfDir()));
- }
-
- public String getShiroPath() {
- String shiroPath = getRelativeDir(String.format("%s/shiro.ini", getConfDir()));
- return new File(shiroPath).exists() ? shiroPath : StringUtils.EMPTY;
- }
-
- public String getInterpreterRemoteRunnerPath() {
- return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER);
- }
-
- public String getInterpreterLocalRepoPath() {
- return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO);
- }
-
- public String getInterpreterMvnRepoPath() {
- return getString(ConfVars.ZEPPELIN_INTERPRETER_DEP_MVNREPO);
- }
-
- public String getRelativeDir(ConfVars c) {
- return getRelativeDir(getString(c));
- }
-
- public String getRelativeDir(String path) {
- if (path != null && path.startsWith("/") || isWindowsPath(path)) {
- return path;
- } else {
- return getString(ConfVars.ZEPPELIN_HOME) + "/" + path;
- }
- }
-
- public boolean isWindowsPath(String path){
- return path.matches("^[A-Za-z]:\\\\.*");
- }
-
- public boolean isAnonymousAllowed() {
- return getBoolean(ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED);
- }
-
- public boolean isNotebokPublic() {
- return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC);
- }
-
- public String getConfDir() {
- return getString(ConfVars.ZEPPELIN_CONF_DIR);
- }
-
- public List<String> getAllowedOrigins()
- {
- if (getString(ConfVars.ZEPPELIN_ALLOWED_ORIGINS).isEmpty()) {
- return Arrays.asList(new String[0]);
- }
-
- return Arrays.asList(getString(ConfVars.ZEPPELIN_ALLOWED_ORIGINS).toLowerCase().split(","));
- }
-
- public String getWebsocketMaxTextMessageSize() {
- return getString(ConfVars.ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE);
- }
-
- public String getJettyName() {
- return getString(ConfVars.ZEPPELIN_SERVER_JETTY_NAME);
- }
-
-
- public String getXFrameOptions() {
- return getString(ConfVars.ZEPPELIN_SERVER_XFRAME_OPTIONS);
- }
-
- public String getXxssProtection() {
- return getString(ConfVars.ZEPPELIN_SERVER_X_XSS_PROTECTION);
- }
-
- public String getStrictTransport() {
- return getString(ConfVars.ZEPPELIN_SERVER_STRICT_TRANSPORT);
- }
-
-
- public Map<String, String> dumpConfigurations(ZeppelinConfiguration conf,
- ConfigurationKeyPredicate predicate) {
- Map<String, String> configurations = new HashMap<>();
-
- for (ZeppelinConfiguration.ConfVars v : ZeppelinConfiguration.ConfVars.values()) {
- String key = v.getVarName();
-
- if (!predicate.apply(key)) {
- continue;
- }
-
- ConfVars.VarType type = v.getType();
- Object value = null;
- if (type == ConfVars.VarType.BOOLEAN) {
- value = conf.getBoolean(v);
- } else if (type == ConfVars.VarType.LONG) {
- value = conf.getLong(v);
- } else if (type == ConfVars.VarType.INT) {
- value = conf.getInt(v);
- } else if (type == ConfVars.VarType.FLOAT) {
- value = conf.getFloat(v);
- } else if (type == ConfVars.VarType.STRING) {
- value = conf.getString(v);
- }
-
- if (value != null) {
- configurations.put(key, value.toString());
- }
- }
- return configurations;
- }
-
- /**
- * Predication whether key/value pair should be included or not
- */
- public interface ConfigurationKeyPredicate {
- boolean apply(String key);
- }
-
- /**
- * Wrapper class.
- */
- public static enum ConfVars {
- ZEPPELIN_HOME("zeppelin.home", "./"),
- ZEPPELIN_ADDR("zeppelin.server.addr", "0.0.0.0"),
- ZEPPELIN_PORT("zeppelin.server.port", 8080),
- ZEPPELIN_SERVER_CONTEXT_PATH("zeppelin.server.context.path", "/"),
- ZEPPELIN_SSL("zeppelin.ssl", false),
- ZEPPELIN_SSL_PORT("zeppelin.server.ssl.port", 8443),
- ZEPPELIN_SSL_CLIENT_AUTH("zeppelin.ssl.client.auth", false),
- ZEPPELIN_SSL_KEYSTORE_PATH("zeppelin.ssl.keystore.path", "keystore"),
- ZEPPELIN_SSL_KEYSTORE_TYPE("zeppelin.ssl.keystore.type", "JKS"),
- ZEPPELIN_SSL_KEYSTORE_PASSWORD("zeppelin.ssl.keystore.password", ""),
- ZEPPELIN_SSL_KEY_MANAGER_PASSWORD("zeppelin.ssl.key.manager.password", null),
- ZEPPELIN_SSL_TRUSTSTORE_PATH("zeppelin.ssl.truststore.path", null),
- ZEPPELIN_SSL_TRUSTSTORE_TYPE("zeppelin.ssl.truststore.type", null),
- ZEPPELIN_SSL_TRUSTSTORE_PASSWORD("zeppelin.ssl.truststore.password", null),
- ZEPPELIN_WAR("zeppelin.war", "zeppelin-web/dist"),
- ZEPPELIN_WAR_TEMPDIR("zeppelin.war.tempdir", "webapps"),
- ZEPPELIN_INTERPRETERS("zeppelin.interpreters", "org.apache.zeppelin.spark.SparkInterpreter,"
- + "org.apache.zeppelin.spark.PySparkInterpreter,"
- + "org.apache.zeppelin.rinterpreter.RRepl,"
- + "org.apache.zeppelin.rinterpreter.KnitR,"
- + "org.apache.zeppelin.spark.SparkRInterpreter,"
- + "org.apache.zeppelin.spark.SparkSqlInterpreter,"
- + "org.apache.zeppelin.spark.DepInterpreter,"
- + "org.apache.zeppelin.markdown.Markdown,"
- + "org.apache.zeppelin.angular.AngularInterpreter,"
- + "org.apache.zeppelin.shell.ShellInterpreter,"
- + "org.apache.zeppelin.livy.LivySparkInterpreter,"
- + "org.apache.zeppelin.livy.LivySparkSQLInterpreter,"
- + "org.apache.zeppelin.livy.LivyPySparkInterpreter,"
- + "org.apache.zeppelin.livy.LivyPySpark3Interpreter,"
- + "org.apache.zeppelin.livy.LivySparkRInterpreter,"
- + "org.apache.zeppelin.alluxio.AlluxioInterpreter,"
- + "org.apache.zeppelin.file.HDFSFileInterpreter,"
- + "org.apache.zeppelin.pig.PigInterpreter,"
- + "org.apache.zeppelin.pig.PigQueryInterpreter,"
- + "org.apache.zeppelin.flink.FlinkInterpreter,"
- + "org.apache.zeppelin.python.PythonInterpreter,"
- + "org.apache.zeppelin.python.PythonInterpreterPandasSql,"
- + "org.apache.zeppelin.python.PythonCondaInterpreter,"
- + "org.apache.zeppelin.python.PythonDockerInterpreter,"
- + "org.apache.zeppelin.ignite.IgniteInterpreter,"
- + "org.apache.zeppelin.ignite.IgniteSqlInterpreter,"
- + "org.apache.zeppelin.lens.LensInterpreter,"
- + "org.apache.zeppelin.cassandra.CassandraInterpreter,"
- + "org.apache.zeppelin.geode.GeodeOqlInterpreter,"
- + "org.apache.zeppelin.kylin.KylinInterpreter,"
- + "org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,"
- + "org.apache.zeppelin.scalding.ScaldingInterpreter,"
- + "org.apache.zeppelin.jdbc.JDBCInterpreter,"
- + "org.apache.zeppelin.hbase.HbaseInterpreter,"
- + "org.apache.zeppelin.bigquery.BigQueryInterpreter,"
- + "org.apache.zeppelin.beam.BeamInterpreter,"
- + "org.apache.zeppelin.scio.ScioInterpreter,"
- + "org.apache.zeppelin.groovy.GroovyInterpreter"
- ),
- ZEPPELIN_INTERPRETER_JSON("zeppelin.interpreter.setting", "interpreter-setting.json"),
- ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"),
- ZEPPELIN_INTERPRETER_LOCALREPO("zeppelin.interpreter.localRepo", "local-repo"),
- ZEPPELIN_INTERPRETER_DEP_MVNREPO("zeppelin.interpreter.dep.mvnRepo",
- "http://repo1.maven.org/maven2/"),
- ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000),
- ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10),
- ZEPPELIN_INTERPRETER_GROUP_ORDER("zeppelin.interpreter.group.order", "spark,md,angular,sh,"
- + "livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,"
- + "scalding,jdbc,hbase,bigquery,beam,pig,scio,groovy"),
- ZEPPELIN_INTERPRETER_OUTPUT_LIMIT("zeppelin.interpreter.output.limit", 1024 * 100),
- ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"),
- ZEPPELIN_NOTEBOOK_DIR("zeppelin.notebook.dir", "notebook"),
- // use specified notebook (id) as homescreen
- ZEPPELIN_NOTEBOOK_HOMESCREEN("zeppelin.notebook.homescreen", null),
- // whether homescreen notebook will be hidden from notebook list or not
- ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE("zeppelin.notebook.homescreen.hide", false),
- ZEPPELIN_NOTEBOOK_S3_BUCKET("zeppelin.notebook.s3.bucket", "zeppelin"),
- ZEPPELIN_NOTEBOOK_S3_ENDPOINT("zeppelin.notebook.s3.endpoint", "s3.amazonaws.com"),
- ZEPPELIN_NOTEBOOK_S3_USER("zeppelin.notebook.s3.user", "user"),
- ZEPPELIN_NOTEBOOK_S3_EMP("zeppelin.notebook.s3.encryptionMaterialsProvider", null),
- ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID("zeppelin.notebook.s3.kmsKeyID", null),
- ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION("zeppelin.notebook.s3.kmsKeyRegion", null),
- ZEPPELIN_NOTEBOOK_S3_SSE("zeppelin.notebook.s3.sse", false),
- ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING("zeppelin.notebook.azure.connectionString", null),
- ZEPPELIN_NOTEBOOK_AZURE_SHARE("zeppelin.notebook.azure.share", "zeppelin"),
- ZEPPELIN_NOTEBOOK_AZURE_USER("zeppelin.notebook.azure.user", "user"),
- ZEPPELIN_NOTEBOOK_MONGO_DATABASE("zeppelin.notebook.mongo.database", "zeppelin"),
- ZEPPELIN_NOTEBOOK_MONGO_COLLECTION("zeppelin.notebook.mongo.collection", "notes"),
- ZEPPELIN_NOTEBOOK_MONGO_URI("zeppelin.notebook.mongo.uri", "mongodb://localhost"),
- ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT("zeppelin.notebook.mongo.autoimport", false),
- ZEPPELIN_NOTEBOOK_STORAGE("zeppelin.notebook.storage", GitNotebookRepo.class.getName()),
- ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC("zeppelin.notebook.one.way.sync", false),
- // whether by default note is public or private
- ZEPPELIN_NOTEBOOK_PUBLIC("zeppelin.notebook.public", true),
- ZEPPELIN_INTERPRETER_REMOTE_RUNNER("zeppelin.interpreter.remoterunner",
- System.getProperty("os.name")
- .startsWith("Windows") ? "bin/interpreter.cmd" : "bin/interpreter.sh"),
- // Decide when new note is created, interpreter settings will be binded automatically or not.
- ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING("zeppelin.notebook.autoInterpreterBinding", true),
- ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"),
- ZEPPELIN_DEP_LOCALREPO("zeppelin.dep.localrepo", "local-repo"),
- ZEPPELIN_HELIUM_REGISTRY("zeppelin.helium.registry", "helium," + HELIUM_PACKAGE_DEFAULT_URL),
- ZEPPELIN_HELIUM_NODE_INSTALLER_URL("zeppelin.helium.node.installer.url",
- "https://nodejs.org/dist/"),
- ZEPPELIN_HELIUM_NPM_INSTALLER_URL("zeppelin.helium.npm.installer.url",
- "http://registry.npmjs.org/"),
- ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL("zeppelin.helium.yarnpkg.installer.url",
- "https://github.com/yarnpkg/yarn/releases/download/"),
- // Allows a way to specify a ',' separated list of allowed origins for rest and websockets
- // i.e. http://localhost:8080
- ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"),
- ZEPPELIN_ANONYMOUS_ALLOWED("zeppelin.anonymous.allowed", true),
- ZEPPELIN_CREDENTIALS_PERSIST("zeppelin.credentials.persist", true),
- ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE("zeppelin.websocket.max.text.message.size", "1024000"),
- ZEPPELIN_SERVER_DEFAULT_DIR_ALLOWED("zeppelin.server.default.dir.allowed", false),
- ZEPPELIN_SERVER_XFRAME_OPTIONS("zeppelin.server.xframe.options", "SAMEORIGIN"),
- ZEPPELIN_SERVER_JETTY_NAME("zeppelin.server.jetty.name", null),
- ZEPPELIN_SERVER_STRICT_TRANSPORT("zeppelin.server.strict.transport", "max-age=631138519"),
- ZEPPELIN_SERVER_X_XSS_PROTECTION("zeppelin.server.xxss.protection", "1"),
-
- ZEPPELIN_HDFS_KEYTAB("zeppelin.hdfs.keytab", ""),
- ZEPPELIN_HDFS_PRINCIPAL("zeppelin.hdfs.principal", "");
-
- private String varName;
- @SuppressWarnings("rawtypes")
- private Class varClass;
- private String stringValue;
- private VarType type;
- private int intValue;
- private float floatValue;
- private boolean booleanValue;
- private long longValue;
-
-
- ConfVars(String varName, String varValue) {
- this.varName = varName;
- this.varClass = String.class;
- this.stringValue = varValue;
- this.intValue = -1;
- this.floatValue = -1;
- this.longValue = -1;
- this.booleanValue = false;
- this.type = VarType.STRING;
- }
-
- ConfVars(String varName, int intValue) {
- this.varName = varName;
- this.varClass = Integer.class;
- this.stringValue = null;
- this.intValue = intValue;
- this.floatValue = -1;
- this.longValue = -1;
- this.booleanValue = false;
- this.type = VarType.INT;
- }
-
- ConfVars(String varName, long longValue) {
- this.varName = varName;
- this.varClass = Integer.class;
- this.stringValue = null;
- this.intValue = -1;
- this.floatValue = -1;
- this.longValue = longValue;
- this.booleanValue = false;
- this.type = VarType.LONG;
- }
-
- ConfVars(String varName, float floatValue) {
- this.varName = varName;
- this.varClass = Float.class;
- this.stringValue = null;
- this.intValue = -1;
- this.longValue = -1;
- this.floatValue = floatValue;
- this.booleanValue = false;
- this.type = VarType.FLOAT;
- }
-
- ConfVars(String varName, boolean booleanValue) {
- this.varName = varName;
- this.varClass = Boolean.class;
- this.stringValue = null;
- this.intValue = -1;
- this.longValue = -1;
- this.floatValue = -1;
- this.booleanValue = booleanValue;
- this.type = VarType.BOOLEAN;
- }
-
- public String getVarName() {
- return varName;
- }
-
- @SuppressWarnings("rawtypes")
- public Class getVarClass() {
- return varClass;
- }
-
- public int getIntValue() {
- return intValue;
- }
-
- public long getLongValue() {
- return longValue;
- }
-
- public float getFloatValue() {
- return floatValue;
- }
-
- public String getStringValue() {
- return stringValue;
- }
-
- public boolean getBooleanValue() {
- return booleanValue;
- }
-
- public VarType getType() {
- return type;
- }
-
- enum VarType {
- STRING {
- @Override
- void checkType(String value) throws Exception {}
- },
- INT {
- @Override
- void checkType(String value) throws Exception {
- Integer.valueOf(value);
- }
- },
- LONG {
- @Override
- void checkType(String value) throws Exception {
- Long.valueOf(value);
- }
- },
- FLOAT {
- @Override
- void checkType(String value) throws Exception {
- Float.valueOf(value);
- }
- },
- BOOLEAN {
- @Override
- void checkType(String value) throws Exception {
- Boolean.valueOf(value);
- }
- };
-
- boolean isType(String value) {
- try {
- checkType(value);
- } catch (Exception e) {
- LOG.error("Exception in ZeppelinConfiguration while isType", e);
- return false;
- }
- return true;
- }
-
- String typeString() {
- return name().toUpperCase();
- }
-
- abstract void checkType(String value) throws Exception;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
index 17a3529..5eecd6b 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
@@ -19,11 +19,12 @@ package org.apache.zeppelin.helium;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterSettingManager;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
import org.apache.zeppelin.notebook.Paragraph;
-import org.apache.zeppelin.resource.DistributedResourcePool;
-import org.apache.zeppelin.resource.ResourcePool;
-import org.apache.zeppelin.resource.ResourcePoolUtils;
-import org.apache.zeppelin.resource.ResourceSet;
+import org.apache.zeppelin.resource.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,19 +48,22 @@ public class Helium {
private final HeliumBundleFactory bundleFactory;
private final HeliumApplicationFactory applicationFactory;
+ private final InterpreterSettingManager interpreterSettingManager;
public Helium(
String heliumConfPath,
String registryPaths,
File registryCacheDir,
HeliumBundleFactory bundleFactory,
- HeliumApplicationFactory applicationFactory)
+ HeliumApplicationFactory applicationFactory,
+ InterpreterSettingManager interpreterSettingManager)
throws IOException {
this.heliumConfPath = heliumConfPath;
this.registryPaths = registryPaths;
this.registryCacheDir = registryCacheDir;
this.bundleFactory = bundleFactory;
this.applicationFactory = applicationFactory;
+ this.interpreterSettingManager = interpreterSettingManager;
heliumConf = loadConf(heliumConfPath);
allPackages = getAllPackageInfo();
}
@@ -350,7 +354,7 @@ public class Helium {
allResources = resourcePool.getAll();
}
} else {
- allResources = ResourcePoolUtils.getAllResources();
+ allResources = interpreterSettingManager.getAllResources();
}
for (List<HeliumPackageSearchResult> pkgs : allPackages.values()) {
@@ -478,4 +482,39 @@ public class Helium {
return mixed;
}
+
+ public ResourceSet getAllResources() {
+ return getAllResourcesExcept(null);
+ }
+
+ private ResourceSet getAllResourcesExcept(String interpreterGroupExcludsion) {
+ ResourceSet resourceSet = new ResourceSet();
+ for (InterpreterGroup intpGroup : interpreterSettingManager.getAllInterpreterGroup()) {
+ if (interpreterGroupExcludsion != null &&
+ intpGroup.getId().equals(interpreterGroupExcludsion)) {
+ continue;
+ }
+
+ RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
+ if (remoteInterpreterProcess == null) {
+ ResourcePool localPool = intpGroup.getResourcePool();
+ if (localPool != null) {
+ resourceSet.addAll(localPool.getAll());
+ }
+ } else if (remoteInterpreterProcess.isRunning()) {
+ List<String> resourceList = remoteInterpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<List<String>>() {
+ @Override
+ public List<String> call(RemoteInterpreterService.Client client) throws Exception {
+ return client.resourcePoolGetAll();
+ }
+ }
+ );
+ for (String res : resourceList) {
+ resourceSet.add(Resource.fromJson(res));
+ }
+ }
+ }
+ return resourceSet;
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java
index 84368a7..5f5330c 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java
@@ -105,38 +105,33 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
private void load(RemoteInterpreterProcess intpProcess, ApplicationState appState)
throws Exception {
- RemoteInterpreterService.Client client = null;
-
synchronized (appState) {
if (appState.getStatus() == ApplicationState.Status.LOADED) {
// already loaded
return;
}
- try {
- appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADING);
- String pkgInfo = pkg.toJson();
- String appId = appState.getId();
-
- client = intpProcess.getClient();
- RemoteApplicationResult ret = client.loadApplication(
- appId,
- pkgInfo,
- paragraph.getNote().getId(),
- paragraph.getId());
-
- if (ret.isSuccess()) {
- appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADED);
- } else {
- throw new ApplicationException(ret.getMsg());
- }
- } catch (TException e) {
- intpProcess.releaseBrokenClient(client);
- throw e;
- } finally {
- if (client != null) {
- intpProcess.releaseClient(client);
- }
+ appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADING);
+ final String pkgInfo = pkg.toJson();
+ final String appId = appState.getId();
+
+ RemoteApplicationResult ret = intpProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<RemoteApplicationResult>() {
+ @Override
+ public RemoteApplicationResult call(RemoteInterpreterService.Client client)
+ throws Exception {
+ return client.loadApplication(
+ appId,
+ pkgInfo,
+ paragraph.getNote().getId(),
+ paragraph.getId());
+ }
+ }
+ );
+ if (ret.isSuccess()) {
+ appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADED);
+ } else {
+ throw new ApplicationException(ret.getMsg());
}
}
}
@@ -199,7 +194,7 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
}
}
- private void unload(ApplicationState appsToUnload) throws ApplicationException {
+ private void unload(final ApplicationState appsToUnload) throws ApplicationException {
synchronized (appsToUnload) {
if (appsToUnload.getStatus() != ApplicationState.Status.LOADED) {
throw new ApplicationException(
@@ -217,26 +212,19 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
throw new ApplicationException("Target interpreter process is not running");
}
- RemoteInterpreterService.Client client;
- try {
- client = intpProcess.getClient();
- } catch (Exception e) {
- throw new ApplicationException(e);
- }
-
- try {
- RemoteApplicationResult ret = client.unloadApplication(appsToUnload.getId());
-
- if (ret.isSuccess()) {
- appStatusChange(paragraph, appsToUnload.getId(), ApplicationState.Status.UNLOADED);
- } else {
- throw new ApplicationException(ret.getMsg());
- }
- } catch (TException e) {
- intpProcess.releaseBrokenClient(client);
- throw new ApplicationException(e);
- } finally {
- intpProcess.releaseClient(client);
+ RemoteApplicationResult ret = intpProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<RemoteApplicationResult>() {
+ @Override
+ public RemoteApplicationResult call(RemoteInterpreterService.Client client)
+ throws Exception {
+ return client.unloadApplication(appsToUnload.getId());
+ }
+ }
+ );
+ if (ret.isSuccess()) {
+ appStatusChange(paragraph, appsToUnload.getId(), ApplicationState.Status.UNLOADED);
+ } else {
+ throw new ApplicationException(ret.getMsg());
}
}
}
@@ -286,7 +274,7 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
}
}
- private void run(ApplicationState app) throws ApplicationException {
+ private void run(final ApplicationState app) throws ApplicationException {
synchronized (app) {
if (app.getStatus() != ApplicationState.Status.LOADED) {
throw new ApplicationException(
@@ -303,29 +291,19 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
if (intpProcess == null) {
throw new ApplicationException("Target interpreter process is not running");
}
- RemoteInterpreterService.Client client = null;
- try {
- client = intpProcess.getClient();
- } catch (Exception e) {
- throw new ApplicationException(e);
- }
-
- try {
- RemoteApplicationResult ret = client.runApplication(app.getId());
-
- if (ret.isSuccess()) {
- // success
- } else {
- throw new ApplicationException(ret.getMsg());
- }
- } catch (TException e) {
- intpProcess.releaseBrokenClient(client);
- client = null;
- throw new ApplicationException(e);
- } finally {
- if (client != null) {
- intpProcess.releaseClient(client);
- }
+ RemoteApplicationResult ret = intpProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<RemoteApplicationResult>() {
+ @Override
+ public RemoteApplicationResult call(RemoteInterpreterService.Client client)
+ throws Exception {
+ return client.runApplication(app.getId());
+ }
+ }
+ );
+ if (ret.isSuccess()) {
+ // success
+ } else {
+ throw new ApplicationException(ret.getMsg());
}
}
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
deleted file mode 100644
index 9403b4f..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
+++ /dev/null
@@ -1,423 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang.NullArgumentException;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
-import org.apache.zeppelin.dep.DependencyResolver;
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.display.AngularObjectRegistryListener;
-import org.apache.zeppelin.helium.ApplicationEventListener;
-import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.sonatype.aether.RepositoryException;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-/**
- * Manage interpreters.
- */
-public class InterpreterFactory implements InterpreterGroupFactory {
- private static final Logger logger = LoggerFactory.getLogger(InterpreterFactory.class);
-
- private Map<String, URLClassLoader> cleanCl =
- Collections.synchronizedMap(new HashMap<String, URLClassLoader>());
-
- private ZeppelinConfiguration conf;
-
- private final InterpreterSettingManager interpreterSettingManager;
- private AngularObjectRegistryListener angularObjectRegistryListener;
- private final RemoteInterpreterProcessListener remoteInterpreterProcessListener;
- private final ApplicationEventListener appEventListener;
-
- private boolean shiroEnabled;
-
- private Map<String, String> env = new HashMap<>();
-
- private Interpreter devInterpreter;
-
- public InterpreterFactory(ZeppelinConfiguration conf,
- AngularObjectRegistryListener angularObjectRegistryListener,
- RemoteInterpreterProcessListener remoteInterpreterProcessListener,
- ApplicationEventListener appEventListener, DependencyResolver depResolver,
- boolean shiroEnabled, InterpreterSettingManager interpreterSettingManager)
- throws InterpreterException, IOException, RepositoryException {
- this.conf = conf;
- this.angularObjectRegistryListener = angularObjectRegistryListener;
- this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
- this.appEventListener = appEventListener;
- this.shiroEnabled = shiroEnabled;
-
- this.interpreterSettingManager = interpreterSettingManager;
- //TODO(jl): Fix it not to use InterpreterGroupFactory
- interpreterSettingManager.setInterpreterGroupFactory(this);
-
- logger.info("shiroEnabled: {}", shiroEnabled);
- }
-
- /**
- * @param id interpreterGroup id. Combination of interpreterSettingId + noteId/userId/shared
- * depends on interpreter mode
- */
- @Override
- public InterpreterGroup createInterpreterGroup(String id, InterpreterOption option)
- throws InterpreterException, NullArgumentException {
-
- //When called from REST API without option we receive NPE
- if (option == null) {
- throw new NullArgumentException("option");
- }
-
- AngularObjectRegistry angularObjectRegistry;
-
- InterpreterGroup interpreterGroup = new InterpreterGroup(id);
- if (option.isRemote()) {
- angularObjectRegistry =
- new RemoteAngularObjectRegistry(id, angularObjectRegistryListener, interpreterGroup);
- } else {
- angularObjectRegistry = new AngularObjectRegistry(id, angularObjectRegistryListener);
-
- // TODO(moon) : create distributed resource pool for local interpreters and set
- }
-
- interpreterGroup.setAngularObjectRegistry(angularObjectRegistry);
- return interpreterGroup;
- }
-
- public void createInterpretersForNote(InterpreterSetting interpreterSetting, String user,
- String noteId, String interpreterSessionKey) {
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId);
- InterpreterOption option = interpreterSetting.getOption();
- Properties properties = interpreterSetting.getFlatProperties();
- // if interpreters are already there, wait until they're being removed
- synchronized (interpreterGroup) {
- long interpreterRemovalWaitStart = System.nanoTime();
- // interpreter process supposed to be terminated by RemoteInterpreterProcess.dereference()
- // in ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT msec. However, if termination of the process and
- // removal from interpreter group take too long, throw an error.
- long minTimeout = 10L * 1000 * 1000000; // 10 sec
- long interpreterRemovalWaitTimeout = Math.max(minTimeout,
- conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT) * 1000000L * 2);
- while (interpreterGroup.containsKey(interpreterSessionKey)) {
- if (System.nanoTime() - interpreterRemovalWaitStart > interpreterRemovalWaitTimeout) {
- throw new InterpreterException("Can not create interpreter");
- }
- try {
- interpreterGroup.wait(1000);
- } catch (InterruptedException e) {
- logger.debug(e.getMessage(), e);
- }
- }
- }
-
- logger.info("Create interpreter instance {} for note {}", interpreterSetting.getName(), noteId);
-
- List<InterpreterInfo> interpreterInfos = interpreterSetting.getInterpreterInfos();
- String path = interpreterSetting.getPath();
- InterpreterRunner runner = interpreterSetting.getInterpreterRunner();
- Interpreter interpreter;
- for (InterpreterInfo info : interpreterInfos) {
- if (option.isRemote()) {
- if (option.isExistingProcess()) {
- interpreter =
- connectToRemoteRepl(interpreterSessionKey, info.getClassName(), option.getHost(),
- option.getPort(), properties, interpreterSetting.getId(), user,
- option.isUserImpersonate);
- } else {
- interpreter = createRemoteRepl(path, interpreterSessionKey, info.getClassName(),
- properties, interpreterSetting.getId(), user, option.isUserImpersonate(), runner);
- }
- } else {
- interpreter = createRepl(interpreterSetting.getPath(), info.getClassName(), properties);
- }
-
- synchronized (interpreterGroup) {
- List<Interpreter> interpreters = interpreterGroup.get(interpreterSessionKey);
- if (null == interpreters) {
- interpreters = new ArrayList<>();
- interpreterGroup.put(interpreterSessionKey, interpreters);
- }
- if (info.isDefaultInterpreter()) {
- interpreters.add(0, interpreter);
- } else {
- interpreters.add(interpreter);
- }
- }
- logger.info("Interpreter {} {} created", interpreter.getClassName(), interpreter.hashCode());
- interpreter.setInterpreterGroup(interpreterGroup);
- }
- }
-
- private Interpreter createRepl(String dirName, String className, Properties property)
- throws InterpreterException {
- logger.info("Create repl {} from {}", className, dirName);
-
- ClassLoader oldcl = Thread.currentThread().getContextClassLoader();
- try {
-
- URLClassLoader ccl = cleanCl.get(dirName);
- if (ccl == null) {
- // classloader fallback
- ccl = URLClassLoader.newInstance(new URL[]{}, oldcl);
- }
-
- boolean separateCL = true;
- try { // check if server's classloader has driver already.
- Class cls = this.getClass().forName(className);
- if (cls != null) {
- separateCL = false;
- }
- } catch (Exception e) {
- logger.error("exception checking server classloader driver", e);
- }
-
- URLClassLoader cl;
-
- if (separateCL == true) {
- cl = URLClassLoader.newInstance(new URL[]{}, ccl);
- } else {
- cl = ccl;
- }
- Thread.currentThread().setContextClassLoader(cl);
-
- Class<Interpreter> replClass = (Class<Interpreter>) cl.loadClass(className);
- Constructor<Interpreter> constructor =
- replClass.getConstructor(new Class[]{Properties.class});
- Interpreter repl = constructor.newInstance(property);
- repl.setClassloaderUrls(ccl.getURLs());
- LazyOpenInterpreter intp = new LazyOpenInterpreter(new ClassloaderInterpreter(repl, cl));
- return intp;
- } catch (SecurityException e) {
- throw new InterpreterException(e);
- } catch (NoSuchMethodException e) {
- throw new InterpreterException(e);
- } catch (IllegalArgumentException e) {
- throw new InterpreterException(e);
- } catch (InstantiationException e) {
- throw new InterpreterException(e);
- } catch (IllegalAccessException e) {
- throw new InterpreterException(e);
- } catch (InvocationTargetException e) {
- throw new InterpreterException(e);
- } catch (ClassNotFoundException e) {
- throw new InterpreterException(e);
- } finally {
- Thread.currentThread().setContextClassLoader(oldcl);
- }
- }
-
- private Interpreter connectToRemoteRepl(String interpreterSessionKey, String className,
- String host, int port, Properties property, String interpreterSettingId, String userName,
- Boolean isUserImpersonate) {
- int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
- int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE);
- String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + interpreterSettingId;
- LazyOpenInterpreter intp = new LazyOpenInterpreter(
- new RemoteInterpreter(property, interpreterSessionKey, className, host, port, localRepoPath,
- connectTimeout, maxPoolSize, remoteInterpreterProcessListener, appEventListener,
- userName, isUserImpersonate, conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT)));
- return intp;
- }
-
- Interpreter createRemoteRepl(String interpreterPath, String interpreterSessionKey,
- String className, Properties property, String interpreterSettingId,
- String userName, Boolean isUserImpersonate, InterpreterRunner interpreterRunner) {
- int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
- String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + interpreterSettingId;
- int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE);
- String interpreterRunnerPath;
- String interpreterGroupName = interpreterSettingManager.get(interpreterSettingId).getName();
- if (null != interpreterRunner) {
- interpreterRunnerPath = interpreterRunner.getPath();
- Path p = Paths.get(interpreterRunnerPath);
- if (!p.isAbsolute()) {
- interpreterRunnerPath = Joiner.on(File.separator)
- .join(interpreterPath, interpreterRunnerPath);
- }
- } else {
- interpreterRunnerPath = conf.getInterpreterRemoteRunnerPath();
- }
-
- RemoteInterpreter remoteInterpreter =
- new RemoteInterpreter(property, interpreterSessionKey, className,
- interpreterRunnerPath, interpreterPath, localRepoPath, connectTimeout, maxPoolSize,
- remoteInterpreterProcessListener, appEventListener, userName, isUserImpersonate,
- conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT), interpreterGroupName);
- remoteInterpreter.addEnv(env);
-
- return new LazyOpenInterpreter(remoteInterpreter);
- }
-
- private List<Interpreter> createOrGetInterpreterList(String user, String noteId,
- InterpreterSetting setting) {
- InterpreterGroup interpreterGroup = setting.getInterpreterGroup(user, noteId);
- synchronized (interpreterGroup) {
- String interpreterSessionKey =
- interpreterSettingManager.getInterpreterSessionKey(user, noteId, setting);
- if (!interpreterGroup.containsKey(interpreterSessionKey)) {
- createInterpretersForNote(setting, user, noteId, interpreterSessionKey);
- }
- return interpreterGroup.get(interpreterSessionKey);
- }
- }
-
- private InterpreterSetting getInterpreterSettingByGroup(List<InterpreterSetting> settings,
- String group) {
- Preconditions.checkNotNull(group, "group should be not null");
-
- for (InterpreterSetting setting : settings) {
- if (group.equals(setting.getName())) {
- return setting;
- }
- }
- return null;
- }
-
- private String getInterpreterClassFromInterpreterSetting(InterpreterSetting setting,
- String name) {
- Preconditions.checkNotNull(name, "name should be not null");
-
- for (InterpreterInfo info : setting.getInterpreterInfos()) {
- String infoName = info.getName();
- if (null != info.getName() && name.equals(infoName)) {
- return info.getClassName();
- }
- }
- return null;
- }
-
- private Interpreter getInterpreter(String user, String noteId, InterpreterSetting setting,
- String name) {
- Preconditions.checkNotNull(noteId, "noteId should be not null");
- Preconditions.checkNotNull(setting, "setting should be not null");
- Preconditions.checkNotNull(name, "name should be not null");
-
- String className;
- if (null != (className = getInterpreterClassFromInterpreterSetting(setting, name))) {
- List<Interpreter> interpreterGroup = createOrGetInterpreterList(user, noteId, setting);
- for (Interpreter interpreter : interpreterGroup) {
- if (className.equals(interpreter.getClassName())) {
- return interpreter;
- }
- }
- }
- return null;
- }
-
- public Interpreter getInterpreter(String user, String noteId, String replName) {
- List<InterpreterSetting> settings = interpreterSettingManager.getInterpreterSettings(noteId);
- InterpreterSetting setting;
- Interpreter interpreter;
-
- if (settings == null || settings.size() == 0) {
- return null;
- }
-
- if (replName == null || replName.trim().length() == 0) {
- // get default settings (first available)
- // TODO(jl): Fix it in case of returning null
- InterpreterSetting defaultSettings = interpreterSettingManager
- .getDefaultInterpreterSetting(settings);
- return createOrGetInterpreterList(user, noteId, defaultSettings).get(0);
- }
-
- String[] replNameSplit = replName.split("\\.");
- if (replNameSplit.length == 2) {
- String group = null;
- String name = null;
- group = replNameSplit[0];
- name = replNameSplit[1];
-
- setting = getInterpreterSettingByGroup(settings, group);
-
- if (null != setting) {
- interpreter = getInterpreter(user, noteId, setting, name);
-
- if (null != interpreter) {
- return interpreter;
- }
- }
-
- throw new InterpreterException(replName + " interpreter not found");
-
- } else {
- // first assume replName is 'name' of interpreter. ('groupName' is ommitted)
- // search 'name' from first (default) interpreter group
- // TODO(jl): Handle with noteId to support defaultInterpreter per note.
- setting = interpreterSettingManager.getDefaultInterpreterSetting(settings);
-
- interpreter = getInterpreter(user, noteId, setting, replName);
-
- if (null != interpreter) {
- return interpreter;
- }
-
- // next, assume replName is 'group' of interpreter ('name' is ommitted)
- // search interpreter group and return first interpreter.
- setting = getInterpreterSettingByGroup(settings, replName);
-
- if (null != setting) {
- List<Interpreter> interpreters = createOrGetInterpreterList(user, noteId, setting);
- if (null != interpreters) {
- return interpreters.get(0);
- }
- }
-
- // Support the legacy way to use it
- for (InterpreterSetting s : settings) {
- if (s.getGroup().equals(replName)) {
- List<Interpreter> interpreters = createOrGetInterpreterList(user, noteId, s);
- if (null != interpreters) {
- return interpreters.get(0);
- }
- }
- }
- }
-
- return null;
- }
-
- public Map<String, String> getEnv() {
- return env;
- }
-
- public void setEnv(Map<String, String> env) {
- this.env = env;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroupFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroupFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroupFactory.java
deleted file mode 100644
index 3b9be40..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroupFactory.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.zeppelin.interpreter;
-
-import org.apache.commons.lang.NullArgumentException;
-
-/**
- * Created InterpreterGroup
- */
-public interface InterpreterGroupFactory {
- InterpreterGroup createInterpreterGroup(String interpreterGroupId, InterpreterOption option);
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java
deleted file mode 100644
index fd632ce..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter;
-
-import com.google.gson.annotations.SerializedName;
-
-import java.util.Map;
-
-/**
- * Information of interpreters in this interpreter setting.
- * this will be serialized for conf/interpreter.json and REST api response.
- */
-public class InterpreterInfo {
- private String name;
- @SerializedName("class") private String className;
- private boolean defaultInterpreter = false;
- private Map<String, Object> editor;
-
- public InterpreterInfo(String className, String name, boolean defaultInterpreter,
- Map<String, Object> editor) {
- this.className = className;
- this.name = name;
- this.defaultInterpreter = defaultInterpreter;
- this.editor = editor;
- }
-
- public String getName() {
- return name;
- }
-
- public String getClassName() {
- return className;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- boolean isDefaultInterpreter() {
- return defaultInterpreter;
- }
-
- public Map<String, Object> getEditor() {
- return editor;
- }
-
- public void setEditor(Map<String, Object> editor) {
- this.editor = editor;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (!(obj instanceof InterpreterInfo)) {
- return false;
- }
- InterpreterInfo other = (InterpreterInfo) obj;
-
- boolean sameName =
- null == getName() ? null == other.getName() : getName().equals(other.getName());
- boolean sameClassName = null == getClassName() ?
- null == other.getClassName() :
- getClassName().equals(other.getClassName());
- boolean sameIsDefaultInterpreter = defaultInterpreter == other.isDefaultInterpreter();
-
- return sameName && sameClassName && sameIsDefaultInterpreter;
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java
deleted file mode 100644
index ca688dc..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import org.apache.zeppelin.common.JsonSerializable;
-import org.sonatype.aether.repository.RemoteRepository;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- *
- */
-public class InterpreterInfoSaving implements JsonSerializable {
-
- private static final Gson gson = new GsonBuilder().setPrettyPrinting().create();
-
- public Map<String, InterpreterSetting> interpreterSettings;
- public Map<String, List<String>> interpreterBindings;
- public List<RemoteRepository> interpreterRepositories;
-
- public String toJson() {
- return gson.toJson(this);
- }
-
- public static InterpreterInfoSaving fromJson(String json) {
- return gson.fromJson(json, InterpreterInfoSaving.class);
- }
-}
[07/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
new file mode 100644
index 0000000..61e4ef0
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote;
+
+import org.apache.zeppelin.display.AngularObject;
+import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.display.AngularObjectRegistryListener;
+import org.apache.zeppelin.display.GUI;
+import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterAngular;
+import org.apache.zeppelin.resource.LocalResourcePool;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+
+public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
+ private static final String INTERPRETER_SCRIPT =
+ System.getProperty("os.name").startsWith("Windows") ?
+ "../bin/interpreter.cmd" :
+ "../bin/interpreter.sh";
+
+ private RemoteInterpreter intp;
+ private InterpreterContext context;
+ private RemoteAngularObjectRegistry localRegistry;
+ private InterpreterSetting interpreterSetting;
+
+ private AtomicInteger onAdd;
+ private AtomicInteger onUpdate;
+ private AtomicInteger onRemove;
+
+ @Before
+ public void setUp() throws Exception {
+ onAdd = new AtomicInteger(0);
+ onUpdate = new AtomicInteger(0);
+ onRemove = new AtomicInteger(0);
+
+ InterpreterOption interpreterOption = new InterpreterOption();
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(MockInterpreterAngular.class.getName(), "mock", true, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT);
+ interpreterSetting = new InterpreterSetting.Builder()
+ .setId("test")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .setRunner(runner)
+ .setInterpreterDir("../interpeters/test")
+ .create();
+
+ intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
+ localRegistry = (RemoteAngularObjectRegistry) intp.getInterpreterGroup().getAngularObjectRegistry();
+
+ context = new InterpreterContext(
+ "note",
+ "id",
+ null,
+ "title",
+ "text",
+ new AuthenticationInfo(),
+ new HashMap<String, Object>(),
+ new GUI(),
+ new AngularObjectRegistry(intp.getInterpreterGroup().getId(), null),
+ new LocalResourcePool("pool1"),
+ new LinkedList<InterpreterContextRunner>(), null);
+
+ intp.open();
+
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ interpreterSetting.close();
+ }
+
+ @Test
+ public void testAngularObjectInterpreterSideCRUD() throws InterruptedException {
+ InterpreterResult ret = intp.interpret("get", context);
+ Thread.sleep(500); // waitFor eventpoller pool event
+ String[] result = ret.message().get(0).getData().split(" ");
+ assertEquals("0", result[0]); // size of registry
+ assertEquals("0", result[1]); // num watcher called
+
+ // create object
+ ret = intp.interpret("add n1 v1", context);
+ Thread.sleep(500);
+ result = ret.message().get(0).getData().split(" ");
+ assertEquals("1", result[0]); // size of registry
+ assertEquals("0", result[1]); // num watcher called
+ assertEquals("v1", localRegistry.get("n1", "note", null).get());
+
+ // update object
+ ret = intp.interpret("update n1 v11", context);
+ result = ret.message().get(0).getData().split(" ");
+ Thread.sleep(500);
+ assertEquals("1", result[0]); // size of registry
+ assertEquals("1", result[1]); // num watcher called
+ assertEquals("v11", localRegistry.get("n1", "note", null).get());
+
+ // remove object
+ ret = intp.interpret("remove n1", context);
+ result = ret.message().get(0).getData().split(" ");
+ Thread.sleep(500);
+ assertEquals("0", result[0]); // size of registry
+ assertEquals("1", result[1]); // num watcher called
+ assertEquals(null, localRegistry.get("n1", "note", null));
+ }
+
+ @Test
+ public void testAngularObjectRemovalOnZeppelinServerSide() throws InterruptedException {
+ // test if angularobject removal from server side propagate to interpreter process's registry.
+ // will happen when notebook is removed.
+
+ InterpreterResult ret = intp.interpret("get", context);
+ Thread.sleep(500); // waitFor eventpoller pool event
+ String[] result = ret.message().get(0).getData().split(" ");
+ assertEquals("0", result[0]); // size of registry
+
+ // create object
+ ret = intp.interpret("add n1 v1", context);
+ Thread.sleep(500);
+ result = ret.message().get(0).getData().split(" ");
+ assertEquals("1", result[0]); // size of registry
+ assertEquals("v1", localRegistry.get("n1", "note", null).get());
+
+ // remove object in local registry.
+ localRegistry.removeAndNotifyRemoteProcess("n1", "note", null);
+ ret = intp.interpret("get", context);
+ Thread.sleep(500); // waitFor eventpoller pool event
+ result = ret.message().get(0).getData().split(" ");
+ assertEquals("0", result[0]); // size of registry
+ }
+
+ @Test
+ public void testAngularObjectAddOnZeppelinServerSide() throws InterruptedException {
+ // test if angularobject add from server side propagate to interpreter process's registry.
+ // will happen when zeppelin server loads notebook and restore the object into registry
+
+ InterpreterResult ret = intp.interpret("get", context);
+ Thread.sleep(500); // waitFor eventpoller pool event
+ String[] result = ret.message().get(0).getData().split(" ");
+ assertEquals("0", result[0]); // size of registry
+
+ // create object
+ localRegistry.addAndNotifyRemoteProcess("n1", "v1", "note", null);
+
+ // get from remote registry
+ ret = intp.interpret("get", context);
+ Thread.sleep(500); // waitFor eventpoller pool event
+ result = ret.message().get(0).getData().split(" ");
+ assertEquals("1", result[0]); // size of registry
+ }
+
+ @Override
+ public void onAdd(String interpreterGroupId, AngularObject object) {
+ onAdd.incrementAndGet();
+ }
+
+ @Override
+ public void onUpdate(String interpreterGroupId, AngularObject object) {
+ onUpdate.incrementAndGet();
+ }
+
+ @Override
+ public void onRemove(String interpreterGroupId, String name, String noteId, String paragraphId) {
+ onRemove.incrementAndGet();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPollerTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPollerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPollerTest.java
new file mode 100644
index 0000000..49aa7aa
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPollerTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote;
+
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
+import org.junit.Test;
+
+import static org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType.NO_OP;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class RemoteInterpreterEventPollerTest {
+
+ @Test
+ public void shouldClearUnreadEventsOnShutdown() throws Exception {
+ RemoteInterpreterProcess interpreterProc = getMockEventsInterpreterProcess();
+ RemoteInterpreterEventPoller eventPoller = new RemoteInterpreterEventPoller(null, null);
+
+ eventPoller.setInterpreterProcess(interpreterProc);
+ eventPoller.shutdown();
+ eventPoller.start();
+ eventPoller.join();
+
+ assertEquals(NO_OP, interpreterProc.getClient().getEvent().getType());
+ }
+
+ private RemoteInterpreterProcess getMockEventsInterpreterProcess() throws Exception {
+ RemoteInterpreterEvent fakeEvent = new RemoteInterpreterEvent();
+ RemoteInterpreterEvent noMoreEvents = new RemoteInterpreterEvent(NO_OP, "");
+ RemoteInterpreterService.Client client = mock(RemoteInterpreterService.Client.class);
+ RemoteInterpreterProcess intProc = mock(RemoteInterpreterProcess.class);
+
+ when(client.getEvent()).thenReturn(fakeEvent, fakeEvent, noMoreEvents);
+ when(intProc.getClient()).thenReturn(client);
+
+ return intProc;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
new file mode 100644
index 0000000..1687060
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote;
+
+import org.apache.zeppelin.display.GUI;
+import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterOutputStream;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.*;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+
+
+/**
+ * Test for remote interpreter output stream
+ */
+public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProcessListener {
+ private static final String INTERPRETER_SCRIPT =
+ System.getProperty("os.name").startsWith("Windows") ?
+ "../bin/interpreter.cmd" :
+ "../bin/interpreter.sh";
+
+ private InterpreterSetting interpreterSetting;
+
+ @Before
+ public void setUp() throws Exception {
+ InterpreterOption interpreterOption = new InterpreterOption();
+
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(MockInterpreterOutputStream.class.getName(), "mock", true, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT);
+ interpreterSetting = new InterpreterSetting.Builder()
+ .setId("test")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .setRunner(runner)
+ .setInterpreterDir("../interpeters/test")
+ .create();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ interpreterSetting.close();
+ }
+
+ private InterpreterContext createInterpreterContext() {
+ return new InterpreterContext(
+ "noteId",
+ "id",
+ null,
+ "title",
+ "text",
+ new AuthenticationInfo(),
+ new HashMap<String, Object>(),
+ new GUI(),
+ null,
+ null,
+ new LinkedList<InterpreterContextRunner>(), null);
+ }
+
+ @Test
+ public void testInterpreterResultOnly() {
+ RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
+ InterpreterResult ret = intp.interpret("SUCCESS::staticresult", createInterpreterContext());
+ assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
+ assertEquals("staticresult", ret.message().get(0).getData());
+
+ ret = intp.interpret("SUCCESS::staticresult2", createInterpreterContext());
+ assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
+ assertEquals("staticresult2", ret.message().get(0).getData());
+
+ ret = intp.interpret("ERROR::staticresult3", createInterpreterContext());
+ assertEquals(InterpreterResult.Code.ERROR, ret.code());
+ assertEquals("staticresult3", ret.message().get(0).getData());
+ }
+
+ @Test
+ public void testInterpreterOutputStreamOnly() {
+ RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
+ InterpreterResult ret = intp.interpret("SUCCESS:streamresult:", createInterpreterContext());
+ assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
+ assertEquals("streamresult", ret.message().get(0).getData());
+
+ ret = intp.interpret("ERROR:streamresult2:", createInterpreterContext());
+ assertEquals(InterpreterResult.Code.ERROR, ret.code());
+ assertEquals("streamresult2", ret.message().get(0).getData());
+ }
+
+ @Test
+ public void testInterpreterResultOutputStreamMixed() {
+ RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
+ InterpreterResult ret = intp.interpret("SUCCESS:stream:static", createInterpreterContext());
+ assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
+ assertEquals("stream", ret.message().get(0).getData());
+ assertEquals("static", ret.message().get(1).getData());
+ }
+
+ @Test
+ public void testOutputType() {
+ RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
+
+ InterpreterResult ret = intp.interpret("SUCCESS:%html hello:", createInterpreterContext());
+ assertEquals(InterpreterResult.Type.HTML, ret.message().get(0).getType());
+ assertEquals("hello", ret.message().get(0).getData());
+
+ ret = intp.interpret("SUCCESS:%html\nhello:", createInterpreterContext());
+ assertEquals(InterpreterResult.Type.HTML, ret.message().get(0).getType());
+ assertEquals("hello", ret.message().get(0).getData());
+
+ ret = intp.interpret("SUCCESS:%html hello:%angular world", createInterpreterContext());
+ assertEquals(InterpreterResult.Type.HTML, ret.message().get(0).getType());
+ assertEquals("hello", ret.message().get(0).getData());
+ assertEquals(InterpreterResult.Type.ANGULAR, ret.message().get(1).getType());
+ assertEquals("world", ret.message().get(1).getData());
+ }
+
+ @Override
+ public void onOutputAppend(String noteId, String paragraphId, int index, String output) {
+
+ }
+
+ @Override
+ public void onOutputUpdated(String noteId, String paragraphId, int index, InterpreterResult.Type type, String output) {
+
+ }
+
+ @Override
+ public void onOutputClear(String noteId, String paragraphId) {
+
+ }
+
+ @Override
+ public void onMetaInfosReceived(String settingId, Map<String, String> metaInfos) {
+
+ }
+
+ @Override
+ public void onGetParagraphRunners(String noteId, String paragraphId, RemoteWorksEventListener callback) {
+ if (callback != null) {
+ callback.onFinished(new LinkedList<>());
+ }
+ }
+
+ @Override
+ public void onRemoteRunParagraph(String noteId, String ParagraphID) throws Exception {
+
+ }
+
+ @Override
+ public void onParaInfosReceived(String noteId, String paragraphId,
+ String interpreterSettingId, Map<String, String> metaInfos) {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
new file mode 100644
index 0000000..ae98dc3
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
@@ -0,0 +1,520 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote;
+
+import org.apache.thrift.transport.TTransportException;
+import org.apache.zeppelin.display.GUI;
+import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
+import org.apache.zeppelin.interpreter.remote.mock.GetEnvPropertyInterpreter;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
+
+public class RemoteInterpreterTest {
+
+
+ private static final String INTERPRETER_SCRIPT =
+ System.getProperty("os.name").startsWith("Windows") ?
+ "../bin/interpreter.cmd" :
+ "../bin/interpreter.sh";
+
+ private InterpreterSetting interpreterSetting;
+
+ @Before
+ public void setUp() throws Exception {
+ InterpreterOption interpreterOption = new InterpreterOption();
+
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo3 = new InterpreterInfo(SleepInterpreter.class.getName(), "sleep", false, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo4 = new InterpreterInfo(GetEnvPropertyInterpreter.class.getName(), "get", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ interpreterInfos.add(interpreterInfo3);
+ interpreterInfos.add(interpreterInfo4);
+ InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT);
+ interpreterSetting = new InterpreterSetting.Builder()
+ .setId("test")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .setRunner(runner)
+ .setInterpreterDir("../interpeters/test")
+ .create();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ interpreterSetting.close();
+ }
+
+ @Test
+ public void testSharedMode() {
+ interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
+
+ Interpreter interpreter1 = interpreterSetting.getDefaultInterpreter("user1", "note1");
+ Interpreter interpreter2 = interpreterSetting.getDefaultInterpreter("user2", "note1");
+ assertTrue(interpreter1 instanceof RemoteInterpreter);
+ RemoteInterpreter remoteInterpreter1 = (RemoteInterpreter) interpreter1;
+ assertTrue(interpreter2 instanceof RemoteInterpreter);
+ RemoteInterpreter remoteInterpreter2 = (RemoteInterpreter) interpreter2;
+
+ InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
+ "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+ null, null, new ArrayList<InterpreterContextRunner>(), null);
+ assertEquals("hello", remoteInterpreter1.interpret("hello", context1).message().get(0).getData());
+ assertEquals(Interpreter.FormType.NATIVE, interpreter1.getFormType());
+ assertEquals(0, remoteInterpreter1.getProgress(context1));
+ assertNotNull(remoteInterpreter1.getOrCreateInterpreterProcess());
+ assertTrue(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess().isRunning());
+
+ assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
+ assertEquals(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess(),
+ remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess());
+
+ // Call InterpreterGroup.close instead of Interpreter.close, otherwise we will have the
+ // RemoteInterpreterProcess leakage.
+ remoteInterpreter1.getInterpreterGroup().close(remoteInterpreter1.getSessionId());
+ assertNull(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess());
+ try {
+ assertEquals("hello", remoteInterpreter1.interpret("hello", context1).message().get(0).getData());
+ fail("Should not be able to call interpret after interpreter is closed");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ try {
+ assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
+ fail("Should not be able to call getProgress after RemoterInterpreterProcess is stoped");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testScopedMode() {
+ interpreterSetting.getOption().setPerUser(InterpreterOption.SCOPED);
+
+ Interpreter interpreter1 = interpreterSetting.getDefaultInterpreter("user1", "note1");
+ Interpreter interpreter2 = interpreterSetting.getDefaultInterpreter("user2", "note1");
+ assertTrue(interpreter1 instanceof RemoteInterpreter);
+ RemoteInterpreter remoteInterpreter1 = (RemoteInterpreter) interpreter1;
+ assertTrue(interpreter2 instanceof RemoteInterpreter);
+ RemoteInterpreter remoteInterpreter2 = (RemoteInterpreter) interpreter2;
+
+ InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
+ "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+ null, null, new ArrayList<InterpreterContextRunner>(), null);
+ assertEquals("hello", remoteInterpreter1.interpret("hello", context1).message().get(0).getData());
+ assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
+ assertEquals(Interpreter.FormType.NATIVE, interpreter1.getFormType());
+ assertEquals(0, remoteInterpreter1.getProgress(context1));
+
+ assertNotNull(remoteInterpreter1.getOrCreateInterpreterProcess());
+ assertTrue(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess().isRunning());
+
+ assertEquals(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess(),
+ remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess());
+ // Call InterpreterGroup.close instead of Interpreter.close, otherwise we will have the
+ // RemoteInterpreterProcess leakage.
+ remoteInterpreter1.getInterpreterGroup().close(remoteInterpreter1.getSessionId());
+ try {
+ assertEquals("hello", remoteInterpreter1.interpret("hello", context1).message().get(0).getData());
+ fail("Should not be able to call interpret after interpreter is closed");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ assertTrue(remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess().isRunning());
+ assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
+ remoteInterpreter2.getInterpreterGroup().close(remoteInterpreter2.getSessionId());
+ try {
+ assertEquals("hello", remoteInterpreter2.interpret("hello", context1));
+ fail("Should not be able to call interpret after interpreter is closed");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ assertNull(remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess());
+ }
+
+ @Test
+ public void testIsolatedMode() {
+ interpreterSetting.getOption().setPerUser(InterpreterOption.ISOLATED);
+
+ Interpreter interpreter1 = interpreterSetting.getDefaultInterpreter("user1", "note1");
+ Interpreter interpreter2 = interpreterSetting.getDefaultInterpreter("user2", "note1");
+ assertTrue(interpreter1 instanceof RemoteInterpreter);
+ RemoteInterpreter remoteInterpreter1 = (RemoteInterpreter) interpreter1;
+ assertTrue(interpreter2 instanceof RemoteInterpreter);
+ RemoteInterpreter remoteInterpreter2 = (RemoteInterpreter) interpreter2;
+
+ InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
+ "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+ null, null, new ArrayList<InterpreterContextRunner>(), null);
+ assertEquals("hello", remoteInterpreter1.interpret("hello", context1).message().get(0).getData());
+ assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
+ assertEquals(Interpreter.FormType.NATIVE, interpreter1.getFormType());
+ assertEquals(0, remoteInterpreter1.getProgress(context1));
+ assertNotNull(remoteInterpreter1.getOrCreateInterpreterProcess());
+ assertTrue(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess().isRunning());
+
+ assertNotEquals(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess(),
+ remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess());
+ // Call InterpreterGroup.close instead of Interpreter.close, otherwise we will have the
+ // RemoteInterpreterProcess leakage.
+ remoteInterpreter1.getInterpreterGroup().close(remoteInterpreter1.getSessionId());
+ assertNull(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess());
+ assertTrue(remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess().isRunning());
+ try {
+ remoteInterpreter1.interpret("hello", context1);
+ fail("Should not be able to call getProgress after interpreter is closed");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
+ remoteInterpreter2.getInterpreterGroup().close(remoteInterpreter2.getSessionId());
+ try {
+ assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
+ fail("Should not be able to call interpret after interpreter is closed");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ assertNull(remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess());
+
+ }
+
+// @Test
+// public void testExecuteIncorrectPrecode() throws TTransportException, IOException {
+// interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
+// interpreterSetting.getProperties().setProperty("zeppelin.SleepInterpreter.precode", "fail test");
+//
+// Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "sleep");
+// InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
+// "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+// null, null, new ArrayList<InterpreterContextRunner>(), null);
+// assertEquals(Code.ERROR, interpreter1.interpret("10", context1).code());
+// }
+//
+// @Test
+// public void testExecuteCorrectPrecode() throws TTransportException, IOException {
+// interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
+// interpreterSetting.getProperties().setProperty("zeppelin.SleepInterpreter.precode", "1");
+//
+// Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "sleep");
+// InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
+// "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+// null, null, new ArrayList<InterpreterContextRunner>(), null);
+// assertEquals(Code.SUCCESS, interpreter1.interpret("10", context1).code());
+// }
+
+ @Test
+ public void testRemoteInterperterErrorStatus() throws TTransportException, IOException {
+ interpreterSetting.setProperty("zeppelin.interpreter.echo.fail", "true");
+ interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
+
+ Interpreter interpreter1 = interpreterSetting.getDefaultInterpreter("user1", "note1");
+ assertTrue(interpreter1 instanceof RemoteInterpreter);
+ RemoteInterpreter remoteInterpreter1 = (RemoteInterpreter) interpreter1;
+
+ InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
+ "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+ null, null, new ArrayList<InterpreterContextRunner>(), null);
+ assertEquals(Code.ERROR, remoteInterpreter1.interpret("hello", context1).code());
+ }
+
+ @Test
+ public void testFIFOScheduler() throws InterruptedException {
+ interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
+ // by default SleepInterpreter would use FIFOScheduler
+
+ final Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "sleep");
+ final InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
+ "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+ null, null, new ArrayList<InterpreterContextRunner>(), null);
+ // run this dummy interpret method first to launch the RemoteInterpreterProcess to avoid the
+ // time overhead of launching the process.
+ interpreter1.interpret("1", context1);
+ Thread thread1 = new Thread() {
+ @Override
+ public void run() {
+ assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code());
+ }
+ };
+ Thread thread2 = new Thread() {
+ @Override
+ public void run() {
+ assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code());
+ }
+ };
+ long start = System.currentTimeMillis();
+ thread1.start();
+ thread2.start();
+ thread1.join();
+ thread2.join();
+ long end = System.currentTimeMillis();
+ assertTrue((end - start) >= 200);
+ }
+
+ @Test
+ public void testParallelScheduler() throws InterruptedException {
+ interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
+ interpreterSetting.setProperty("zeppelin.SleepInterpreter.parallel", "true");
+
+ final Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "sleep");
+ final InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
+ "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+ null, null, new ArrayList<InterpreterContextRunner>(), null);
+
+ // run this dummy interpret method first to launch the RemoteInterpreterProcess to avoid the
+ // time overhead of launching the process.
+ interpreter1.interpret("1", context1);
+ Thread thread1 = new Thread() {
+ @Override
+ public void run() {
+ assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code());
+ }
+ };
+ Thread thread2 = new Thread() {
+ @Override
+ public void run() {
+ assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code());
+ }
+ };
+ long start = System.currentTimeMillis();
+ thread1.start();
+ thread2.start();
+ thread1.join();
+ thread2.join();
+ long end = System.currentTimeMillis();
+ assertTrue((end - start) <= 200);
+ }
+
+// @Test
+// public void testRunOrderPreserved() throws InterruptedException {
+// Properties p = new Properties();
+// intpGroup.put("note", new LinkedList<Interpreter>());
+//
+// final RemoteInterpreter intpA = createMockInterpreterA(p);
+//
+// intpGroup.get("note").add(intpA);
+// intpA.setInterpreterGroup(intpGroup);
+//
+// intpA.open();
+//
+// int concurrency = 3;
+// final List<InterpreterResultMessage> results = new LinkedList<>();
+//
+// Scheduler scheduler = intpA.getScheduler();
+// for (int i = 0; i < concurrency; i++) {
+// final String jobId = Integer.toString(i);
+// scheduler.submit(new Job(jobId, Integer.toString(i), null, 200) {
+// private Object r;
+//
+// @Override
+// public Object getReturn() {
+// return r;
+// }
+//
+// @Override
+// public void setResult(Object results) {
+// this.r = results;
+// }
+//
+// @Override
+// public int progress() {
+// return 0;
+// }
+//
+// @Override
+// public Map<String, Object> info() {
+// return null;
+// }
+//
+// @Override
+// protected Object jobRun() throws Throwable {
+// InterpreterResult ret = intpA.interpret(getJobName(), new InterpreterContext(
+// "note",
+// jobId,
+// null,
+// "title",
+// "text",
+// new AuthenticationInfo(),
+// new HashMap<String, Object>(),
+// new GUI(),
+// new AngularObjectRegistry(intpGroup.getId(), null),
+// new LocalResourcePool("pool1"),
+// new LinkedList<InterpreterContextRunner>(), null));
+//
+// synchronized (results) {
+// results.addAll(ret.message());
+// results.notify();
+// }
+// return null;
+// }
+//
+// @Override
+// protected boolean jobAbort() {
+// return false;
+// }
+//
+// });
+// }
+//
+// // wait for job finished
+// synchronized (results) {
+// while (results.size() != concurrency) {
+// results.wait(300);
+// }
+// }
+//
+// int i = 0;
+// for (InterpreterResultMessage result : results) {
+// assertEquals(Integer.toString(i++), result.getData());
+// }
+// assertEquals(concurrency, i);
+//
+// intpA.close();
+// }
+
+
+// @Test
+// public void testRemoteInterpreterSharesTheSameSchedulerInstanceInTheSameGroup() {
+// Properties p = new Properties();
+// intpGroup.put("note", new LinkedList<Interpreter>());
+//
+// RemoteInterpreter intpA = createMockInterpreterA(p);
+//
+// intpGroup.get("note").add(intpA);
+// intpA.setInterpreterGroup(intpGroup);
+//
+// RemoteInterpreter intpB = createMockInterpreterB(p);
+//
+// intpGroup.get("note").add(intpB);
+// intpB.setInterpreterGroup(intpGroup);
+//
+// intpA.open();
+// intpB.open();
+//
+// assertEquals(intpA.getScheduler(), intpB.getScheduler());
+// }
+
+// @Test
+// public void testMultiInterpreterSession() {
+// Properties p = new Properties();
+// intpGroup.put("sessionA", new LinkedList<Interpreter>());
+// intpGroup.put("sessionB", new LinkedList<Interpreter>());
+//
+// RemoteInterpreter intpAsessionA = createMockInterpreterA(p, "sessionA");
+// intpGroup.get("sessionA").add(intpAsessionA);
+// intpAsessionA.setInterpreterGroup(intpGroup);
+//
+// RemoteInterpreter intpBsessionA = createMockInterpreterB(p, "sessionA");
+// intpGroup.get("sessionA").add(intpBsessionA);
+// intpBsessionA.setInterpreterGroup(intpGroup);
+//
+// intpAsessionA.open();
+// intpBsessionA.open();
+//
+// assertEquals(intpAsessionA.getScheduler(), intpBsessionA.getScheduler());
+//
+// RemoteInterpreter intpAsessionB = createMockInterpreterA(p, "sessionB");
+// intpGroup.get("sessionB").add(intpAsessionB);
+// intpAsessionB.setInterpreterGroup(intpGroup);
+//
+// RemoteInterpreter intpBsessionB = createMockInterpreterB(p, "sessionB");
+// intpGroup.get("sessionB").add(intpBsessionB);
+// intpBsessionB.setInterpreterGroup(intpGroup);
+//
+// intpAsessionB.open();
+// intpBsessionB.open();
+//
+// assertEquals(intpAsessionB.getScheduler(), intpBsessionB.getScheduler());
+// assertNotEquals(intpAsessionA.getScheduler(), intpAsessionB.getScheduler());
+// }
+
+// @Test
+// public void should_push_local_angular_repo_to_remote() throws Exception {
+// //Given
+// final Client client = mock(Client.class);
+// final RemoteInterpreter intr = null;
+//// new RemoteInterpreter(new Properties(), "noteId",
+//// MockInterpreterA.class.getName(), "runner", "path", "localRepo", env, 10 * 1000, null,
+//// null, "anonymous", false);
+// final AngularObjectRegistry registry = new AngularObjectRegistry("spark", null);
+// registry.add("name", "DuyHai DOAN", "nodeId", "paragraphId");
+// final InterpreterGroup interpreterGroup = new InterpreterGroup("groupId");
+// interpreterGroup.setAngularObjectRegistry(registry);
+// intr.setInterpreterGroup(interpreterGroup);
+//
+// final java.lang.reflect.Type registryType = new TypeToken<Map<String,
+// Map<String, AngularObject>>>() {
+// }.getType();
+// final Gson gson = new Gson();
+// final String expected = gson.toJson(registry.getRegistry(), registryType);
+//
+// //When
+//// intr.pushAngularObjectRegistryToRemote(client);
+//
+// //Then
+// Mockito.verify(client).angularRegistryPush(expected);
+// }
+
+ @Test
+ public void testEnvStringPattern() {
+ assertFalse(RemoteInterpreterUtils.isEnvString(null));
+ assertFalse(RemoteInterpreterUtils.isEnvString(""));
+ assertFalse(RemoteInterpreterUtils.isEnvString("abcDEF"));
+ assertFalse(RemoteInterpreterUtils.isEnvString("ABC-DEF"));
+ assertTrue(RemoteInterpreterUtils.isEnvString("ABCDEF"));
+ assertTrue(RemoteInterpreterUtils.isEnvString("ABC_DEF"));
+ assertTrue(RemoteInterpreterUtils.isEnvString("ABC_DEF123"));
+ }
+
+ @Test
+ public void testEnvironmentAndProperty() {
+ interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
+ interpreterSetting.setProperty("ENV_1", "VALUE_1");
+ interpreterSetting.setProperty("property_1", "value_1");
+
+ final Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "get");
+ final InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
+ "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+ null, null, new ArrayList<InterpreterContextRunner>(), null);
+
+ assertEquals("VALUE_1", interpreter1.interpret("getEnv ENV_1", context1).message().get(0).getData());
+ assertEquals("null", interpreter1.interpret("getEnv ENV_2", context1).message().get(0).getData());
+
+ assertEquals("value_1", interpreter1.interpret("getProperty property_1", context1).message().get(0).getData());
+ assertEquals("null", interpreter1.interpret("getProperty property_2", context1).message().get(0).getData());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java
new file mode 100644
index 0000000..5f7426a
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote;
+
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertTrue;
+
+public class RemoteInterpreterUtilsTest {
+
+ @Test
+ public void testFindRandomAvailablePortOnAllLocalInterfaces() throws IOException {
+ assertTrue(RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces() > 0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/GetEnvPropertyInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/GetEnvPropertyInterpreter.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/GetEnvPropertyInterpreter.java
new file mode 100644
index 0000000..a039a59
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/GetEnvPropertyInterpreter.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.interpreter.remote.mock;
+
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.apache.zeppelin.scheduler.Scheduler;
+import org.apache.zeppelin.scheduler.SchedulerFactory;
+
+import java.util.List;
+import java.util.Properties;
+
+
+public class GetEnvPropertyInterpreter extends Interpreter {
+
+ public GetEnvPropertyInterpreter(Properties property) {
+ super(property);
+ }
+
+ @Override
+ public void open() {
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ String[] cmd = st.split(" ");
+ if (cmd[0].equals("getEnv")) {
+ return new InterpreterResult(InterpreterResult.Code.SUCCESS, System.getenv(cmd[1]) == null ? "null" : System.getenv(cmd[1]));
+ } else if (cmd[0].equals("getProperty")){
+ return new InterpreterResult(InterpreterResult.Code.SUCCESS, System.getProperty(cmd[1]) == null ? "null" : System.getProperty(cmd[1]));
+ } else {
+ return new InterpreterResult(InterpreterResult.Code.ERROR, cmd[0]);
+ }
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+
+ }
+
+ @Override
+ public FormType getFormType() {
+ return FormType.NATIVE;
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+
+ @Override
+ public List<InterpreterCompletion> completion(String buf, int cursor,
+ InterpreterContext interpreterContext) {
+ return null;
+ }
+
+ @Override
+ public Scheduler getScheduler() {
+ return SchedulerFactory.singleton().createOrGetFIFOScheduler("interpreter_" + this.hashCode());
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java
new file mode 100644
index 0000000..5a3e57c
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote.mock;
+
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.apache.zeppelin.scheduler.Scheduler;
+import org.apache.zeppelin.scheduler.SchedulerFactory;
+
+import java.util.List;
+import java.util.Properties;
+
+public class MockInterpreterA extends Interpreter {
+
+ private String lastSt;
+
+ public MockInterpreterA(Properties property) {
+ super(property);
+ }
+
+ @Override
+ public void open() {
+ //new RuntimeException().printStackTrace();
+ }
+
+ @Override
+ public void close() {
+ }
+
+ public String getLastStatement() {
+ return lastSt;
+ }
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ if (property.containsKey("progress")) {
+ context.setProgress(Integer.parseInt(getProperty("progress")));
+ }
+ try {
+ Thread.sleep(Long.parseLong(st));
+ this.lastSt = st;
+ } catch (NumberFormatException | InterruptedException e) {
+ throw new InterpreterException(e);
+ }
+ return new InterpreterResult(Code.SUCCESS, st);
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+
+ }
+
+ @Override
+ public FormType getFormType() {
+ return FormType.NATIVE;
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+
+ @Override
+ public List<InterpreterCompletion> completion(String buf, int cursor,
+ InterpreterContext interpreterContext) {
+ return null;
+ }
+
+ @Override
+ public Scheduler getScheduler() {
+ if (getProperty("parallel") != null && getProperty("parallel").equals("true")) {
+ return SchedulerFactory.singleton().createOrGetParallelScheduler("interpreter_" + this.hashCode(), 10);
+ } else {
+ return SchedulerFactory.singleton().createOrGetFIFOScheduler("interpreter_" + this.hashCode());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java
new file mode 100644
index 0000000..ec89241
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote.mock;
+
+import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.display.AngularObjectWatcher;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class MockInterpreterAngular extends Interpreter {
+
+ AtomicInteger numWatch = new AtomicInteger(0);
+
+ public MockInterpreterAngular(Properties property) {
+ super(property);
+ }
+
+ @Override
+ public void open() {
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ String[] stmt = st.split(" ");
+ String cmd = stmt[0];
+ String name = null;
+ if (stmt.length >= 2) {
+ name = stmt[1];
+ }
+ String value = null;
+ if (stmt.length == 3) {
+ value = stmt[2];
+ }
+
+ AngularObjectRegistry registry = context.getAngularObjectRegistry();
+
+ if (cmd.equals("add")) {
+ registry.add(name, value, context.getNoteId(), null);
+ registry.get(name, context.getNoteId(), null).addWatcher(new AngularObjectWatcher
+ (null) {
+
+ @Override
+ public void watch(Object oldObject, Object newObject,
+ InterpreterContext context) {
+ numWatch.incrementAndGet();
+ }
+
+ });
+ } else if (cmd.equalsIgnoreCase("update")) {
+ registry.get(name, context.getNoteId(), null).set(value);
+ } else if (cmd.equals("remove")) {
+ registry.remove(name, context.getNoteId(), null);
+ }
+
+ try {
+ Thread.sleep(500); // wait for watcher executed
+ } catch (InterruptedException e) {
+ logger.error("Exception in MockInterpreterAngular while interpret Thread.sleep", e);
+ }
+
+ String msg = registry.getAll(context.getNoteId(), null).size() + " " + Integer.toString(numWatch
+ .get());
+ return new InterpreterResult(Code.SUCCESS, msg);
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+ }
+
+ @Override
+ public FormType getFormType() {
+ return FormType.NATIVE;
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+
+ @Override
+ public List<InterpreterCompletion> completion(String buf, int cursor,
+ InterpreterContext interpreterContext) {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterB.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterB.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterB.java
new file mode 100644
index 0000000..ff3ff9f
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterB.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote.mock;
+
+import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.apache.zeppelin.scheduler.Scheduler;
+
+import java.util.List;
+import java.util.Properties;
+
+public class MockInterpreterB extends Interpreter {
+
+ public MockInterpreterB(Properties property) {
+ super(property);
+ }
+
+ @Override
+ public void open() {
+ //new RuntimeException().printStackTrace();
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ MockInterpreterA intpA = getInterpreterA();
+ String intpASt = intpA.getLastStatement();
+ long timeToSleep = Long.parseLong(st);
+ if (intpASt != null) {
+ timeToSleep += Long.parseLong(intpASt);
+ }
+ try {
+ Thread.sleep(timeToSleep);
+ } catch (NumberFormatException | InterruptedException e) {
+ throw new InterpreterException(e);
+ }
+ return new InterpreterResult(Code.SUCCESS, Long.toString(timeToSleep));
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+
+ }
+
+ @Override
+ public FormType getFormType() {
+ return FormType.NATIVE;
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+
+ @Override
+ public List<InterpreterCompletion> completion(String buf, int cursor,
+ InterpreterContext interpreterContext) {
+ return null;
+ }
+
+ public MockInterpreterA getInterpreterA() {
+ InterpreterGroup interpreterGroup = getInterpreterGroup();
+ synchronized (interpreterGroup) {
+ for (List<Interpreter> interpreters : interpreterGroup.values()) {
+ boolean belongsToSameNoteGroup = false;
+ MockInterpreterA a = null;
+ for (Interpreter intp : interpreters) {
+ if (intp.getClassName().equals(MockInterpreterA.class.getName())) {
+ Interpreter p = intp;
+ while (p instanceof WrappedInterpreter) {
+ p = ((WrappedInterpreter) p).getInnerInterpreter();
+ }
+ a = (MockInterpreterA) p;
+ }
+
+ Interpreter p = intp;
+ while (p instanceof WrappedInterpreter) {
+ p = ((WrappedInterpreter) p).getInnerInterpreter();
+ }
+ if (this == p) {
+ belongsToSameNoteGroup = true;
+ }
+ }
+ if (belongsToSameNoteGroup) {
+ return a;
+ }
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Scheduler getScheduler() {
+ MockInterpreterA intpA = getInterpreterA();
+ if (intpA != null) {
+ return intpA.getScheduler();
+ }
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java
new file mode 100644
index 0000000..1890cbc
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.interpreter.remote.mock;
+
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.apache.zeppelin.scheduler.Scheduler;
+import org.apache.zeppelin.scheduler.SchedulerFactory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * MockInterpreter to test outputstream
+ */
+public class MockInterpreterOutputStream extends Interpreter {
+ private String lastSt;
+
+ public MockInterpreterOutputStream(Properties property) {
+ super(property);
+ }
+
+ @Override
+ public void open() {
+ //new RuntimeException().printStackTrace();
+ }
+
+ @Override
+ public void close() {
+ }
+
+ public String getLastStatement() {
+ return lastSt;
+ }
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ String[] ret = st.split(":");
+ try {
+ if (ret[1] != null) {
+ context.out.write(ret[1]);
+ }
+ } catch (IOException e) {
+ throw new InterpreterException(e);
+ }
+ return new InterpreterResult(InterpreterResult.Code.valueOf(ret[0]), (ret.length > 2) ?
+ ret[2] : "");
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+
+ }
+
+ @Override
+ public FormType getFormType() {
+ return FormType.NATIVE;
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+
+ @Override
+ public List<InterpreterCompletion> completion(String buf, int cursor,
+ InterpreterContext interpreterContext) {
+ return null;
+ }
+
+ @Override
+ public Scheduler getScheduler() {
+ return SchedulerFactory.singleton().createOrGetFIFOScheduler("interpreter_" + this.hashCode());
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterResourcePool.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterResourcePool.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterResourcePool.java
new file mode 100644
index 0000000..ee9f15c
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterResourcePool.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote.mock;
+
+import com.google.gson.Gson;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.apache.zeppelin.resource.Resource;
+import org.apache.zeppelin.resource.ResourcePool;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class MockInterpreterResourcePool extends Interpreter {
+
+ AtomicInteger numWatch = new AtomicInteger(0);
+
+ public MockInterpreterResourcePool(Properties property) {
+ super(property);
+ }
+
+ @Override
+ public void open() {
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ String[] stmt = st.split(" ");
+ String cmd = stmt[0];
+ String noteId = null;
+ String paragraphId = null;
+ String name = null;
+ if (stmt.length >= 2) {
+ String[] npn = stmt[1].split(":");
+ if (npn.length >= 3) {
+ noteId = npn[0];
+ paragraphId = npn[1];
+ name = npn[2];
+ } else {
+ name = stmt[1];
+ }
+ }
+ String value = null;
+ if (stmt.length >= 3) {
+ value = stmt[2];
+ }
+
+ ResourcePool resourcePool = context.getResourcePool();
+ Object ret = null;
+ if (cmd.equals("put")) {
+ resourcePool.put(noteId, paragraphId, name, value);
+ } else if (cmd.equalsIgnoreCase("get")) {
+ Resource resource = resourcePool.get(noteId, paragraphId, name);
+ if (resource != null) {
+ ret = resourcePool.get(noteId, paragraphId, name).get();
+ } else {
+ ret = "";
+ }
+ } else if (cmd.equals("remove")) {
+ ret = resourcePool.remove(noteId, paragraphId, name);
+ } else if (cmd.equals("getAll")) {
+ ret = resourcePool.getAll();
+ } else if (cmd.equals("invoke")) {
+ Resource resource = resourcePool.get(noteId, paragraphId, name);
+ if (stmt.length >=4) {
+ Resource res = resource.invokeMethod(value, null, null, stmt[3]);
+ ret = res.get();
+ } else {
+ ret = resource.invokeMethod(value, null, null);
+ }
+ }
+
+ try {
+ Thread.sleep(500); // wait for watcher executed
+ } catch (InterruptedException e) {
+ }
+
+ Gson gson = new Gson();
+ return new InterpreterResult(Code.SUCCESS, gson.toJson(ret));
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+ }
+
+ @Override
+ public FormType getFormType() {
+ return FormType.NATIVE;
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+
+ @Override
+ public List<InterpreterCompletion> completion(String buf, int cursor,
+ InterpreterContext interpreterContext) {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
new file mode 100644
index 0000000..a1afe0e
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.scheduler;
+
+import org.apache.zeppelin.display.GUI;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterContextRunner;
+import org.apache.zeppelin.interpreter.InterpreterInfo;
+import org.apache.zeppelin.interpreter.InterpreterOption;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterRunner;
+import org.apache.zeppelin.interpreter.InterpreterSetting;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
+import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterA;
+import org.apache.zeppelin.resource.LocalResourcePool;
+import org.apache.zeppelin.scheduler.Job.Status;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
+
+ private static final String INTERPRETER_SCRIPT =
+ System.getProperty("os.name").startsWith("Windows") ?
+ "../bin/interpreter.cmd" :
+ "../bin/interpreter.sh";
+
+ private InterpreterSetting interpreterSetting;
+ private SchedulerFactory schedulerSvc;
+ private static final int TICK_WAIT = 100;
+ private static final int MAX_WAIT_CYCLES = 100;
+
+ @Before
+ public void setUp() throws Exception {
+ schedulerSvc = new SchedulerFactory();
+
+ InterpreterOption interpreterOption = new InterpreterOption();
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(MockInterpreterA.class.getName(), "mock", true, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT);
+ interpreterSetting = new InterpreterSetting.Builder()
+ .setId("test")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .setRunner(runner)
+ .setInterpreterDir("../interpeters/test")
+ .create();
+ }
+
+ @After
+ public void tearDown() {
+ interpreterSetting.close();
+ }
+
+ @Test
+ public void test() throws Exception {
+ final RemoteInterpreter intpA = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
+
+ intpA.open();
+
+ Scheduler scheduler = schedulerSvc.createOrGetRemoteScheduler("test", "note",
+ intpA,
+ 10);
+
+ Job job = new Job("jobId", "jobName", null, 200) {
+ Object results;
+
+ @Override
+ public Object getReturn() {
+ return results;
+ }
+
+ @Override
+ public int progress() {
+ return 0;
+ }
+
+ @Override
+ public Map<String, Object> info() {
+ return null;
+ }
+
+ @Override
+ protected Object jobRun() throws Throwable {
+ intpA.interpret("1000", new InterpreterContext(
+ "note",
+ "jobId",
+ null,
+ "title",
+ "text",
+ new AuthenticationInfo(),
+ new HashMap<String, Object>(),
+ new GUI(),
+ null,
+ new LocalResourcePool("pool1"),
+ new LinkedList<InterpreterContextRunner>(), null));
+ return "1000";
+ }
+
+ @Override
+ protected boolean jobAbort() {
+ return false;
+ }
+
+ @Override
+ public void setResult(Object results) {
+ this.results = results;
+ }
+ };
+ scheduler.submit(job);
+
+ int cycles = 0;
+ while (!job.isRunning() && cycles < MAX_WAIT_CYCLES) {
+ Thread.sleep(TICK_WAIT);
+ cycles++;
+ }
+ assertTrue(job.isRunning());
+
+ Thread.sleep(5 * TICK_WAIT);
+ assertEquals(0, scheduler.getJobsWaiting().size());
+ assertEquals(1, scheduler.getJobsRunning().size());
+
+ cycles = 0;
+ while (!job.isTerminated() && cycles < MAX_WAIT_CYCLES) {
+ Thread.sleep(TICK_WAIT);
+ cycles++;
+ }
+
+ assertTrue(job.isTerminated());
+ assertEquals(0, scheduler.getJobsWaiting().size());
+ assertEquals(0, scheduler.getJobsRunning().size());
+
+ intpA.close();
+ schedulerSvc.removeScheduler("test");
+ }
+
+ @Test
+ public void testAbortOnPending() throws Exception {
+ final RemoteInterpreter intpA = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
+ intpA.open();
+
+ Scheduler scheduler = schedulerSvc.createOrGetRemoteScheduler("test", "note", intpA, 10);
+
+ Job job1 = new Job("jobId1", "jobName1", null, 200) {
+ Object results;
+ InterpreterContext context = new InterpreterContext(
+ "note",
+ "jobId1",
+ null,
+ "title",
+ "text",
+ new AuthenticationInfo(),
+ new HashMap<String, Object>(),
+ new GUI(),
+ null,
+ new LocalResourcePool("pool1"),
+ new LinkedList<InterpreterContextRunner>(), null);
+
+ @Override
+ public Object getReturn() {
+ return results;
+ }
+
+ @Override
+ public int progress() {
+ return 0;
+ }
+
+ @Override
+ public Map<String, Object> info() {
+ return null;
+ }
+
+ @Override
+ protected Object jobRun() throws Throwable {
+ intpA.interpret("1000", context);
+ return "1000";
+ }
+
+ @Override
+ protected boolean jobAbort() {
+ if (isRunning()) {
+ intpA.cancel(context);
+ }
+ return true;
+ }
+
+ @Override
+ public void setResult(Object results) {
+ this.results = results;
+ }
+ };
+
+ Job job2 = new Job("jobId2", "jobName2", null, 200) {
+ public Object results;
+ InterpreterContext context = new InterpreterContext(
+ "note",
+ "jobId2",
+ null,
+ "title",
+ "text",
+ new AuthenticationInfo(),
+ new HashMap<String, Object>(),
+ new GUI(),
+ null,
+ new LocalResourcePool("pool1"),
+ new LinkedList<InterpreterContextRunner>(), null);
+
+ @Override
+ public Object getReturn() {
+ return results;
+ }
+
+ @Override
+ public int progress() {
+ return 0;
+ }
+
+ @Override
+ public Map<String, Object> info() {
+ return null;
+ }
+
+ @Override
+ protected Object jobRun() throws Throwable {
+ intpA.interpret("1000", context);
+ return "1000";
+ }
+
+ @Override
+ protected boolean jobAbort() {
+ if (isRunning()) {
+ intpA.cancel(context);
+ }
+ return true;
+ }
+
+ @Override
+ public void setResult(Object results) {
+ this.results = results;
+ }
+ };
+
+ job2.setResult("result2");
+
+ scheduler.submit(job1);
+ scheduler.submit(job2);
+
+
+ int cycles = 0;
+ while (!job1.isRunning() && cycles < MAX_WAIT_CYCLES) {
+ Thread.sleep(TICK_WAIT);
+ cycles++;
+ }
+ assertTrue(job1.isRunning());
+ assertTrue(job2.getStatus() == Status.PENDING);
+
+ job2.abort();
+
+ cycles = 0;
+ while (!job1.isTerminated() && cycles < MAX_WAIT_CYCLES) {
+ Thread.sleep(TICK_WAIT);
+ cycles++;
+ }
+
+ assertNotNull(job1.getDateFinished());
+ assertTrue(job1.isTerminated());
+ assertNull(job2.getDateFinished());
+ assertTrue(job2.isTerminated());
+ assertEquals("result2", job2.getReturn());
+
+ intpA.close();
+ schedulerSvc.removeScheduler("test");
+ }
+
+ @Override
+ public void onOutputAppend(String noteId, String paragraphId, int index, String output) {
+
+ }
+
+ @Override
+ public void onOutputUpdated(String noteId, String paragraphId, int index, InterpreterResult.Type type, String output) {
+
+ }
+
+ @Override
+ public void onOutputClear(String noteId, String paragraphId) {
+
+ }
+
+ @Override
+ public void onMetaInfosReceived(String settingId, Map<String, String> metaInfos) {
+
+ }
+
+ @Override
+ public void onGetParagraphRunners(String noteId, String paragraphId, RemoteWorksEventListener callback) {
+ if (callback != null) {
+ callback.onFinished(new LinkedList<>());
+ }
+ }
+
+ @Override
+ public void onRemoteRunParagraph(String noteId, String PsaragraphID) throws Exception {
+ }
+
+ @Override
+ public void onParaInfosReceived(String noteId, String paragraphId,
+ String interpreterSettingId, Map<String, String> metaInfos) {
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/resources/conf/interpreter.json
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/resources/conf/interpreter.json b/zeppelin-interpreter/src/test/resources/conf/interpreter.json
new file mode 100644
index 0000000..45e1d60
--- /dev/null
+++ b/zeppelin-interpreter/src/test/resources/conf/interpreter.json
@@ -0,0 +1,115 @@
+{
+ "interpreterSettings": {
+ "2C3RWCVAG": {
+ "id": "2C3RWCVAG",
+ "name": "test",
+ "group": "test",
+ "properties": {
+ "property_1": "value_1",
+ "property_2": "new_value_2",
+ "property_3": "value_3"
+ },
+ "status": "READY",
+ "interpreterGroup": [
+ {
+ "name": "echo",
+ "class": "org.apache.zeppelin.interpreter.EchoInterpreter",
+ "defaultInterpreter": true,
+ "editor": {
+ "language": "java",
+ "editOnDblClick": false
+ }
+ }
+ ],
+ "dependencies": [],
+ "option": {
+ "remote": true,
+ "port": -1,
+ "perNote": "shared",
+ "perUser": "shared",
+ "isExistingProcess": false,
+ "setPermission": false,
+ "users": [],
+ "isUserImpersonate": false
+ }
+ },
+
+ "2CKWE7B19": {
+ "id": "2CKWE7B19",
+ "name": "test2",
+ "group": "test",
+ "properties": {
+ "property_1": "value_1",
+ "property_2": "new_value_2",
+ "property_3": "value_3"
+ },
+ "status": "READY",
+ "interpreterGroup": [
+ {
+ "name": "echo",
+ "class": "org.apache.zeppelin.interpreter.EchoInterpreter",
+ "defaultInterpreter": true,
+ "editor": {
+ "language": "java",
+ "editOnDblClick": false
+ }
+ }
+ ],
+ "dependencies": [],
+ "option": {
+ "remote": true,
+ "port": -1,
+ "perNote": "shared",
+ "perUser": "shared",
+ "isExistingProcess": false,
+ "setPermission": false,
+ "users": [],
+ "isUserImpersonate": false
+ }
+ }
+ },
+ "interpreterBindings": {
+ "2C6793KRV": [
+ "2C48Y7FSJ",
+ "2C63XW4XE",
+ "2C66GE1VB",
+ "2C5VH924X",
+ "2C4BJDRRZ",
+ "2C3SQSB7V",
+ "2C4HKDCQW",
+ "2C3DR183X",
+ "2C66Z9XPQ",
+ "2C3PTPMUH",
+ "2C69WE69N",
+ "2C5SRRXHM",
+ "2C4ZD49PF",
+ "2C6V3D44K",
+ "2C4UB1UZA",
+ "2C5S1R21W",
+ "2C5DCRVGM",
+ "2C686X8ZH",
+ "2C3RWCVAG",
+ "2C3JKFMJU",
+ "2C3VECEG2"
+ ]
+ },
+ "interpreterRepositories": [
+ {
+ "id": "central",
+ "type": "default",
+ "url": "http://repo1.maven.org/maven2/",
+ "releasePolicy": {
+ "enabled": true,
+ "updatePolicy": "daily",
+ "checksumPolicy": "warn"
+ },
+ "snapshotPolicy": {
+ "enabled": true,
+ "updatePolicy": "daily",
+ "checksumPolicy": "warn"
+ },
+ "mirroredRepositories": [],
+ "repositoryManager": false
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/resources/interpreter/test/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/resources/interpreter/test/interpreter-setting.json b/zeppelin-interpreter/src/test/resources/interpreter/test/interpreter-setting.json
new file mode 100644
index 0000000..1ba1b94
--- /dev/null
+++ b/zeppelin-interpreter/src/test/resources/interpreter/test/interpreter-setting.json
@@ -0,0 +1,42 @@
+[
+ {
+ "group": "test",
+ "name": "double_echo",
+ "className": "org.apache.zeppelin.interpreter.DoubleEchoInterpreter",
+ "properties": {
+ "property_1": {
+ "envName": "PROPERTY_1",
+ "propertyName": "property_1",
+ "defaultValue": "value_1",
+ "description": "desc_1"
+ },
+ "property_2": {
+ "envName": "PROPERTY_2",
+ "propertyName": "property_2",
+ "defaultValue": "value_2",
+ "description": "desc_2"
+ }
+ }
+ },
+
+ {
+ "group": "test",
+ "name": "echo",
+ "defaultInterpreter": true,
+ "className": "org.apache.zeppelin.interpreter.EchoInterpreter",
+ "properties": {
+ "property_1": {
+ "envName": "PROPERTY_1",
+ "propertyName": "property_1",
+ "defaultValue": "value_1",
+ "description": "desc_1"
+ },
+ "property_2": {
+ "envName": "PROPERTY_2",
+ "propertyName": "property_2",
+ "defaultValue": "value_2",
+ "description": "desc_2"
+ }
+ }
+ }
+]
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/resources/log4j.properties b/zeppelin-interpreter/src/test/resources/log4j.properties
index d8a7839..6f34691 100644
--- a/zeppelin-interpreter/src/test/resources/log4j.properties
+++ b/zeppelin-interpreter/src/test/resources/log4j.properties
@@ -26,4 +26,6 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c:%L - %m%n
#
# Root logger option
-log4j.rootLogger=INFO, stdout
\ No newline at end of file
+log4j.rootLogger=INFO, stdout
+log4j.logger.org.apache.zeppelin.interpreter=DEBUG
+log4j.logger.org.apache.zeppelin.scheduler=DEBUG
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
index cd0210e..c1dba5c 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
@@ -185,7 +185,7 @@ public class InterpreterRestApi {
String noteId = request == null ? null : request.getNoteId();
if (null == noteId) {
- interpreterSettingManager.close(setting);
+ interpreterSettingManager.close(settingId);
} else {
interpreterSettingManager.restart(settingId, noteId, SecurityUtils.getPrincipal());
}
@@ -208,7 +208,7 @@ public class InterpreterRestApi {
@GET
@ZeppelinApi
public Response listInterpreter(String message) {
- Map<String, InterpreterSetting> m = interpreterSettingManager.getAvailableInterpreterSettings();
+ Map<String, InterpreterSetting> m = interpreterSettingManager.getInterpreterSettingTemplates();
return new JsonResponse<>(Status.OK, "", m).build();
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
index 7453470..53ee114 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
@@ -93,13 +93,11 @@ public class ZeppelinServer extends Application {
private NotebookRepoSync notebookRepo;
private NotebookAuthorization notebookAuthorization;
private Credentials credentials;
- private DependencyResolver depResolver;
public ZeppelinServer() throws Exception {
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
- this.depResolver = new DependencyResolver(
- conf.getString(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO));
+
InterpreterOutput.limit = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT);
@@ -129,13 +127,26 @@ public class ZeppelinServer extends Application {
new File(conf.getRelativeDir("zeppelin-web/src/app/spell")));
}
+ this.schedulerFactory = SchedulerFactory.singleton();
+ this.interpreterSettingManager = new InterpreterSettingManager(conf, notebookWsServer,
+ notebookWsServer, notebookWsServer);
+ this.replFactory = new InterpreterFactory(interpreterSettingManager);
+ this.notebookRepo = new NotebookRepoSync(conf);
+ this.noteSearchService = new LuceneSearch();
+ this.notebookAuthorization = NotebookAuthorization.init(conf);
+ this.credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath());
+ notebook = new Notebook(conf,
+ notebookRepo, schedulerFactory, replFactory, interpreterSettingManager, notebookWsServer,
+ noteSearchService, notebookAuthorization, credentials);
+
ZeppelinServer.helium = new Helium(
conf.getHeliumConfPath(),
conf.getHeliumRegistry(),
new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO),
"helium-registry-cache"),
heliumBundleFactory,
- heliumApplicationFactory);
+ heliumApplicationFactory,
+ interpreterSettingManager);
// create bundle
try {
@@ -144,20 +155,6 @@ public class ZeppelinServer extends Application {
LOG.error(e.getMessage(), e);
}
- this.schedulerFactory = new SchedulerFactory();
- this.interpreterSettingManager = new InterpreterSettingManager(conf, depResolver,
- new InterpreterOption(true));
- this.replFactory = new InterpreterFactory(conf, notebookWsServer,
- notebookWsServer, heliumApplicationFactory, depResolver, SecurityUtils.isAuthenticated(),
- interpreterSettingManager);
- this.notebookRepo = new NotebookRepoSync(conf);
- this.noteSearchService = new LuceneSearch();
- this.notebookAuthorization = NotebookAuthorization.init(conf);
- this.credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath());
- notebook = new Notebook(conf,
- notebookRepo, schedulerFactory, replFactory, interpreterSettingManager, notebookWsServer,
- noteSearchService, notebookAuthorization, credentials);
-
// to update notebook from application event from remote process.
heliumApplicationFactory.setNotebook(notebook);
// to update fire websocket event on application event.
@@ -206,7 +203,7 @@ public class ZeppelinServer extends Application {
LOG.info("Shutting down Zeppelin Server ... ");
try {
jettyWebServer.stop();
- notebook.getInterpreterSettingManager().shutdown();
+ notebook.getInterpreterSettingManager().close();
notebook.close();
Thread.sleep(3000);
} catch (Exception e) {
[09/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
new file mode 100644
index 0000000..bb90dd8
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
@@ -0,0 +1,371 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import org.apache.thrift.TException;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.display.AngularObject;
+import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.display.GUI;
+import org.apache.zeppelin.display.Input;
+import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
+import org.apache.zeppelin.scheduler.Job;
+import org.apache.zeppelin.scheduler.Scheduler;
+import org.apache.zeppelin.scheduler.SchedulerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Proxy for Interpreter instance that runs on separate process
+ */
+public class RemoteInterpreter extends Interpreter {
+ private static final Logger LOGGER = LoggerFactory.getLogger(RemoteInterpreter.class);
+ private static final Gson gson = new Gson();
+
+
+ private String className;
+ private String sessionId;
+ private String userName;
+ private FormType formType;
+
+ private RemoteInterpreterProcess interpreterProcess;
+ private volatile boolean isOpened = false;
+ private volatile boolean isCreated = false;
+
+ /**
+ * Remote interpreter and manage interpreter process
+ */
+ public RemoteInterpreter(Properties properties,
+ String sessionId,
+ String className,
+ String userName) {
+ super(properties);
+ this.sessionId = sessionId;
+ this.className = className;
+ this.userName = userName;
+ }
+
+ public boolean isOpened() {
+ return isOpened;
+ }
+
+ @Override
+ public String getClassName() {
+ return className;
+ }
+
+ public String getSessionId() {
+ return this.sessionId;
+ }
+
+ public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess() {
+ if (this.interpreterProcess != null) {
+ return this.interpreterProcess;
+ }
+ InterpreterGroup intpGroup = getInterpreterGroup();
+ this.interpreterProcess = intpGroup.getOrCreateInterpreterProcess();
+ synchronized (interpreterProcess) {
+ if (!interpreterProcess.isRunning()) {
+ interpreterProcess.start(userName, false);
+ interpreterProcess.getRemoteInterpreterEventPoller()
+ .setInterpreterProcess(interpreterProcess);
+ interpreterProcess.getRemoteInterpreterEventPoller().setInterpreterGroup(intpGroup);
+ interpreterProcess.getRemoteInterpreterEventPoller().start();
+ }
+ }
+ return interpreterProcess;
+ }
+
+ @Override
+ public void open() {
+ synchronized (this) {
+ if (!isOpened) {
+ // create all the interpreters of the same session first, then Open the internal interpreter
+ // of this RemoteInterpreter.
+ // The why we we create all the interpreter of the session is because some interpreter
+ // depends on other interpreter. e.g. PySparkInterpreter depends on SparkInterpreter.
+ // also see method Interpreter.getInterpreterInTheSameSessionByClassName
+ for (Interpreter interpreter : getInterpreterGroup().getOrCreateSession(
+ userName, sessionId)) {
+ ((RemoteInterpreter) interpreter).internal_create();
+ }
+
+ interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ LOGGER.info("Open RemoteInterpreter {}", getClassName());
+ client.open(sessionId, className);
+ // Push angular object loaded from JSON file to remote interpreter
+ synchronized (getInterpreterGroup()) {
+ if (!getInterpreterGroup().isAngularRegistryPushed()) {
+ pushAngularObjectRegistryToRemote(client);
+ getInterpreterGroup().setAngularRegistryPushed(true);
+ }
+ }
+ return null;
+ }
+ });
+ isOpened = true;
+ }
+ }
+ }
+
+ private void internal_create() {
+ synchronized (this) {
+ if (!isCreated) {
+ RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
+ interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ LOGGER.info("Create RemoteInterpreter {}", getClassName());
+ client.createInterpreter(getInterpreterGroup().getId(), sessionId,
+ className, (Map) property, userName);
+ return null;
+ }
+ });
+ isCreated = true;
+ }
+ }
+ }
+
+
+ @Override
+ public void close() {
+ if (isOpened) {
+ RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
+ interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ client.close(sessionId, className);
+ return null;
+ }
+ });
+ } else {
+ LOGGER.warn("close is called when RemoterInterpreter is not opened for " + className);
+ }
+ }
+
+ @Override
+ public InterpreterResult interpret(final String st, final InterpreterContext context) {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("st:\n{}", st);
+ }
+
+ final FormType form = getFormType();
+ RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
+ InterpreterContextRunnerPool interpreterContextRunnerPool = interpreterProcess
+ .getInterpreterContextRunnerPool();
+ List<InterpreterContextRunner> runners = context.getRunners();
+ if (runners != null && runners.size() != 0) {
+ // assume all runners in this InterpreterContext have the same note id
+ String noteId = runners.get(0).getNoteId();
+
+ interpreterContextRunnerPool.clear(noteId);
+ interpreterContextRunnerPool.addAll(noteId, runners);
+ }
+ return interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<InterpreterResult>() {
+ @Override
+ public InterpreterResult call(Client client) throws Exception {
+
+ RemoteInterpreterResult remoteResult = client.interpret(
+ sessionId, className, st, convert(context));
+ Map<String, Object> remoteConfig = (Map<String, Object>) gson.fromJson(
+ remoteResult.getConfig(), new TypeToken<Map<String, Object>>() {
+ }.getType());
+ context.getConfig().clear();
+ context.getConfig().putAll(remoteConfig);
+ GUI currentGUI = context.getGui();
+ if (form == FormType.NATIVE) {
+ GUI remoteGui = GUI.fromJson(remoteResult.getGui());
+ currentGUI.clear();
+ currentGUI.setParams(remoteGui.getParams());
+ currentGUI.setForms(remoteGui.getForms());
+ } else if (form == FormType.SIMPLE) {
+ final Map<String, Input> currentForms = currentGUI.getForms();
+ final Map<String, Object> currentParams = currentGUI.getParams();
+ final GUI remoteGUI = GUI.fromJson(remoteResult.getGui());
+ final Map<String, Input> remoteForms = remoteGUI.getForms();
+ final Map<String, Object> remoteParams = remoteGUI.getParams();
+ currentForms.putAll(remoteForms);
+ currentParams.putAll(remoteParams);
+ }
+
+ InterpreterResult result = convert(remoteResult);
+ return result;
+ }
+ }
+ );
+
+ }
+
+ @Override
+ public void cancel(final InterpreterContext context) {
+ if (!isOpened) {
+ LOGGER.warn("Cancel is called when RemoterInterpreter is not opened for " + className);
+ return;
+ }
+ RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
+ interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ client.cancel(sessionId, className, convert(context));
+ return null;
+ }
+ });
+ }
+
+ @Override
+ public FormType getFormType() {
+ if (formType != null) {
+ return formType;
+ }
+
+ // it is possible to call getFormType before it is opened
+ synchronized (this) {
+ if (!isOpened) {
+ open();
+ }
+ }
+ RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
+ FormType type = interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<FormType>() {
+ @Override
+ public FormType call(Client client) throws Exception {
+ formType = FormType.valueOf(client.getFormType(sessionId, className));
+ return formType;
+ }
+ });
+ return type;
+ }
+
+ @Override
+ public int getProgress(final InterpreterContext context) {
+ if (!isOpened) {
+ LOGGER.warn("getProgress is called when RemoterInterpreter is not opened for " + className);
+ return 0;
+ }
+ RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
+ return interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<Integer>() {
+ @Override
+ public Integer call(Client client) throws Exception {
+ return client.getProgress(sessionId, className, convert(context));
+ }
+ });
+ }
+
+
+ @Override
+ public List<InterpreterCompletion> completion(final String buf, final int cursor,
+ final InterpreterContext interpreterContext) {
+ if (!isOpened) {
+ LOGGER.warn("completion is called when RemoterInterpreter is not opened for " + className);
+ return new ArrayList<>();
+ }
+ RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
+ return interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<List<InterpreterCompletion>>() {
+ @Override
+ public List<InterpreterCompletion> call(Client client) throws Exception {
+ return client.completion(sessionId, className, buf, cursor,
+ convert(interpreterContext));
+ }
+ });
+ }
+
+ public String getStatus(final String jobId) {
+ if (!isOpened) {
+ LOGGER.warn("getStatus is called when RemoteInterpreter is not opened for " + className);
+ return Job.Status.UNKNOWN.name();
+ }
+ RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
+ return interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<String>() {
+ @Override
+ public String call(Client client) throws Exception {
+ return client.getStatus(sessionId, jobId);
+ }
+ });
+ }
+
+ //TODO(zjffdu) Share the Scheduler in the same session or in the same InterpreterGroup ?
+ @Override
+ public Scheduler getScheduler() {
+ int maxConcurrency = Integer.parseInt(
+ property.getProperty("zeppelin.interpreter.max.poolsize",
+ ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE.getIntValue() + ""));
+ return SchedulerFactory.singleton().createOrGetRemoteScheduler(
+ RemoteInterpreter.class.getName() + "-" + sessionId,
+ sessionId, this, maxConcurrency);
+ }
+
+ private RemoteInterpreterContext convert(InterpreterContext ic) {
+ return new RemoteInterpreterContext(ic.getNoteId(), ic.getParagraphId(), ic.getReplName(),
+ ic.getParagraphTitle(), ic.getParagraphText(), gson.toJson(ic.getAuthenticationInfo()),
+ gson.toJson(ic.getConfig()), gson.toJson(ic.getGui()), gson.toJson(ic.getRunners()));
+ }
+
+ private InterpreterResult convert(RemoteInterpreterResult result) {
+ InterpreterResult r = new InterpreterResult(
+ InterpreterResult.Code.valueOf(result.getCode()));
+
+ for (RemoteInterpreterResultMessage m : result.getMsg()) {
+ r.add(InterpreterResult.Type.valueOf(m.getType()), m.getData());
+ }
+
+ return r;
+ }
+
+ /**
+ * Push local angular object registry to
+ * remote interpreter. This method should be
+ * call ONLY once when the first Interpreter is created
+ */
+ private void pushAngularObjectRegistryToRemote(Client client) throws TException {
+ final AngularObjectRegistry angularObjectRegistry = this.getInterpreterGroup()
+ .getAngularObjectRegistry();
+ if (angularObjectRegistry != null && angularObjectRegistry.getRegistry() != null) {
+ final Map<String, Map<String, AngularObject>> registry = angularObjectRegistry
+ .getRegistry();
+ LOGGER.info("Push local angular object registry from ZeppelinServer to" +
+ " remote interpreter group {}", this.getInterpreterGroup().getId());
+ final java.lang.reflect.Type registryType = new TypeToken<Map<String,
+ Map<String, AngularObject>>>() {
+ }.getType();
+ client.angularRegistryPush(gson.toJson(registry, registryType));
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "RemoteInterpreter_" + className + "_" + sessionId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
index 26c9d79..f3bce2f 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
@@ -29,6 +29,7 @@ import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.RemoteZeppelinServerResource;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.apache.zeppelin.interpreter.thrift.ZeppelinServerResourceParagraphRunner;
import org.apache.zeppelin.resource.Resource;
@@ -38,6 +39,7 @@ import org.apache.zeppelin.resource.ResourceSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
@@ -84,7 +86,6 @@ public class RemoteInterpreterEventPoller extends Thread {
@Override
public void run() {
- Client client = null;
AppendOutputRunner runner = new AppendOutputRunner(listener);
ScheduledFuture<?> appendFuture = appendService.scheduleWithFixedDelay(
runner, 0, AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
@@ -100,26 +101,14 @@ public class RemoteInterpreterEventPoller extends Thread {
continue;
}
- try {
- client = interpreterProcess.getClient();
- } catch (Exception e1) {
- logger.error("Can't get RemoteInterpreterEvent", e1);
- waitQuietly();
- continue;
- }
-
- RemoteInterpreterEvent event = null;
- boolean broken = false;
- try {
- event = client.getEvent();
- } catch (TException e) {
- broken = true;
- logger.error("Can't get RemoteInterpreterEvent", e);
- waitQuietly();
- continue;
- } finally {
- interpreterProcess.releaseClient(client, broken);
- }
+ RemoteInterpreterEvent event = interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<RemoteInterpreterEvent>() {
+ @Override
+ public RemoteInterpreterEvent call(Client client) throws Exception {
+ return client.getEvent();
+ }
+ }
+ );
AngularObjectRegistry angularObjectRegistry = interpreterGroup.getAngularObjectRegistry();
@@ -286,10 +275,7 @@ public class RemoteInterpreterEventPoller extends Thread {
boolean broken = false;
final Gson gson = new Gson();
final String eventOwnerKey = reqResourceBody.getOwnerKey();
- Client interpreterServerMain = null;
try {
- interpreterServerMain = interpreterProcess.getClient();
- final Client eventClient = interpreterServerMain;
if (resourceType == RemoteZeppelinServerResource.Type.PARAGRAPH_RUNNERS) {
final List<ZeppelinServerResourceParagraphRunner> remoteRunners = new LinkedList<>();
@@ -308,7 +294,6 @@ public class RemoteInterpreterEventPoller extends Thread {
@Override
public void onFinished(Object resultObject) {
- boolean clientBroken = false;
if (resultObject != null && resultObject instanceof List) {
List<InterpreterContextRunner> runnerList =
(List<InterpreterContextRunner>) resultObject;
@@ -324,15 +309,15 @@ public class RemoteInterpreterEventPoller extends Thread {
resResource.setResourceType(RemoteZeppelinServerResource.Type.PARAGRAPH_RUNNERS);
resResource.setData(remoteRunners);
- try {
- eventClient.onReceivedZeppelinResource(resResource.toJson());
- } catch (Exception e) {
- clientBroken = true;
- logger.error("Can't get RemoteInterpreterEvent", e);
- waitQuietly();
- } finally {
- interpreterProcess.releaseClient(eventClient, clientBroken);
- }
+ interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ client.onReceivedZeppelinResource(resResource.toJson());
+ return null;
+ }
+ }
+ );
}
}
@@ -346,39 +331,32 @@ public class RemoteInterpreterEventPoller extends Thread {
reqRunnerContext.getNoteId(), reqRunnerContext.getParagraphId(), callBackEvent);
}
} catch (Exception e) {
- broken = true;
logger.error("Can't get RemoteInterpreterEvent", e);
waitQuietly();
- } finally {
- interpreterProcess.releaseClient(interpreterServerMain, broken);
}
}
- private void sendResourcePoolResponseGetAll(ResourceSet resourceSet) {
- Client client = null;
- boolean broken = false;
- try {
- client = interpreterProcess.getClient();
- List<String> resourceList = new LinkedList<>();
- Gson gson = new Gson();
- for (Resource r : resourceSet) {
- resourceList.add(gson.toJson(r));
- }
- client.resourcePoolResponseGetAll(resourceList);
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- broken = true;
- } finally {
- if (client != null) {
- interpreterProcess.releaseClient(client, broken);
- }
- }
+ private void sendResourcePoolResponseGetAll(final ResourceSet resourceSet) {
+ interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ List<String> resourceList = new LinkedList<>();
+ for (Resource r : resourceSet) {
+ resourceList.add(r.toJson());
+ }
+ client.resourcePoolResponseGetAll(resourceList);
+ return null;
+ }
+ }
+ );
}
private ResourceSet getAllResourcePoolExcept() {
ResourceSet resourceSet = new ResourceSet();
- for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
+ for (InterpreterGroup intpGroup : interpreterGroup.getInterpreterSetting()
+ .getInterpreterSettingManager().getAllInterpreterGroup()) {
if (intpGroup.getId().equals(interpreterGroup.getId())) {
continue;
}
@@ -390,115 +368,94 @@ public class RemoteInterpreterEventPoller extends Thread {
resourceSet.addAll(localPool.getAll());
}
} else if (interpreterProcess.isRunning()) {
- Client client = null;
- boolean broken = false;
- try {
- client = remoteInterpreterProcess.getClient();
- List<String> resourceList = client.resourcePoolGetAll();
- Gson gson = new Gson();
- for (String res : resourceList) {
- resourceSet.add(Resource.fromJson(res));
- }
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- broken = true;
- } finally {
- if (client != null) {
- intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
- }
+ List<String> resourceList = remoteInterpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<List<String>>() {
+ @Override
+ public List<String> call(Client client) throws Exception {
+ return client.resourcePoolGetAll();
+ }
+ }
+ );
+ for (String res : resourceList) {
+ resourceSet.add(Resource.fromJson(res));
}
}
}
return resourceSet;
}
- private void sendResourceResponseGet(ResourceId resourceId, Object o) {
- Client client = null;
- boolean broken = false;
- try {
- client = interpreterProcess.getClient();
- Gson gson = new Gson();
- String rid = gson.toJson(resourceId);
- ByteBuffer obj;
- if (o == null) {
- obj = ByteBuffer.allocate(0);
- } else {
- obj = Resource.serializeObject(o);
- }
- client.resourceResponseGet(rid, obj);
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- broken = true;
- } finally {
- if (client != null) {
- interpreterProcess.releaseClient(client, broken);
- }
- }
+ private void sendResourceResponseGet(final ResourceId resourceId, final Object o) {
+ interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ String rid = resourceId.toJson();
+ ByteBuffer obj;
+ if (o == null) {
+ obj = ByteBuffer.allocate(0);
+ } else {
+ obj = Resource.serializeObject(o);
+ }
+ client.resourceResponseGet(rid, obj);
+ return null;
+ }
+ }
+ );
}
- private Object getResource(ResourceId resourceId) {
- InterpreterGroup intpGroup = InterpreterGroup.getByInterpreterGroupId(
- resourceId.getResourcePoolId());
+ private Object getResource(final ResourceId resourceId) {
+ InterpreterGroup intpGroup = interpreterGroup.getInterpreterSetting()
+ .getInterpreterSettingManager()
+ .getInterpreterGroupById(resourceId.getResourcePoolId());
if (intpGroup == null) {
return null;
}
RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
- if (remoteInterpreterProcess == null) {
- ResourcePool localPool = intpGroup.getResourcePool();
- if (localPool != null) {
- return localPool.get(resourceId.getName());
- }
- } else if (interpreterProcess.isRunning()) {
- Client client = null;
- boolean broken = false;
- try {
- client = remoteInterpreterProcess.getClient();
- ByteBuffer res = client.resourceGet(
- resourceId.getNoteId(),
- resourceId.getParagraphId(),
- resourceId.getName());
- Object o = Resource.deserializeObject(res);
- return o;
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- broken = true;
- } finally {
- if (client != null) {
- intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
+ ByteBuffer buffer = remoteInterpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<ByteBuffer>() {
+ @Override
+ public ByteBuffer call(Client client) throws Exception {
+ return client.resourceGet(
+ resourceId.getNoteId(),
+ resourceId.getParagraphId(),
+ resourceId.getName());
+ }
}
- }
- }
- return null;
- }
+ );
- public void sendInvokeMethodResult(InvokeResourceMethodEventMessage message, Object o) {
- Client client = null;
- boolean broken = false;
try {
- client = interpreterProcess.getClient();
- Gson gson = new Gson();
- String invokeMessage = gson.toJson(message);
- ByteBuffer obj;
- if (o == null) {
- obj = ByteBuffer.allocate(0);
- } else {
- obj = Resource.serializeObject(o);
- }
- client.resourceResponseInvokeMethod(invokeMessage, obj);
+ Object o = Resource.deserializeObject(buffer);
+ return o;
} catch (Exception e) {
logger.error(e.getMessage(), e);
- broken = true;
- } finally {
- if (client != null) {
- interpreterProcess.releaseClient(client, broken);
- }
}
+ return null;
}
- private Object invokeResourceMethod(InvokeResourceMethodEventMessage message) {
- ResourceId resourceId = message.resourceId;
- InterpreterGroup intpGroup = InterpreterGroup.getByInterpreterGroupId(
- resourceId.getResourcePoolId());
+ public void sendInvokeMethodResult(final InvokeResourceMethodEventMessage message,
+ final Object o) {
+ interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ String invokeMessage = message.toJson();
+ ByteBuffer obj;
+ if (o == null) {
+ obj = ByteBuffer.allocate(0);
+ } else {
+ obj = Resource.serializeObject(o);
+ }
+ client.resourceResponseInvokeMethod(invokeMessage, obj);
+ return null;
+ }
+ }
+ );
+ }
+
+ private Object invokeResourceMethod(final InvokeResourceMethodEventMessage message) {
+ final ResourceId resourceId = message.resourceId;
+ InterpreterGroup intpGroup = interpreterGroup.getInterpreterSetting()
+ .getInterpreterSettingManager().getInterpreterGroupById(resourceId.getResourcePoolId());
if (intpGroup == null) {
return null;
}
@@ -529,25 +486,25 @@ public class RemoteInterpreterEventPoller extends Thread {
return null;
}
} else if (interpreterProcess.isRunning()) {
- Client client = null;
- boolean broken = false;
+ ByteBuffer res = interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<ByteBuffer>() {
+ @Override
+ public ByteBuffer call(Client client) throws Exception {
+ return client.resourceInvokeMethod(
+ resourceId.getNoteId(),
+ resourceId.getParagraphId(),
+ resourceId.getName(),
+ message.toJson());
+ }
+ }
+ );
+
try {
- client = remoteInterpreterProcess.getClient();
- ByteBuffer res = client.resourceInvokeMethod(
- resourceId.getNoteId(),
- resourceId.getParagraphId(),
- resourceId.getName(),
- gson.toJson(message));
- Object o = Resource.deserializeObject(res);
- return o;
+ return Resource.deserializeObject(res);
} catch (Exception e) {
logger.error(e.getMessage(), e);
- broken = true;
- } finally {
- if (client != null) {
- intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
- }
}
+ return null;
}
return null;
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
new file mode 100644
index 0000000..19356fb
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote;
+
+import org.apache.commons.exec.*;
+import org.apache.commons.exec.environment.EnvironmentUtils;
+import org.apache.zeppelin.helium.ApplicationEventListener;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Map;
+
+/**
+ * This class manages start / stop of remote interpreter process
+ */
+public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
+ implements ExecuteResultHandler {
+ private static final Logger logger = LoggerFactory.getLogger(
+ RemoteInterpreterManagedProcess.class);
+ private final String interpreterRunner;
+
+ private DefaultExecutor executor;
+ private ExecuteWatchdog watchdog;
+ boolean running = false;
+ private int port = -1;
+ private final String interpreterDir;
+ private final String localRepoDir;
+ private final String interpreterGroupName;
+
+ private Map<String, String> env;
+
+ public RemoteInterpreterManagedProcess(
+ String intpRunner,
+ String intpDir,
+ String localRepoDir,
+ Map<String, String> env,
+ int connectTimeout,
+ RemoteInterpreterProcessListener listener,
+ ApplicationEventListener appListener,
+ String interpreterGroupName) {
+ super(new RemoteInterpreterEventPoller(listener, appListener),
+ connectTimeout);
+ this.interpreterRunner = intpRunner;
+ this.env = env;
+ this.interpreterDir = intpDir;
+ this.localRepoDir = localRepoDir;
+ this.interpreterGroupName = interpreterGroupName;
+ }
+
+ RemoteInterpreterManagedProcess(String intpRunner,
+ String intpDir,
+ String localRepoDir,
+ Map<String, String> env,
+ RemoteInterpreterEventPoller remoteInterpreterEventPoller,
+ int connectTimeout,
+ String interpreterGroupName) {
+ super(remoteInterpreterEventPoller,
+ connectTimeout);
+ this.interpreterRunner = intpRunner;
+ this.env = env;
+ this.interpreterDir = intpDir;
+ this.localRepoDir = localRepoDir;
+ this.interpreterGroupName = interpreterGroupName;
+ }
+
+ @Override
+ public String getHost() {
+ return "localhost";
+ }
+
+ @Override
+ public int getPort() {
+ return port;
+ }
+
+ @Override
+ public void start(String userName, Boolean isUserImpersonate) {
+ // start server process
+ try {
+ port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces();
+ logger.info("Choose port {} for RemoteInterpreterProcess", port);
+ } catch (IOException e1) {
+ throw new InterpreterException(e1);
+ }
+
+ CommandLine cmdLine = CommandLine.parse(interpreterRunner);
+ cmdLine.addArgument("-d", false);
+ cmdLine.addArgument(interpreterDir, false);
+ cmdLine.addArgument("-p", false);
+ cmdLine.addArgument(Integer.toString(port), false);
+ if (isUserImpersonate && !userName.equals("anonymous")) {
+ cmdLine.addArgument("-u", false);
+ cmdLine.addArgument(userName, false);
+ }
+ cmdLine.addArgument("-l", false);
+ cmdLine.addArgument(localRepoDir, false);
+ cmdLine.addArgument("-g", false);
+ cmdLine.addArgument(interpreterGroupName, false);
+
+ executor = new DefaultExecutor();
+
+ ByteArrayOutputStream cmdOut = new ByteArrayOutputStream();
+ ProcessLogOutputStream processOutput = new ProcessLogOutputStream(logger);
+ processOutput.setOutputStream(cmdOut);
+
+ executor.setStreamHandler(new PumpStreamHandler(processOutput));
+ watchdog = new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT);
+ executor.setWatchdog(watchdog);
+
+ try {
+ Map procEnv = EnvironmentUtils.getProcEnvironment();
+ procEnv.putAll(env);
+
+ logger.info("Run interpreter process {}", cmdLine);
+ executor.execute(cmdLine, procEnv, this);
+ running = true;
+ } catch (IOException e) {
+ running = false;
+ throw new InterpreterException(e);
+ }
+
+
+ long startTime = System.currentTimeMillis();
+ while (System.currentTimeMillis() - startTime < getConnectTimeout()) {
+ if (!running) {
+ try {
+ cmdOut.flush();
+ } catch (IOException e) {
+ // nothing to do
+ }
+ throw new InterpreterException(new String(cmdOut.toByteArray()));
+ }
+
+ try {
+ if (RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", port)) {
+ break;
+ } else {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ logger.error("Exception in RemoteInterpreterProcess while synchronized reference " +
+ "Thread.sleep", e);
+ }
+ }
+ } catch (Exception e) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Remote interpreter not yet accessible at localhost:" + port);
+ }
+ }
+ }
+ processOutput.setOutputStream(null);
+ }
+
+ public void stop() {
+ if (isRunning()) {
+ logger.info("kill interpreter process");
+ try {
+ callRemoteFunction(new RemoteFunction<Void>() {
+ @Override
+ public Void call(RemoteInterpreterService.Client client) throws Exception {
+ client.shutdown();
+ return null;
+ }
+ });
+ } catch (Exception e) {
+ logger.warn("ignore the exception when shutting down");
+ }
+ watchdog.destroyProcess();
+ }
+
+ executor = null;
+ watchdog = null;
+ running = false;
+ logger.info("Remote process terminated");
+ }
+
+ @Override
+ public void onProcessComplete(int exitValue) {
+ logger.info("Interpreter process exited {}", exitValue);
+ running = false;
+
+ }
+
+ @Override
+ public void onProcessFailed(ExecuteException e) {
+ logger.info("Interpreter process failed {}", e);
+ running = false;
+ }
+
+ public boolean isRunning() {
+ return running;
+ }
+
+ private static class ProcessLogOutputStream extends LogOutputStream {
+
+ private Logger logger;
+ OutputStream out;
+
+ public ProcessLogOutputStream(Logger logger) {
+ this.logger = logger;
+ }
+
+ @Override
+ protected void processLine(String s, int i) {
+ this.logger.debug(s);
+ }
+
+ @Override
+ public void write(byte [] b) throws IOException {
+ super.write(b);
+
+ if (out != null) {
+ synchronized (this) {
+ if (out != null) {
+ out.write(b);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void write(byte [] b, int offset, int len) throws IOException {
+ super.write(b, offset, len);
+
+ if (out != null) {
+ synchronized (this) {
+ if (out != null) {
+ out.write(b, offset, len);
+ }
+ }
+ }
+ }
+
+ public void setOutputStream(OutputStream out) {
+ synchronized (this) {
+ this.out = out;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
index 1d48a1e..a78088c 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
@@ -20,10 +20,13 @@ import com.google.gson.Gson;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.thrift.TException;
import org.apache.zeppelin.helium.ApplicationEventListener;
+import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
/**
@@ -32,9 +35,6 @@ import java.util.concurrent.atomic.AtomicInteger;
public abstract class RemoteInterpreterProcess {
private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterProcess.class);
- // number of sessions that are attached to this process
- private final AtomicInteger referenceCount;
-
private GenericObjectPool<Client> clientPool;
private final RemoteInterpreterEventPoller remoteInterpreterEventPoller;
private final InterpreterContextRunnerPool interpreterContextRunnerPool;
@@ -46,16 +46,20 @@ public abstract class RemoteInterpreterProcess {
ApplicationEventListener appListener) {
this(new RemoteInterpreterEventPoller(listener, appListener),
connectTimeout);
+ this.remoteInterpreterEventPoller.setInterpreterProcess(this);
}
RemoteInterpreterProcess(RemoteInterpreterEventPoller remoteInterpreterEventPoller,
int connectTimeout) {
this.interpreterContextRunnerPool = new InterpreterContextRunnerPool();
- referenceCount = new AtomicInteger(0);
this.remoteInterpreterEventPoller = remoteInterpreterEventPoller;
this.connectTimeout = connectTimeout;
}
+ public RemoteInterpreterEventPoller getRemoteInterpreterEventPoller() {
+ return remoteInterpreterEventPoller;
+ }
+
public abstract String getHost();
public abstract int getPort();
public abstract void start(String userName, Boolean isUserImpersonate);
@@ -66,37 +70,18 @@ public abstract class RemoteInterpreterProcess {
return connectTimeout;
}
- public int reference(InterpreterGroup interpreterGroup, String userName,
- Boolean isUserImpersonate) {
- synchronized (referenceCount) {
- if (!isRunning()) {
- start(userName, isUserImpersonate);
- }
-
- if (clientPool == null) {
- clientPool = new GenericObjectPool<>(new ClientFactory(getHost(), getPort()));
- clientPool.setTestOnBorrow(true);
-
- remoteInterpreterEventPoller.setInterpreterGroup(interpreterGroup);
- remoteInterpreterEventPoller.setInterpreterProcess(this);
- remoteInterpreterEventPoller.start();
- }
- return referenceCount.incrementAndGet();
- }
- }
-
- public Client getClient() throws Exception {
+ public synchronized Client getClient() throws Exception {
if (clientPool == null || clientPool.isClosed()) {
- return null;
+ clientPool = new GenericObjectPool<>(new ClientFactory(getHost(), getPort()));
}
return clientPool.borrowObject();
}
- public void releaseClient(Client client) {
+ private void releaseClient(Client client) {
releaseClient(client, false);
}
- public void releaseClient(Client client, boolean broken) {
+ private void releaseClient(Client client, boolean broken) {
if (broken) {
releaseBrokenClient(client);
} else {
@@ -108,7 +93,7 @@ public abstract class RemoteInterpreterProcess {
}
}
- public void releaseBrokenClient(Client client) {
+ private void releaseBrokenClient(Client client) {
try {
clientPool.invalidateObject(client);
} catch (Exception e) {
@@ -116,90 +101,6 @@ public abstract class RemoteInterpreterProcess {
}
}
- public int dereference() {
- synchronized (referenceCount) {
- int r = referenceCount.decrementAndGet();
- if (r == 0) {
- logger.info("shutdown interpreter process");
- remoteInterpreterEventPoller.shutdown();
-
- // first try shutdown
- Client client = null;
- try {
- client = getClient();
- client.shutdown();
- } catch (Exception e) {
- // safely ignore exception while client.shutdown() may terminates remote process
- logger.info("Exception in RemoteInterpreterProcess while synchronized dereference, can " +
- "safely ignore exception while client.shutdown() may terminates remote process");
- logger.debug(e.getMessage(), e);
- } finally {
- if (client != null) {
- // no longer used
- releaseBrokenClient(client);
- }
- }
-
- clientPool.clear();
- clientPool.close();
-
- // wait for some time (connectTimeout) and force kill
- // remote process server.serve() loop is not always finishing gracefully
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() - startTime < connectTimeout) {
- if (this.isRunning()) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- logger.error("Exception in RemoteInterpreterProcess while synchronized dereference " +
- "Thread.sleep", e);
- }
- } else {
- break;
- }
- }
- }
- return r;
- }
- }
-
- public int referenceCount() {
- synchronized (referenceCount) {
- return referenceCount.get();
- }
- }
-
- public int getNumActiveClient() {
- if (clientPool == null) {
- return 0;
- } else {
- return clientPool.getNumActive();
- }
- }
-
- public int getNumIdleClient() {
- if (clientPool == null) {
- return 0;
- } else {
- return clientPool.getNumIdle();
- }
- }
-
- public void setMaxPoolSize(int size) {
- if (clientPool != null) {
- //Size + 2 for progress poller , cancel operation
- clientPool.setMaxTotal(size + 2);
- }
- }
-
- public int getMaxPoolSize() {
- if (clientPool != null) {
- return clientPool.getMaxTotal();
- } else {
- return 0;
- }
- }
-
/**
* Called when angular object is updated in client side to propagate
* change to the remote process
@@ -239,4 +140,33 @@ public abstract class RemoteInterpreterProcess {
public InterpreterContextRunnerPool getInterpreterContextRunnerPool() {
return interpreterContextRunnerPool;
}
+
+ public <T> T callRemoteFunction(RemoteFunction<T> func) {
+ Client client = null;
+ boolean broken = false;
+ try {
+ client = getClient();
+ if (client != null) {
+ return func.call(client);
+ }
+ } catch (TException e) {
+ broken = true;
+ throw new InterpreterException(e);
+ } catch (Exception e1) {
+ throw new InterpreterException(e1);
+ } finally {
+ if (client != null) {
+ releaseClient(client, broken);
+ }
+ }
+ return null;
+ }
+
+ /**
+ *
+ * @param <T>
+ */
+ public interface RemoteFunction<T> {
+ T call(Client client) throws Exception;
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java
new file mode 100644
index 0000000..bb176be
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.interpreter.remote;
+
+import org.apache.zeppelin.helium.ApplicationEventListener;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class connects to existing process
+ */
+public class RemoteInterpreterRunningProcess extends RemoteInterpreterProcess {
+ private final Logger logger = LoggerFactory.getLogger(RemoteInterpreterRunningProcess.class);
+ private final String host;
+ private final int port;
+
+ public RemoteInterpreterRunningProcess(
+ int connectTimeout,
+ RemoteInterpreterProcessListener listener,
+ ApplicationEventListener appListener,
+ String host,
+ int port
+ ) {
+ super(connectTimeout, listener, appListener);
+ this.host = host;
+ this.port = port;
+ }
+
+ @Override
+ public String getHost() {
+ return host;
+ }
+
+ @Override
+ public int getPort() {
+ return port;
+ }
+
+ @Override
+ public void start(String userName, Boolean isUserImpersonate) {
+ // assume process is externally managed. nothing to do
+ }
+
+ @Override
+ public void stop() {
+ // assume process is externally managed. nothing to do
+ }
+
+ @Override
+ public boolean isRunning() {
+ return RemoteInterpreterUtils.checkIfRemoteEndpointAccessible(getHost(), getPort());
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
index 3853468..3d8123e 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
@@ -106,6 +106,7 @@ public class RemoteInterpreterServer
@Override
public void shutdown() throws TException {
+ logger.info("Shutting down...");
eventClient.waitForEventQueueBecomesEmpty(DEFAULT_SHUTDOWN_TIMEOUT);
if (interpreterGroup != null) {
interpreterGroup.close();
@@ -159,7 +160,7 @@ public class RemoteInterpreterServer
}
@Override
- public void createInterpreter(String interpreterGroupId, String sessionKey, String
+ public void createInterpreter(String interpreterGroupId, String sessionId, String
className, Map<String, String> properties, String userName) throws TException {
if (interpreterGroup == null) {
interpreterGroup = new InterpreterGroup(interpreterGroupId);
@@ -190,20 +191,11 @@ public class RemoteInterpreterServer
replClass.getConstructor(new Class[] {Properties.class});
Interpreter repl = constructor.newInstance(p);
repl.setClassloaderUrls(new URL[]{});
-
- synchronized (interpreterGroup) {
- List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
- if (interpreters == null) {
- interpreters = new LinkedList<>();
- interpreterGroup.put(sessionKey, interpreters);
- }
-
- interpreters.add(new LazyOpenInterpreter(repl));
- }
-
logger.info("Instantiate interpreter {}", className);
repl.setInterpreterGroup(interpreterGroup);
repl.setUserName(userName);
+
+ interpreterGroup.addInterpreterToSession(new LazyOpenInterpreter(repl), sessionId);
} catch (ClassNotFoundException | NoSuchMethodException | SecurityException
| InstantiationException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException e) {
@@ -237,13 +229,13 @@ public class RemoteInterpreterServer
}
}
- protected Interpreter getInterpreter(String sessionKey, String className) throws TException {
+ protected Interpreter getInterpreter(String sessionId, String className) throws TException {
if (interpreterGroup == null) {
throw new TException(
new InterpreterException("Interpreter instance " + className + " not created"));
}
synchronized (interpreterGroup) {
- List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
+ List<Interpreter> interpreters = interpreterGroup.get(sessionId);
if (interpreters == null) {
throw new TException(
new InterpreterException("Interpreter " + className + " not initialized"));
@@ -259,19 +251,20 @@ public class RemoteInterpreterServer
}
@Override
- public void open(String noteId, String className) throws TException {
- Interpreter intp = getInterpreter(noteId, className);
+ public void open(String sessionId, String className) throws TException {
+ logger.info(String.format("Open Interpreter %s for session %s ", className, sessionId));
+ Interpreter intp = getInterpreter(sessionId, className);
intp.open();
}
@Override
- public void close(String sessionKey, String className) throws TException {
+ public void close(String sessionId, String className) throws TException {
// unload all applications
for (String appId : runningApplications.keySet()) {
RunningApplication appInfo = runningApplications.get(appId);
// see NoteInterpreterLoader.SHARED_SESSION
- if (appInfo.noteId.equals(sessionKey) || sessionKey.equals("shared_session")) {
+ if (appInfo.noteId.equals(sessionId) || sessionId.equals("shared_session")) {
try {
logger.info("Unload App {} ", appInfo.pkg.getName());
appInfo.app.unload();
@@ -286,7 +279,7 @@ public class RemoteInterpreterServer
// close interpreters
List<Interpreter> interpreters;
synchronized (interpreterGroup) {
- interpreters = interpreterGroup.get(sessionKey);
+ interpreters = interpreterGroup.get(sessionId);
}
if (interpreters != null) {
Iterator<Interpreter> it = interpreters.iterator();
@@ -322,7 +315,6 @@ public class RemoteInterpreterServer
intp,
st,
context);
-
scheduler.submit(job);
while (!job.isTerminated()) {
@@ -566,30 +558,34 @@ public class RemoteInterpreterServer
}
@Override
- public int getProgress(String noteId, String className,
+ public int getProgress(String sessionId, String className,
RemoteInterpreterContext interpreterContext)
throws TException {
Integer manuallyProvidedProgress = progressMap.get(interpreterContext.getParagraphId());
if (manuallyProvidedProgress != null) {
return manuallyProvidedProgress;
} else {
- Interpreter intp = getInterpreter(noteId, className);
+ Interpreter intp = getInterpreter(sessionId, className);
+ if (intp == null) {
+ throw new TException("No interpreter {} existed for session {}".format(
+ className, sessionId));
+ }
return intp.getProgress(convert(interpreterContext, null));
}
}
@Override
- public String getFormType(String noteId, String className) throws TException {
- Interpreter intp = getInterpreter(noteId, className);
+ public String getFormType(String sessionId, String className) throws TException {
+ Interpreter intp = getInterpreter(sessionId, className);
return intp.getFormType().toString();
}
@Override
- public List<InterpreterCompletion> completion(String noteId,
+ public List<InterpreterCompletion> completion(String sessionId,
String className, String buf, int cursor, RemoteInterpreterContext remoteInterpreterContext)
throws TException {
- Interpreter intp = getInterpreter(noteId, className);
+ Interpreter intp = getInterpreter(sessionId, className);
List completion = intp.completion(buf, cursor, convert(remoteInterpreterContext, null));
return completion;
}
@@ -766,16 +762,16 @@ public class RemoteInterpreterServer
}
@Override
- public String getStatus(String sessionKey, String jobId)
+ public String getStatus(String sessionId, String jobId)
throws TException {
if (interpreterGroup == null) {
- return "Unknown";
+ return Status.UNKNOWN.name();
}
synchronized (interpreterGroup) {
- List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
+ List<Interpreter> interpreters = interpreterGroup.get(sessionId);
if (interpreters == null) {
- return "Unknown";
+ return Status.UNKNOWN.name();
}
for (Interpreter intp : interpreters) {
@@ -792,7 +788,7 @@ public class RemoteInterpreterServer
}
}
}
- return "Unknown";
+ return Status.UNKNOWN.name();
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java
deleted file mode 100644
index b26995a..0000000
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.resource;
-
-import org.apache.zeppelin.interpreter.InterpreterGroup;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
-import org.slf4j.Logger;
-
-import java.util.List;
-
-/**
- * Utilities for ResourcePool
- */
-public class ResourcePoolUtils {
- static Logger logger = org.slf4j.LoggerFactory.getLogger(ResourcePoolUtils.class);
-
- public static ResourceSet getAllResources() {
- return getAllResourcesExcept(null);
- }
-
- public static ResourceSet getAllResourcesExcept(String interpreterGroupExcludsion) {
- ResourceSet resourceSet = new ResourceSet();
- for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
- if (interpreterGroupExcludsion != null &&
- intpGroup.getId().equals(interpreterGroupExcludsion)) {
- continue;
- }
-
- RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
- if (remoteInterpreterProcess == null) {
- ResourcePool localPool = intpGroup.getResourcePool();
- if (localPool != null) {
- resourceSet.addAll(localPool.getAll());
- }
- } else if (remoteInterpreterProcess.isRunning()) {
- RemoteInterpreterService.Client client = null;
- boolean broken = false;
- try {
- client = remoteInterpreterProcess.getClient();
- if (client == null) {
- // remote interpreter may not started yet or terminated.
- continue;
- }
- List<String> resourceList = client.resourcePoolGetAll();
- for (String res : resourceList) {
- resourceSet.add(Resource.fromJson(res));
- }
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- broken = true;
- } finally {
- if (client != null) {
- intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
- }
- }
- }
- }
- return resourceSet;
- }
-
- public static void removeResourcesBelongsToNote(String noteId) {
- removeResourcesBelongsToParagraph(noteId, null);
- }
-
- public static void removeResourcesBelongsToParagraph(String noteId, String paragraphId) {
- for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
- ResourceSet resourceSet = new ResourceSet();
- RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
- if (remoteInterpreterProcess == null) {
- ResourcePool localPool = intpGroup.getResourcePool();
- if (localPool != null) {
- resourceSet.addAll(localPool.getAll());
- }
- if (noteId != null) {
- resourceSet = resourceSet.filterByNoteId(noteId);
- }
- if (paragraphId != null) {
- resourceSet = resourceSet.filterByParagraphId(paragraphId);
- }
-
- for (Resource r : resourceSet) {
- localPool.remove(
- r.getResourceId().getNoteId(),
- r.getResourceId().getParagraphId(),
- r.getResourceId().getName());
- }
- } else if (remoteInterpreterProcess.isRunning()) {
- RemoteInterpreterService.Client client = null;
- boolean broken = false;
- try {
- client = remoteInterpreterProcess.getClient();
- List<String> resourceList = client.resourcePoolGetAll();
- for (String res : resourceList) {
- resourceSet.add(Resource.fromJson(res));
- }
-
- if (noteId != null) {
- resourceSet = resourceSet.filterByNoteId(noteId);
- }
- if (paragraphId != null) {
- resourceSet = resourceSet.filterByParagraphId(paragraphId);
- }
-
- for (Resource r : resourceSet) {
- client.resourceRemove(
- r.getResourceId().getNoteId(),
- r.getResourceId().getParagraphId(),
- r.getResourceId().getName());
- }
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- broken = true;
- } finally {
- if (client != null) {
- intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
- }
- }
- }
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java
index d0025d8..191902a 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java
@@ -41,6 +41,7 @@ public abstract class Job {
/**
* Job status.
*
+ * UNKNOWN - Job is not found in remote
* READY - Job is not running, ready to run.
* PENDING - Job is submitted to scheduler. but not running yet
* RUNNING - Job is running.
@@ -48,8 +49,8 @@ public abstract class Job {
* ERROR - Job finished run. with error
* ABORT - Job finished by abort
*/
- public static enum Status {
- READY, PENDING, RUNNING, FINISHED, ERROR, ABORT;
+ public enum Status {
+ UNKNOWN, READY, PENDING, RUNNING, FINISHED, ERROR, ABORT;
public boolean isReady() {
return this == READY;
@@ -70,14 +71,14 @@ public abstract class Job {
Date dateCreated;
Date dateStarted;
Date dateFinished;
- Status status;
+ volatile Status status;
static Logger LOGGER = LoggerFactory.getLogger(Job.class);
transient boolean aborted = false;
- private String errorMessage;
- private transient Throwable exception;
+ private volatile String errorMessage;
+ private transient volatile Throwable exception;
private transient JobListener listener;
private long progressUpdateIntervalMs;
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java
index f9ddc4e..e41540b 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java
@@ -17,11 +17,9 @@
package org.apache.zeppelin.scheduler;
-import org.apache.thrift.TException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.scheduler.Job.Status;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,6 +32,7 @@ import java.util.concurrent.ExecutorService;
/**
* RemoteScheduler runs in ZeppelinServer and proxies Scheduler running on RemoteInterpreter
+ *
*/
public class RemoteScheduler implements Scheduler {
Logger logger = LoggerFactory.getLogger(RemoteScheduler.class);
@@ -45,17 +44,17 @@ public class RemoteScheduler implements Scheduler {
boolean terminate = false;
private String name;
private int maxConcurrency;
- private final String noteId;
- private RemoteInterpreterProcess interpreterProcess;
+ private final String sessionId;
+ private RemoteInterpreter remoteInterpreter;
- public RemoteScheduler(String name, ExecutorService executor, String noteId,
- RemoteInterpreterProcess interpreterProcess, SchedulerListener listener,
+ public RemoteScheduler(String name, ExecutorService executor, String sessionId,
+ RemoteInterpreter remoteInterpreter, SchedulerListener listener,
int maxConcurrency) {
this.name = name;
this.executor = executor;
this.listener = listener;
- this.noteId = noteId;
- this.interpreterProcess = interpreterProcess;
+ this.sessionId = sessionId;
+ this.remoteInterpreter = remoteInterpreter;
this.maxConcurrency = maxConcurrency;
}
@@ -167,14 +166,15 @@ public class RemoteScheduler implements Scheduler {
private long initialPeriodMsec;
private long initialPeriodCheckIntervalMsec;
private long checkIntervalMsec;
- private boolean terminate;
+ private volatile boolean terminate;
private JobListener listener;
private Job job;
- Status lastStatus;
+ volatile Status lastStatus;
public JobStatusPoller(long initialPeriodMsec,
long initialPeriodCheckIntervalMsec, long checkIntervalMsec, Job job,
JobListener listener) {
+ setName("JobStatusPoller-" + job.getId());
this.initialPeriodMsec = initialPeriodMsec;
this.initialPeriodCheckIntervalMsec = initialPeriodCheckIntervalMsec;
this.checkIntervalMsec = checkIntervalMsec;
@@ -209,7 +209,7 @@ public class RemoteScheduler implements Scheduler {
}
Status newStatus = getStatus();
- if (newStatus == null) { // unknown
+ if (newStatus == Status.UNKNOWN) { // unknown
continue;
}
@@ -231,7 +231,9 @@ public class RemoteScheduler implements Scheduler {
private Status getLastStatus() {
if (terminate == true) {
- if (lastStatus != Status.FINISHED &&
+ if (job.getErrorMessage() != null) {
+ return Status.ERROR;
+ } else if (lastStatus != Status.FINISHED &&
lastStatus != Status.ERROR &&
lastStatus != Status.ABORT) {
return Status.FINISHED;
@@ -239,58 +241,35 @@ public class RemoteScheduler implements Scheduler {
return (lastStatus == null) ? Status.FINISHED : lastStatus;
}
} else {
- return (lastStatus == null) ? Status.FINISHED : lastStatus;
+ return (lastStatus == null) ? Status.UNKNOWN : lastStatus;
}
}
public synchronized Job.Status getStatus() {
- if (interpreterProcess.referenceCount() <= 0) {
+ if (!remoteInterpreter.isOpened()) {
return getLastStatus();
}
-
- Client client;
- try {
- client = interpreterProcess.getClient();
- } catch (Exception e) {
- logger.error("Can't get status information", e);
- lastStatus = Status.ERROR;
- return Status.ERROR;
- }
-
- boolean broken = false;
- try {
- String statusStr = client.getStatus(noteId, job.getId());
- if ("Unknown".equals(statusStr)) {
- // not found this job in the remote schedulers.
- // maybe not submitted, maybe already finished
- //Status status = getLastStatus();
- listener.afterStatusChange(job, null, null);
- return job.getStatus();
- }
- Status status = Status.valueOf(statusStr);
- lastStatus = status;
- listener.afterStatusChange(job, null, status);
- return status;
- } catch (TException e) {
- broken = true;
- logger.error("Can't get status information", e);
- lastStatus = Status.ERROR;
- return Status.ERROR;
- } catch (Exception e) {
- logger.error("Unknown status", e);
- lastStatus = Status.ERROR;
- return Status.ERROR;
- } finally {
- interpreterProcess.releaseClient(client, broken);
+ Status status = Status.valueOf(remoteInterpreter.getStatus(job.getId()));
+ if (status == Status.UNKNOWN) {
+ // not found this job in the remote schedulers.
+ // maybe not submitted, maybe already finished
+ //Status status = getLastStatus();
+ listener.afterStatusChange(job, null, null);
+ return job.getStatus();
}
+ lastStatus = status;
+ listener.afterStatusChange(job, null, status);
+ return status;
}
}
+ //TODO(zjffdu) need to refactor the schdule module which is too complicated
private class JobRunner implements Runnable, JobListener {
+ private final Logger logger = LoggerFactory.getLogger(JobRunner.class);
private Scheduler scheduler;
private Job job;
- private boolean jobExecuted;
- boolean jobSubmittedRemotely;
+ private volatile boolean jobExecuted;
+ volatile boolean jobSubmittedRemotely;
public JobRunner(Scheduler scheduler, Job job) {
this.scheduler = scheduler;
@@ -338,20 +317,22 @@ public class RemoteScheduler implements Scheduler {
}
// set job status based on result.
- Status lastStatus = jobStatusPoller.getStatus();
Object jobResult = job.getReturn();
- if (jobResult != null && jobResult instanceof InterpreterResult) {
- if (((InterpreterResult) jobResult).code() == Code.ERROR) {
- lastStatus = Status.ERROR;
- }
- }
- if (job.getException() != null) {
- lastStatus = Status.ERROR;
+ if (job.isAborted()) {
+ job.setStatus(Status.ABORT);
+ } else if (job.getException() != null) {
+// logger.info("Job ABORT, " + job.getId());
+ job.setStatus(Status.ERROR);
+ } else if (jobResult != null && jobResult instanceof InterpreterResult
+ && ((InterpreterResult) jobResult).code() == Code.ERROR) {
+// logger.info("Job Error, " + job.getId());
+ job.setStatus(Status.ERROR);
+ } else {
+// logger.info("Job Finished, " + job.getId());
+ job.setStatus(Status.FINISHED);
}
synchronized (queue) {
- job.setStatus(lastStatus);
-
if (listener != null) {
listener.jobFinished(scheduler, job);
}
@@ -374,25 +355,6 @@ public class RemoteScheduler implements Scheduler {
@Override
public void afterStatusChange(Job job, Status before, Status after) {
- if (after == null) { // unknown. maybe before sumitted remotely, maybe already finished.
- if (jobExecuted) {
- jobSubmittedRemotely = true;
- Object jobResult = job.getReturn();
- if (job.isAborted()) {
- job.setStatus(Status.ABORT);
- } else if (job.getException() != null) {
- job.setStatus(Status.ERROR);
- } else if (jobResult != null && jobResult instanceof InterpreterResult
- && ((InterpreterResult) jobResult).code() == Code.ERROR) {
- job.setStatus(Status.ERROR);
- } else {
- job.setStatus(Status.FINISHED);
- }
- }
- return;
- }
-
-
// Update remoteStatus
if (jobExecuted == false) {
if (after == Status.FINISHED || after == Status.ABORT
@@ -402,14 +364,18 @@ public class RemoteScheduler implements Scheduler {
return;
} else if (after == Status.RUNNING) {
jobSubmittedRemotely = true;
+ job.setStatus(Status.RUNNING);
+// logger.info("Job RUNNING, " + job.getId());
}
} else {
jobSubmittedRemotely = true;
}
- // status polled by status poller
- if (job.getStatus() != after) {
- job.setStatus(after);
+ // only set status when it is RUNNING
+ // We would set other status based on the interpret result
+ if (after == Status.RUNNING) {
+// logger.info("Job RUNNING, " + job.getId());
+ job.setStatus(Status.RUNNING);
}
}
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java
index af52dec..5871ca5 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java
@@ -24,17 +24,18 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * TODO(moon) : add description.
+ * Factory class for creating schedulers
+ *
*/
public class SchedulerFactory implements SchedulerListener {
private static final Logger logger = LoggerFactory.getLogger(SchedulerFactory.class);
- ExecutorService executor;
- Map<String, Scheduler> schedulers = new LinkedHashMap<>();
+ private ExecutorService executor;
+ private Map<String, Scheduler> schedulers = new LinkedHashMap<>();
private static SchedulerFactory singleton;
private static Long singletonLock = new Long(0);
@@ -54,17 +55,17 @@ public class SchedulerFactory implements SchedulerListener {
return singleton;
}
- public SchedulerFactory() throws Exception {
- executor = ExecutorFactory.singleton().createOrGet("schedulerFactory", 100);
+ SchedulerFactory() throws Exception {
+ executor = ExecutorFactory.singleton().createOrGet("SchedulerFactory", 100);
}
public void destroy() {
- ExecutorFactory.singleton().shutdown("schedulerFactory");
+ ExecutorFactory.singleton().shutdown("SchedulerFactory");
}
public Scheduler createOrGetFIFOScheduler(String name) {
synchronized (schedulers) {
- if (schedulers.containsKey(name) == false) {
+ if (!schedulers.containsKey(name)) {
Scheduler s = new FIFOScheduler(name, executor, this);
schedulers.put(name, s);
executor.execute(s);
@@ -75,7 +76,7 @@ public class SchedulerFactory implements SchedulerListener {
public Scheduler createOrGetParallelScheduler(String name, int maxConcurrency) {
synchronized (schedulers) {
- if (schedulers.containsKey(name) == false) {
+ if (!schedulers.containsKey(name)) {
Scheduler s = new ParallelScheduler(name, executor, this, maxConcurrency);
schedulers.put(name, s);
executor.execute(s);
@@ -86,17 +87,17 @@ public class SchedulerFactory implements SchedulerListener {
public Scheduler createOrGetRemoteScheduler(
String name,
- String noteId,
- RemoteInterpreterProcess interpreterProcess,
+ String sessionId,
+ RemoteInterpreter remoteInterpreter,
int maxConcurrency) {
synchronized (schedulers) {
- if (schedulers.containsKey(name) == false) {
+ if (!schedulers.containsKey(name)) {
Scheduler s = new RemoteScheduler(
name,
executor,
- noteId,
- interpreterProcess,
+ sessionId,
+ remoteInterpreter,
this,
maxConcurrency);
schedulers.put(name, s);
@@ -106,38 +107,24 @@ public class SchedulerFactory implements SchedulerListener {
}
}
- public Scheduler removeScheduler(String name) {
+ public void removeScheduler(String name) {
synchronized (schedulers) {
Scheduler s = schedulers.remove(name);
if (s != null) {
s.stop();
}
}
- return null;
- }
-
- public Collection<Scheduler> listScheduler(String name) {
- List<Scheduler> s = new LinkedList<>();
- synchronized (schedulers) {
- for (Scheduler ss : schedulers.values()) {
- s.add(ss);
- }
- }
- return s;
}
@Override
public void jobStarted(Scheduler scheduler, Job job) {
- logger.info("Job " + job.getJobName() + " started by scheduler " + scheduler.getName());
+ logger.info("Job " + job.getId() + " started by scheduler " + scheduler.getName());
}
@Override
public void jobFinished(Scheduler scheduler, Job job) {
- logger.info("Job " + job.getJobName() + " finished by scheduler " + scheduler.getName());
+ logger.info("Job " + job.getId() + " finished by scheduler " + scheduler.getName());
}
-
-
-
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/TableDataProxy.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/TableDataProxy.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/TableDataProxy.java
index 8673476..1926528 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/TableDataProxy.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/TableDataProxy.java
@@ -17,7 +17,6 @@
package org.apache.zeppelin.tabledata;
import org.apache.zeppelin.resource.Resource;
-import org.apache.zeppelin.resource.ResourcePoolUtils;
import java.util.Iterator;
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java
new file mode 100644
index 0000000..14c03a1
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.util;
+
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * Generate Tiny ID.
+ */
+public class IdHashes {
+ private static final char[] DICTIONARY = new char[] {'1', '2', '3', '4', '5', '6', '7', '8', '9',
+ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
+ 'W', 'X', 'Y', 'Z'};
+
+ /**
+ * encodes the given string into the base of the dictionary provided in the constructor.
+ *
+ * @param value the number to encode.
+ * @return the encoded string.
+ */
+ private static String encode(Long value) {
+
+ List<Character> result = new ArrayList<>();
+ BigInteger base = new BigInteger("" + DICTIONARY.length);
+ int exponent = 1;
+ BigInteger remaining = new BigInteger(value.toString());
+ while (true) {
+ BigInteger a = base.pow(exponent); // 16^1 = 16
+ BigInteger b = remaining.mod(a); // 119 % 16 = 7 | 112 % 256 = 112
+ BigInteger c = base.pow(exponent - 1);
+ BigInteger d = b.divide(c);
+
+ // if d > dictionary.length, we have a problem. but BigInteger doesnt have
+ // a greater than method :-( hope for the best. theoretically, d is always
+ // an index of the dictionary!
+ result.add(DICTIONARY[d.intValue()]);
+ remaining = remaining.subtract(b); // 119 - 7 = 112 | 112 - 112 = 0
+
+ // finished?
+ if (remaining.equals(BigInteger.ZERO)) {
+ break;
+ }
+
+ exponent++;
+ }
+
+ // need to reverse it, since the start of the list contains the least significant values
+ StringBuffer sb = new StringBuffer();
+ for (int i = result.size() - 1; i >= 0; i--) {
+ sb.append(result.get(i));
+ }
+ return sb.toString();
+ }
+
+ public static String generateId() {
+ return encode(System.currentTimeMillis() + new Random().nextInt());
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/Util.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/Util.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/Util.java
new file mode 100644
index 0000000..6153f49
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/Util.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.util;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.io.IOException;
+import java.util.Properties;
+
+/**
+ * TODO(moon) : add description.
+ */
+public class Util {
+ private static final String PROJECT_PROPERTIES_VERSION_KEY = "version";
+ private static final String GIT_PROPERTIES_COMMIT_ID_KEY = "git.commit.id.abbrev";
+ private static final String GIT_PROPERTIES_COMMIT_TS_KEY = "git.commit.time";
+
+ private static Properties projectProperties;
+ private static Properties gitProperties;
+
+ static {
+ projectProperties = new Properties();
+ gitProperties = new Properties();
+ try {
+ projectProperties.load(Util.class.getResourceAsStream("/project.properties"));
+ gitProperties.load(Util.class.getResourceAsStream("/git.properties"));
+ } catch (IOException e) {
+ //Fail to read project.properties
+ }
+ }
+
+ /**
+ * Get Zeppelin version
+ *
+ * @return Current Zeppelin version
+ */
+ public static String getVersion() {
+ return StringUtils.defaultIfEmpty(projectProperties.getProperty(PROJECT_PROPERTIES_VERSION_KEY),
+ StringUtils.EMPTY);
+ }
+
+ /**
+ * Get Zeppelin Git latest commit id
+ *
+ * @return Latest Zeppelin commit id
+ */
+ public static String getGitCommitId() {
+ return StringUtils.defaultIfEmpty(gitProperties.getProperty(GIT_PROPERTIES_COMMIT_ID_KEY),
+ StringUtils.EMPTY);
+ }
+
+ /**
+ * Get Zeppelin Git latest commit timestamp
+ *
+ * @return Latest Zeppelin commit timestamp
+ */
+ public static String getGitTimestamp() {
+ return StringUtils.defaultIfEmpty(gitProperties.getProperty(GIT_PROPERTIES_COMMIT_TS_KEY),
+ StringUtils.EMPTY);
+ }
+}
[08/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java
new file mode 100644
index 0000000..21d7526
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java
@@ -0,0 +1,74 @@
+package org.apache.zeppelin.interpreter;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.display.AngularObjectRegistryListener;
+import org.apache.zeppelin.helium.ApplicationEventListener;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
+import org.junit.After;
+import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+import static org.mockito.Mockito.mock;
+
+
+/**
+ * This class will load configuration files under
+ * src/test/resources/interpreter
+ * src/test/resources/conf
+ *
+ * to construct InterpreterSettingManager and InterpreterFactory properly
+ *
+ */
+public abstract class AbstractInterpreterTest {
+ protected static final Logger LOGGER = LoggerFactory.getLogger(AbstractInterpreterTest.class);
+ private static final String INTERPRETER_SCRIPT =
+ System.getProperty("os.name").startsWith("Windows") ?
+ "../bin/interpreter.cmd" :
+ "../bin/interpreter.sh";
+
+ protected InterpreterSettingManager interpreterSettingManager;
+ protected InterpreterFactory interpreterFactory;
+ protected File testRootDir;
+ protected File interpreterDir;
+ protected File confDir;
+ protected File notebookDir;
+ protected ZeppelinConfiguration conf;
+
+ @Before
+ public void setUp() throws Exception {
+ // copy the resources files to a temp folder
+ testRootDir = new File(System.getProperty("java.io.tmpdir") + "/Zeppelin_Test_" + System.currentTimeMillis());
+ testRootDir.mkdirs();
+ LOGGER.info("Create tmp directory: {} as root folder of ZEPPELIN_INTERPRETER_DIR & ZEPPELIN_CONF_DIR", testRootDir.getAbsolutePath());
+ interpreterDir = new File(testRootDir, "interpreter");
+ confDir = new File(testRootDir, "conf");
+ notebookDir = new File(testRootDir, "notebook");
+
+ interpreterDir.mkdirs();
+ confDir.mkdirs();
+ notebookDir.mkdirs();
+
+ FileUtils.copyDirectory(new File("src/test/resources/interpreter"), interpreterDir);
+ FileUtils.copyDirectory(new File("src/test/resources/conf"), confDir);
+
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), confDir.getAbsolutePath());
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_DIR.getVarName(), interpreterDir.getAbsolutePath());
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER.getVarName(), INTERPRETER_SCRIPT);
+
+ conf = new ZeppelinConfiguration();
+ interpreterSettingManager = new InterpreterSettingManager(conf,
+ mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
+ interpreterFactory = new InterpreterFactory(interpreterSettingManager);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ interpreterSettingManager.close();
+ FileUtils.deleteDirectory(testRootDir);
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DoubleEchoInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DoubleEchoInterpreter.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DoubleEchoInterpreter.java
new file mode 100644
index 0000000..8eea4b2
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DoubleEchoInterpreter.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.zeppelin.interpreter;
+
+import java.util.Properties;
+
+/**
+ *
+ */
+public class DoubleEchoInterpreter extends Interpreter {
+
+ public DoubleEchoInterpreter(Properties property) {
+ super(property);
+ }
+
+ @Override
+ public void open() {
+
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ return new InterpreterResult(InterpreterResult.Code.SUCCESS, st + "," + st);
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+
+ }
+
+ @Override
+ public FormType getFormType() {
+ return null;
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DummyInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DummyInterpreter.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DummyInterpreter.java
deleted file mode 100644
index a7a6eb9..0000000
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DummyInterpreter.java
+++ /dev/null
@@ -1,43 +0,0 @@
-package org.apache.zeppelin.interpreter;
-
-import java.util.Properties;
-
-/**
- *
- */
-public class DummyInterpreter extends Interpreter {
-
- public DummyInterpreter(Properties property) {
- super(property);
- }
-
- @Override
- public void open() {
-
- }
-
- @Override
- public void close() {
-
- }
-
- @Override
- public InterpreterResult interpret(String st, InterpreterContext context) {
- return null;
- }
-
- @Override
- public void cancel(InterpreterContext context) {
-
- }
-
- @Override
- public FormType getFormType() {
- return null;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java
new file mode 100644
index 0000000..e7a04f3
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.zeppelin.interpreter;
+
+import java.util.Properties;
+
+/**
+ * Just return the received statement back
+ */
+public class EchoInterpreter extends Interpreter {
+
+ public EchoInterpreter(Properties property) {
+ super(property);
+ }
+
+ @Override
+ public void open() {
+
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ if (Boolean.parseBoolean(property.getProperty("zeppelin.interpreter.echo.fail", "false"))) {
+ return new InterpreterResult(InterpreterResult.Code.ERROR);
+ } else {
+ return new InterpreterResult(InterpreterResult.Code.SUCCESS, st);
+ }
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+
+ }
+
+ @Override
+ public FormType getFormType() {
+ return FormType.NATIVE;
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
new file mode 100644
index 0000000..f3137d9
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class InterpreterFactoryTest extends AbstractInterpreterTest {
+
+ @Test
+ public void testGetFactory() throws IOException {
+ // no default interpreter because there's no interpreter setting binded to this note
+ assertNull(interpreterFactory.getInterpreter("user1", "note1", ""));
+
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds());
+ assertTrue(interpreterFactory.getInterpreter("user1", "note1", "") instanceof RemoteInterpreter);
+ RemoteInterpreter remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("user1", "note1", "");
+ // EchoInterpreter is the default interpreter (see zeppelin-interpreter/src/test/resources/conf/interpreter.json)
+ assertEquals(EchoInterpreter.class.getName(), remoteInterpreter.getClassName());
+
+ assertTrue(interpreterFactory.getInterpreter("user1", "note1", "test") instanceof RemoteInterpreter);
+ remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("user1", "note1", "test");
+ assertEquals(EchoInterpreter.class.getName(), remoteInterpreter.getClassName());
+
+ assertTrue(interpreterFactory.getInterpreter("user1", "note1", "echo") instanceof RemoteInterpreter);
+ remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("user1", "note1", "echo");
+ assertEquals(EchoInterpreter.class.getName(), remoteInterpreter.getClassName());
+
+ assertTrue(interpreterFactory.getInterpreter("user1", "note1", "double_echo") instanceof RemoteInterpreter);
+ remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("user1", "note1", "double_echo");
+ assertEquals(DoubleEchoInterpreter.class.getName(), remoteInterpreter.getClassName());
+ }
+
+ @Test(expected = InterpreterException.class)
+ public void testUnknownRepl1() throws IOException {
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds());
+ interpreterFactory.getInterpreter("user1", "note1", "test.unknown_repl");
+ }
+
+ @Test
+ public void testUnknownRepl2() throws IOException {
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds());
+ assertNull(interpreterFactory.getInterpreter("user1", "note1", "unknown_repl"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterGroupTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterGroupTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterGroupTest.java
new file mode 100644
index 0000000..11607bb
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterGroupTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonatype.aether.RepositoryException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+
+public class InterpreterGroupTest {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterGroupTest.class);
+
+ private InterpreterSetting interpreterSetting;
+
+ @Before
+ public void setUp() throws IOException, RepositoryException {
+ InterpreterOption interpreterOption = new InterpreterOption();
+ interpreterOption.setPerUser(InterpreterOption.SCOPED);
+ interpreterOption.setRemote(false);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+ }
+
+ @Test
+ public void testInterpreterGroup() {
+ InterpreterGroup interpreterGroup = new InterpreterGroup("group_1", interpreterSetting);
+ assertEquals(0, interpreterGroup.getSessionNum());
+
+ // create session_1
+ List<Interpreter> interpreters = interpreterGroup.getOrCreateSession("user1", "session_1");
+ assertEquals(2, interpreters.size());
+ assertEquals(EchoInterpreter.class.getName(), interpreters.get(0).getClassName());
+ assertEquals(DoubleEchoInterpreter.class.getName(), interpreters.get(1).getClassName());
+ assertEquals(1, interpreterGroup.getSessionNum());
+
+ // get the same interpreters when interpreterGroup.getOrCreateSession is invoked again
+ assertEquals(interpreters, interpreterGroup.getOrCreateSession("user1", "session_1"));
+ assertEquals(1, interpreterGroup.getSessionNum());
+
+ // create session_2
+ List<Interpreter> interpreters2 = interpreterGroup.getOrCreateSession("user1", "session_2");
+ assertEquals(2, interpreters2.size());
+ assertEquals(EchoInterpreter.class.getName(), interpreters2.get(0).getClassName());
+ assertEquals(DoubleEchoInterpreter.class.getName(), interpreters2.get(1).getClassName());
+ assertEquals(2, interpreterGroup.getSessionNum());
+
+ // close session_1
+ interpreterGroup.close("session_1");
+ assertEquals(1, interpreterGroup.getSessionNum());
+
+ // close InterpreterGroup
+ interpreterGroup.close();
+ assertEquals(0, interpreterGroup.getSessionNum());
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcherTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcherTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcherTest.java
index e376809..f3a30fb 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcherTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcherTest.java
@@ -21,6 +21,7 @@ import static org.junit.Assert.*;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
import org.junit.After;
import org.junit.Before;
@@ -29,7 +30,7 @@ import org.junit.Test;
public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChangeListener {
private File tmpDir;
private File fileChanged;
- private int numChanged;
+ private AtomicInteger numChanged;
private InterpreterOutputChangeWatcher watcher;
@Before
@@ -40,7 +41,7 @@ public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChan
tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
tmpDir.mkdirs();
fileChanged = null;
- numChanged = 0;
+ numChanged = new AtomicInteger(0);
}
@After
@@ -66,7 +67,7 @@ public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChan
@Test
public void test() throws IOException, InterruptedException {
assertNull(fileChanged);
- assertEquals(0, numChanged);
+ assertEquals(0, numChanged.get());
Thread.sleep(1000);
// create new file
@@ -92,14 +93,14 @@ public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChan
}
assertNotNull(fileChanged);
- assertEquals(1, numChanged);
+ assertEquals(1, numChanged.get());
}
@Override
public void fileChanged(File file) {
fileChanged = file;
- numChanged++;
+ numChanged.incrementAndGet();
synchronized(this) {
notify();
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java
new file mode 100644
index 0000000..c74760f
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.zeppelin.interpreter;
+
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.dep.Dependency;
+import org.apache.zeppelin.display.AngularObjectRegistryListener;
+import org.apache.zeppelin.helium.ApplicationEventListener;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
+import org.junit.Test;
+import org.sonatype.aether.RepositoryException;
+import org.sonatype.aether.repository.RemoteRepository;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+
+
+public class InterpreterSettingManagerTest extends AbstractInterpreterTest {
+
+ @Test
+ public void testInitInterpreterSettingManager() throws IOException, RepositoryException {
+ assertEquals(2, interpreterSettingManager.get().size());
+ InterpreterSetting interpreterSetting = interpreterSettingManager.getByName("test");
+ assertEquals("test", interpreterSetting.getName());
+ assertEquals("test", interpreterSetting.getGroup());
+ assertEquals(2, interpreterSetting.getInterpreterInfos().size());
+ // 3 other builtin properties:
+ // * zeppelin.interpeter.output.limit
+ // * zeppelin.interpreter.localRepo
+ // * zeppelin.interpreter.max.poolsize
+ assertEquals(6, interpreterSetting.getJavaProperties().size());
+ assertEquals("value_1", interpreterSetting.getJavaProperties().getProperty("property_1"));
+ assertEquals("new_value_2", interpreterSetting.getJavaProperties().getProperty("property_2"));
+ assertEquals("value_3", interpreterSetting.getJavaProperties().getProperty("property_3"));
+ assertEquals("shared", interpreterSetting.getOption().perNote);
+ assertEquals("shared", interpreterSetting.getOption().perUser);
+ assertEquals(0, interpreterSetting.getDependencies().size());
+ assertNotNull(interpreterSetting.getAngularObjectRegistryListener());
+ assertNotNull(interpreterSetting.getRemoteInterpreterProcessListener());
+ assertNotNull(interpreterSetting.getAppEventListener());
+ assertNotNull(interpreterSetting.getDependencyResolver());
+ assertNotNull(interpreterSetting.getInterpreterSettingManager());
+
+ List<RemoteRepository> repositories = interpreterSettingManager.getRepositories();
+ assertEquals(2, repositories.size());
+ assertEquals("central", repositories.get(0).getId());
+
+ // Load it again
+ InterpreterSettingManager interpreterSettingManager2 = new InterpreterSettingManager(conf,
+ mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
+ assertEquals(2, interpreterSettingManager2.get().size());
+ interpreterSetting = interpreterSettingManager2.getByName("test");
+ assertEquals("test", interpreterSetting.getName());
+ assertEquals("test", interpreterSetting.getGroup());
+ assertEquals(2, interpreterSetting.getInterpreterInfos().size());
+ assertEquals(6, interpreterSetting.getJavaProperties().size());
+ assertEquals("value_1", interpreterSetting.getJavaProperties().getProperty("property_1"));
+ assertEquals("new_value_2", interpreterSetting.getJavaProperties().getProperty("property_2"));
+ assertEquals("value_3", interpreterSetting.getJavaProperties().getProperty("property_3"));
+ assertEquals("shared", interpreterSetting.getOption().perNote);
+ assertEquals("shared", interpreterSetting.getOption().perUser);
+ assertEquals(0, interpreterSetting.getDependencies().size());
+
+ repositories = interpreterSettingManager2.getRepositories();
+ assertEquals(2, repositories.size());
+ assertEquals("central", repositories.get(0).getId());
+
+ }
+
+ @Test
+ public void testCreateUpdateRemoveSetting() throws IOException {
+ // create new interpreter setting
+ InterpreterOption option = new InterpreterOption();
+ option.setPerNote("scoped");
+ option.setPerUser("scoped");
+ Map<String, InterpreterProperty> properties = new HashMap<>();
+ properties.put("property_4", new InterpreterProperty("property_4","value_4"));
+
+ try {
+ interpreterSettingManager.createNewSetting("test2", "test", new ArrayList<Dependency>(), option, properties);
+ fail("Should fail due to interpreter already existed");
+ } catch (IOException e) {
+ assertTrue(e.getMessage().contains("already existed"));
+ }
+
+ interpreterSettingManager.createNewSetting("test3", "test", new ArrayList<Dependency>(), option, properties);
+ assertEquals(3, interpreterSettingManager.get().size());
+ InterpreterSetting interpreterSetting = interpreterSettingManager.getByName("test3");
+ assertEquals("test3", interpreterSetting.getName());
+ assertEquals("test", interpreterSetting.getGroup());
+ // 3 other builtin properties:
+ // * zeppelin.interpeter.output.limit
+ // * zeppelin.interpreter.localRepo
+ // * zeppelin.interpreter.max.poolsize
+ assertEquals(4, interpreterSetting.getJavaProperties().size());
+ assertEquals("value_4", interpreterSetting.getJavaProperties().getProperty("property_4"));
+ assertEquals("scoped", interpreterSetting.getOption().perNote);
+ assertEquals("scoped", interpreterSetting.getOption().perUser);
+ assertEquals(0, interpreterSetting.getDependencies().size());
+ assertNotNull(interpreterSetting.getAngularObjectRegistryListener());
+ assertNotNull(interpreterSetting.getRemoteInterpreterProcessListener());
+ assertNotNull(interpreterSetting.getAppEventListener());
+ assertNotNull(interpreterSetting.getDependencyResolver());
+ assertNotNull(interpreterSetting.getInterpreterSettingManager());
+
+ // load it again, it should be saved in interpreter-setting.json. So we can restore it properly
+ InterpreterSettingManager interpreterSettingManager2 = new InterpreterSettingManager(conf,
+ mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
+ assertEquals(3, interpreterSettingManager2.get().size());
+ interpreterSetting = interpreterSettingManager2.getByName("test3");
+ assertEquals("test3", interpreterSetting.getName());
+ assertEquals("test", interpreterSetting.getGroup());
+ assertEquals(6, interpreterSetting.getJavaProperties().size());
+ assertEquals("value_4", interpreterSetting.getJavaProperties().getProperty("property_4"));
+ assertEquals("scoped", interpreterSetting.getOption().perNote);
+ assertEquals("scoped", interpreterSetting.getOption().perUser);
+ assertEquals(0, interpreterSetting.getDependencies().size());
+
+ // update interpreter setting
+ InterpreterOption newOption = new InterpreterOption();
+ newOption.setPerNote("scoped");
+ newOption.setPerUser("isolated");
+ Map<String, InterpreterProperty> newProperties = new HashMap<>(properties);
+ newProperties.put("property_4", new InterpreterProperty("property_4", "new_value_4"));
+ List<Dependency> newDependencies = new ArrayList<>();
+ newDependencies.add(new Dependency("com.databricks:spark-avro_2.11:3.1.0"));
+ interpreterSettingManager.setPropertyAndRestart(interpreterSetting.getId(), newOption, newProperties, newDependencies);
+ interpreterSetting = interpreterSettingManager.get(interpreterSetting.getId());
+ assertEquals("test3", interpreterSetting.getName());
+ assertEquals("test", interpreterSetting.getGroup());
+ assertEquals(4, interpreterSetting.getJavaProperties().size());
+ assertEquals("new_value_4", interpreterSetting.getJavaProperties().getProperty("property_4"));
+ assertEquals("scoped", interpreterSetting.getOption().perNote);
+ assertEquals("isolated", interpreterSetting.getOption().perUser);
+ assertEquals(1, interpreterSetting.getDependencies().size());
+ assertNotNull(interpreterSetting.getAngularObjectRegistryListener());
+ assertNotNull(interpreterSetting.getRemoteInterpreterProcessListener());
+ assertNotNull(interpreterSetting.getAppEventListener());
+ assertNotNull(interpreterSetting.getDependencyResolver());
+ assertNotNull(interpreterSetting.getInterpreterSettingManager());
+
+ // restart in note page
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds());
+ interpreterSettingManager.setInterpreterBinding("user2", "note2", interpreterSettingManager.getSettingIds());
+ interpreterSettingManager.setInterpreterBinding("user3", "note3", interpreterSettingManager.getSettingIds());
+ // create 3 sessions as it is scoped mode
+ interpreterSetting.getOption().setPerUser("scoped");
+ interpreterSetting.getDefaultInterpreter("user1", "note1");
+ interpreterSetting.getDefaultInterpreter("user2", "note2");
+ interpreterSetting.getDefaultInterpreter("user3", "note3");
+ InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
+ assertEquals(3, interpreterGroup.getSessionNum());
+ // only close user1's session
+ interpreterSettingManager.restart(interpreterSetting.getId(), "note1", "user1");
+ assertEquals(2, interpreterGroup.getSessionNum());
+ // close all the sessions
+ interpreterSettingManager.restart(interpreterSetting.getId(), "note1", "anonymous");
+ assertEquals(0, interpreterGroup.getSessionNum());
+
+ // remove interpreter setting
+ interpreterSettingManager.remove(interpreterSetting.getId());
+ assertEquals(2, interpreterSettingManager.get().size());
+
+ // load it again
+ InterpreterSettingManager interpreterSettingManager3 = new InterpreterSettingManager(new ZeppelinConfiguration(),
+ mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
+ assertEquals(2, interpreterSettingManager3.get().size());
+
+ }
+
+ @Test
+ public void testInterpreterBinding() throws IOException {
+ assertNull(interpreterSettingManager.getInterpreterBinding("note1"));
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds());
+ assertEquals(interpreterSettingManager.getInterpreterSettingIds(), interpreterSettingManager.getInterpreterBinding("note1"));
+ }
+
+ @Test
+ public void testUpdateInterpreterBinding_PerNoteShared() throws IOException {
+ InterpreterSetting defaultInterpreterSetting = interpreterSettingManager.get().get(0);
+ defaultInterpreterSetting.getOption().setPerNote("shared");
+
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds());
+ // create interpreter of the first binded interpreter setting
+ interpreterFactory.getInterpreter("user1", "note1", "");
+ assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
+
+ // choose the first setting
+ List<String> newSettingIds = new ArrayList<>();
+ newSettingIds.add(interpreterSettingManager.getInterpreterSettingIds().get(1));
+
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", newSettingIds);
+ assertEquals(newSettingIds, interpreterSettingManager.getInterpreterBinding("note1"));
+ // InterpreterGroup will still be alive as it is shared
+ assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
+ }
+
+ @Test
+ public void testUpdateInterpreterBinding_PerNoteIsolated() throws IOException {
+ InterpreterSetting defaultInterpreterSetting = interpreterSettingManager.get().get(0);
+ defaultInterpreterSetting.getOption().setPerNote("isolated");
+
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds());
+ // create interpreter of the first binded interpreter setting
+ interpreterFactory.getInterpreter("user1", "note1", "");
+ assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
+
+ // choose the first setting
+ List<String> newSettingIds = new ArrayList<>();
+ newSettingIds.add(interpreterSettingManager.getInterpreterSettingIds().get(1));
+
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", newSettingIds);
+ assertEquals(newSettingIds, interpreterSettingManager.getInterpreterBinding("note1"));
+ // InterpreterGroup will be closed as it is only belong to this note
+ assertEquals(0, defaultInterpreterSetting.getAllInterpreterGroups().size());
+
+ }
+
+ @Test
+ public void testUpdateInterpreterBinding_PerNoteScoped() throws IOException {
+ InterpreterSetting defaultInterpreterSetting = interpreterSettingManager.get().get(0);
+ defaultInterpreterSetting.getOption().setPerNote("scoped");
+
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds());
+ interpreterSettingManager.setInterpreterBinding("user1", "note2", interpreterSettingManager.getInterpreterSettingIds());
+ // create 2 interpreter of the first binded interpreter setting for note1 and note2
+ interpreterFactory.getInterpreter("user1", "note1", "");
+ interpreterFactory.getInterpreter("user1", "note2", "");
+ assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(2, defaultInterpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // choose the first setting
+ List<String> newSettingIds = new ArrayList<>();
+ newSettingIds.add(interpreterSettingManager.getInterpreterSettingIds().get(1));
+
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", newSettingIds);
+ assertEquals(newSettingIds, interpreterSettingManager.getInterpreterBinding("note1"));
+ // InterpreterGroup will be still alive but session belong to note1 will be closed
+ assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java
new file mode 100644
index 0000000..3c061a9
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java
@@ -0,0 +1,411 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class InterpreterSettingTest {
+
+ @Test
+ public void testCreateInterpreters() {
+ InterpreterOption interpreterOption = new InterpreterOption();
+ interpreterOption.setPerUser(InterpreterOption.SHARED);
+ interpreterOption.setRemote(false);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+
+ // create default interpreter for user1 and note1
+ assertEquals(EchoInterpreter.class.getName(), interpreterSetting.getDefaultInterpreter("user1", "note1").getClassName());
+
+ // create interpreter echo for user1 and note1
+ assertEquals(EchoInterpreter.class.getName(), interpreterSetting.getInterpreter("user1", "note1", "echo").getClassName());
+ assertEquals(interpreterSetting.getDefaultInterpreter("user1", "note1"), interpreterSetting.getInterpreter("user1", "note1", "echo"));
+
+ // create interpreter double_echo for user1 and note1
+ assertEquals(DoubleEchoInterpreter.class.getName(), interpreterSetting.getInterpreter("user1", "note1", "double_echo").getClassName());
+
+ // create non-existed interpreter
+ assertNull(interpreterSetting.getInterpreter("user1", "note1", "invalid_echo"));
+ }
+
+ @Test
+ public void testSharedMode() {
+ InterpreterOption interpreterOption = new InterpreterOption();
+ interpreterOption.setPerUser(InterpreterOption.SHARED);
+ interpreterOption.setRemote(false);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+
+ // create default interpreter for user1 and note1
+ interpreterSetting.getDefaultInterpreter("user1", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+
+ // create default interpreter for user2 and note1
+ interpreterSetting.getDefaultInterpreter("user2", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+
+ // create default interpreter user1 and note2
+ interpreterSetting.getDefaultInterpreter("user1", "note2");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+
+ // only 1 session is created, this session is shared across users and notes
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ interpreterSetting.closeInterpreters("note1", "user1");
+ assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
+ }
+
+ @Test
+ public void testPerUserScopedMode() {
+ InterpreterOption interpreterOption = new InterpreterOption();
+ interpreterOption.setPerUser(InterpreterOption.SCOPED);
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+
+ // create interpreter for user1 and note1
+ interpreterSetting.getDefaultInterpreter("user1", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // create interpreter for user2 and note1
+ interpreterSetting.getDefaultInterpreter("user2", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ interpreterSetting.closeInterpreters("user1", "note1");
+ // InterpreterGroup is still there, but one session is removed
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ interpreterSetting.closeInterpreters("user2", "note1");
+ assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
+ }
+
+ @Test
+ public void testPerNoteScopedMode() {
+ InterpreterOption interpreterOption = new InterpreterOption();
+ interpreterOption.setPerNote(InterpreterOption.SCOPED);
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+
+ // create interpreter for user1 and note1
+ interpreterSetting.getDefaultInterpreter("user1", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // create interpreter for user1 and note2
+ interpreterSetting.getDefaultInterpreter("user1", "note2");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ interpreterSetting.closeInterpreters("user1", "note1");
+ // InterpreterGroup is still there, but one session is removed
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ interpreterSetting.closeInterpreters("user1", "note2");
+ assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
+ }
+
+ @Test
+ public void testPerUserIsolatedMode() {
+ InterpreterOption interpreterOption = new InterpreterOption();
+ interpreterOption.setPerUser(InterpreterOption.ISOLATED);
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+
+ // create interpreter for user1 and note1
+ interpreterSetting.getDefaultInterpreter("user1", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // create interpreter for user2 and note1
+ interpreterSetting.getDefaultInterpreter("user2", "note1");
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
+
+ // Each user own one InterpreterGroup and one session per InterpreterGroup
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(1).getSessionNum());
+
+ interpreterSetting.closeInterpreters("user1", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ interpreterSetting.closeInterpreters("user2", "note1");
+ assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
+ }
+
+ @Test
+ public void testPerNoteIsolatedMode() {
+ InterpreterOption interpreterOption = new InterpreterOption();
+ interpreterOption.setPerNote(InterpreterOption.ISOLATED);
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+
+ // create interpreter for user1 and note1
+ interpreterSetting.getDefaultInterpreter("user1", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // create interpreter for user2 and note2
+ interpreterSetting.getDefaultInterpreter("user1", "note2");
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
+ // Each user own one InterpreterGroup and one session per InterpreterGroup
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(1).getSessionNum());
+
+ interpreterSetting.closeInterpreters("user1", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ interpreterSetting.closeInterpreters("user1", "note2");
+ assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
+ }
+
+ @Test
+ public void testPerUserIsolatedPerNoteScopedMode() {
+ InterpreterOption interpreterOption = new InterpreterOption();
+ interpreterOption.setPerUser(InterpreterOption.ISOLATED);
+ interpreterOption.setPerNote(InterpreterOption.SCOPED);
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+
+ // create interpreter for user1 and note1
+ interpreterSetting.getDefaultInterpreter("user1", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ interpreterSetting.getDefaultInterpreter("user1", "note2");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // create interpreter for user2 and note1
+ interpreterSetting.getDefaultInterpreter("user2", "note1");
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
+
+ // group1 for user1 has 2 sessions, and group2 for user2 has 1 session
+ assertEquals(interpreterSetting.getInterpreterGroup("user1", "note1"), interpreterSetting.getInterpreterGroup("user1", "note2"));
+ assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note1").getSessionNum());
+ assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note2").getSessionNum());
+ assertEquals(1, interpreterSetting.getInterpreterGroup("user2", "note1").getSessionNum());
+
+ // close one session for user1
+ interpreterSetting.closeInterpreters("user1", "note1");
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").getSessionNum());
+
+ // close another session for user1
+ interpreterSetting.closeInterpreters("user1", "note2");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+
+ // close session for user2
+ interpreterSetting.closeInterpreters("user2", "note1");
+ assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
+ }
+
+ @Test
+ public void testPerUserIsolatedPerNoteIsolatedMode() {
+ InterpreterOption interpreterOption = new InterpreterOption();
+ interpreterOption.setPerUser(InterpreterOption.ISOLATED);
+ interpreterOption.setPerNote(InterpreterOption.ISOLATED);
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+
+ // create interpreter for user1 and note1
+ interpreterSetting.getDefaultInterpreter("user1", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+
+ // create interpreter for user1 and note2
+ interpreterSetting.getDefaultInterpreter("user1", "note2");
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
+
+ // create interpreter for user2 and note1
+ interpreterSetting.getDefaultInterpreter("user2", "note1");
+ assertEquals(3, interpreterSetting.getAllInterpreterGroups().size());
+
+ // create interpreter for user2 and note2
+ interpreterSetting.getDefaultInterpreter("user2", "note2");
+ assertEquals(4, interpreterSetting.getAllInterpreterGroups().size());
+
+ for (InterpreterGroup interpreterGroup : interpreterSetting.getAllInterpreterGroups()) {
+ // each InterpreterGroup has one session
+ assertEquals(1, interpreterGroup.getSessionNum());
+ }
+
+ // close one session for user1 and note1
+ interpreterSetting.closeInterpreters("user1", "note1");
+ assertEquals(3, interpreterSetting.getAllInterpreterGroups().size());
+
+ // close one session for user1 and note2
+ interpreterSetting.closeInterpreters("user1", "note2");
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
+
+ // close one session for user2 and note1
+ interpreterSetting.closeInterpreters("user2", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+
+ // close one session for user2 and note2
+ interpreterSetting.closeInterpreters("user2", "note2");
+ assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
+ }
+
+ @Test
+ public void testPerUserScopedPerNoteScopedMode() {
+ InterpreterOption interpreterOption = new InterpreterOption();
+ interpreterOption.setPerUser(InterpreterOption.SCOPED);
+ interpreterOption.setPerNote(InterpreterOption.SCOPED);
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+
+ // create interpreter for user1 and note1
+ interpreterSetting.getDefaultInterpreter("user1", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // create interpreter for user1 and note2
+ interpreterSetting.getDefaultInterpreter("user1", "note2");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // create interpreter for user2 and note1
+ interpreterSetting.getDefaultInterpreter("user2", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(3, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // create interpreter for user2 and note2
+ interpreterSetting.getDefaultInterpreter("user2", "note2");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(4, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // close one session for user1 and note1
+ interpreterSetting.closeInterpreters("user1", "note1");
+ assertEquals(3, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // close one session for user1 and note2
+ interpreterSetting.closeInterpreters("user1", "note2");
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // close one session for user2 and note1
+ interpreterSetting.closeInterpreters("user2", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // close one session for user2 and note2
+ interpreterSetting.closeInterpreters("user2", "note2");
+ assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java
index 305268c..d46eaa7 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java
@@ -24,13 +24,14 @@ import org.junit.Test;
import static org.junit.Assert.assertEquals;
+//TODO(zjffdu) add more test for Interpreter which is a very important class
public class InterpreterTest {
@Test
public void testDefaultProperty() {
Properties p = new Properties();
p.put("p1", "v1");
- Interpreter intp = new DummyInterpreter(p);
+ Interpreter intp = new EchoInterpreter(p);
assertEquals(1, intp.getProperty().size());
assertEquals("v1", intp.getProperty().get("p1"));
@@ -41,7 +42,7 @@ public class InterpreterTest {
public void testOverriddenProperty() {
Properties p = new Properties();
p.put("p1", "v1");
- Interpreter intp = new DummyInterpreter(p);
+ Interpreter intp = new EchoInterpreter(p);
Properties overriddenProperty = new Properties();
overriddenProperty.put("p1", "v2");
intp.setProperty(overriddenProperty);
@@ -73,7 +74,7 @@ public class InterpreterTest {
Properties p = new Properties();
p.put("p1", "replName #{noteId}, #{paragraphTitle}, #{paragraphId}, #{paragraphText}, #{replName}, #{noteId}, #{user}," +
" #{authenticationInfo}");
- Interpreter intp = new DummyInterpreter(p);
+ Interpreter intp = new EchoInterpreter(p);
intp.setUserName(user);
String actual = intp.getProperty("p1");
InterpreterContext.remove();
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java
new file mode 100644
index 0000000..9deafcf
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java
@@ -0,0 +1,60 @@
+package org.apache.zeppelin.interpreter;
+
+import org.apache.zeppelin.scheduler.Scheduler;
+import org.apache.zeppelin.scheduler.SchedulerFactory;
+
+import java.util.Properties;
+
+/**
+ * Interpreter that only accept long value and sleep for such period
+ */
+public class SleepInterpreter extends Interpreter {
+
+ public SleepInterpreter(Properties property) {
+ super(property);
+ }
+
+ @Override
+ public void open() {
+
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ try {
+ Thread.sleep(Long.parseLong(st));
+ return new InterpreterResult(InterpreterResult.Code.SUCCESS);
+ } catch (Exception e) {
+ return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
+ }
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+
+ }
+
+ @Override
+ public FormType getFormType() {
+ return FormType.NATIVE;
+ }
+
+ @Override
+ public Scheduler getScheduler() {
+ if (Boolean.parseBoolean(property.getProperty("zeppelin.SleepInterpreter.parallel", "false"))) {
+ return SchedulerFactory.singleton().createOrGetParallelScheduler(
+ "Parallel-" + SleepInterpreter.class.getName(), 10);
+ }
+ return super.getScheduler();
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java
new file mode 100644
index 0000000..e934f1a
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java
@@ -0,0 +1,86 @@
+package org.apache.zeppelin.interpreter.install;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class InstallInterpreterTest {
+ private File tmpDir;
+ private InstallInterpreter installer;
+ private File interpreterBaseDir;
+
+ @Before
+ public void setUp() throws IOException {
+ tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
+ new File(tmpDir, "conf").mkdirs();
+ interpreterBaseDir = new File(tmpDir, "interpreter");
+ File localRepoDir = new File(tmpDir, "local-repo");
+ interpreterBaseDir.mkdir();
+ localRepoDir.mkdir();
+
+ File interpreterListFile = new File(tmpDir, "conf/interpreter-list");
+
+
+ // create interpreter list file
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
+
+ String interpreterList = "";
+ interpreterList += "intp1 org.apache.commons:commons-csv:1.1 test interpreter 1\n";
+ interpreterList += "intp2 org.apache.commons:commons-math3:3.6.1 test interpreter 2\n";
+
+ FileUtils.writeStringToFile(new File(tmpDir, "conf/interpreter-list"), interpreterList);
+
+ installer = new InstallInterpreter(interpreterListFile, interpreterBaseDir, localRepoDir
+ .getAbsolutePath());
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ FileUtils.deleteDirectory(tmpDir);
+ }
+
+
+ @Test
+ public void testList() {
+ assertEquals(2, installer.list().size());
+ }
+
+ @Test
+ public void install() {
+ assertEquals(0, interpreterBaseDir.listFiles().length);
+
+ installer.install("intp1");
+ assertTrue(new File(interpreterBaseDir, "intp1").isDirectory());
+ }
+
+ @Test
+ public void installAll() {
+ installer.installAll();
+ assertTrue(new File(interpreterBaseDir, "intp1").isDirectory());
+ assertTrue(new File(interpreterBaseDir, "intp2").isDirectory());
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
new file mode 100644
index 0000000..a533c12
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
@@ -0,0 +1,106 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.zeppelin.interpreter.mock;
+
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.apache.zeppelin.scheduler.Scheduler;
+import org.apache.zeppelin.scheduler.SchedulerFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class MockInterpreter1 extends Interpreter {
+
+ Map<String, Object> vars = new HashMap<>();
+
+ public MockInterpreter1(Properties property) {
+ super(property);
+ }
+ boolean open;
+
+
+ @Override
+ public void open() {
+ open = true;
+ }
+
+ @Override
+ public void close() {
+ open = false;
+ }
+
+
+ public boolean isOpen() {
+ return open;
+ }
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ InterpreterResult result;
+
+ if ("getId".equals(st)) {
+ // get unique id of this interpreter instance
+ result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "" + this.hashCode());
+ } else if (st.startsWith("sleep")) {
+ try {
+ Thread.sleep(Integer.parseInt(st.split(" ")[1]));
+ } catch (InterruptedException e) {
+ // nothing to do
+ }
+ result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl1: " + st);
+ } else {
+ result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl1: " + st);
+ }
+
+ if (context.getResourcePool() != null) {
+ context.getResourcePool().put(context.getNoteId(), context.getParagraphId(), "result", result);
+ }
+
+ return result;
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+ }
+
+ @Override
+ public FormType getFormType() {
+ return FormType.SIMPLE;
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+
+ @Override
+ public Scheduler getScheduler() {
+ return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_"+this.hashCode());
+ }
+
+ @Override
+ public List<InterpreterCompletion> completion(String buf, int cursor,
+ InterpreterContext interpreterContext) {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter11.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter11.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter11.java
new file mode 100644
index 0000000..d53716f
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter11.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.mock;
+
+
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.apache.zeppelin.scheduler.Scheduler;
+import org.apache.zeppelin.scheduler.SchedulerFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class MockInterpreter11 extends Interpreter {
+ Map<String, Object> vars = new HashMap<>();
+
+ public MockInterpreter11(Properties property) {
+ super(property);
+ }
+
+ boolean open;
+
+ @Override
+ public void open() {
+ open = true;
+ }
+
+ @Override
+ public void close() {
+ open = false;
+ }
+
+ public boolean isOpen() {
+ return open;
+ }
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ return new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl11: " + st);
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+ }
+
+ @Override
+ public FormType getFormType() {
+ return FormType.SIMPLE;
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+
+ @Override
+ public Scheduler getScheduler() {
+ return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_" + this.hashCode());
+ }
+
+ @Override
+ public List<InterpreterCompletion> completion(String buf, int cursor,
+ InterpreterContext interpreterContext) {
+ return null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java
new file mode 100644
index 0000000..f36df56
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.mock;
+
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.apache.zeppelin.scheduler.Scheduler;
+import org.apache.zeppelin.scheduler.SchedulerFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class MockInterpreter2 extends Interpreter{
+ Map<String, Object> vars = new HashMap<>();
+
+ public MockInterpreter2(Properties property) {
+ super(property);
+ }
+
+ boolean open;
+
+ @Override
+ public void open() {
+ open = true;
+ }
+
+ @Override
+ public void close() {
+ open = false;
+ }
+
+ public boolean isOpen() {
+ return open;
+ }
+
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ InterpreterResult result;
+
+ if ("getId".equals(st)) {
+ // get unique id of this interpreter instance
+ result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "" + this.hashCode());
+ } else if (st.startsWith("sleep")) {
+ try {
+ Thread.sleep(Integer.parseInt(st.split(" ")[1]));
+ } catch (InterruptedException e) {
+ // nothing to do
+ }
+ result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl2: " + st);
+ } else {
+ result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl2: " + st);
+ }
+
+ if (context.getResourcePool() != null) {
+ context.getResourcePool().put(context.getNoteId(), context.getParagraphId(), "result", result);
+ }
+ return result;
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+ }
+
+ @Override
+ public FormType getFormType() {
+ return FormType.SIMPLE;
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+
+ @Override
+ public Scheduler getScheduler() {
+ return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_"+this.hashCode());
+ }
+
+ @Override
+ public List<InterpreterCompletion> completion(String buf, int cursor,
+ InterpreterContext interpreterContext) {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java
new file mode 100644
index 0000000..c9dc5c0
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote;
+
+import org.apache.log4j.AppenderSkeleton;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
+import org.junit.After;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Mockito.*;
+
+public class AppendOutputRunnerTest {
+
+ private static final int NUM_EVENTS = 10000;
+ private static final int NUM_CLUBBED_EVENTS = 100;
+ private static final ScheduledExecutorService service = Executors.newSingleThreadScheduledExecutor();
+ private static ScheduledFuture<?> future = null;
+ /* It is being accessed by multiple threads.
+ * While loop for 'loopForBufferCompletion' could
+ * run for-ever.
+ */
+ private volatile static int numInvocations = 0;
+
+ @After
+ public void afterEach() {
+ if (future != null) {
+ future.cancel(true);
+ }
+ }
+
+ @Test
+ public void testSingleEvent() throws InterruptedException {
+ RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
+ String[][] buffer = {{"note", "para", "data\n"}};
+
+ loopForCompletingEvents(listener, 1, buffer);
+ verify(listener, times(1)).onOutputAppend(any(String.class), any(String.class), anyInt(), any(String.class));
+ verify(listener, times(1)).onOutputAppend("note", "para", 0, "data\n");
+ }
+
+ @Test
+ public void testMultipleEventsOfSameParagraph() throws InterruptedException {
+ RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
+ String note1 = "note1";
+ String para1 = "para1";
+ String[][] buffer = {
+ {note1, para1, "data1\n"},
+ {note1, para1, "data2\n"},
+ {note1, para1, "data3\n"}
+ };
+
+ loopForCompletingEvents(listener, 1, buffer);
+ verify(listener, times(1)).onOutputAppend(any(String.class), any(String.class), anyInt(), any(String.class));
+ verify(listener, times(1)).onOutputAppend(note1, para1, 0, "data1\ndata2\ndata3\n");
+ }
+
+ @Test
+ public void testMultipleEventsOfDifferentParagraphs() throws InterruptedException {
+ RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
+ String note1 = "note1";
+ String note2 = "note2";
+ String para1 = "para1";
+ String para2 = "para2";
+ String[][] buffer = {
+ {note1, para1, "data1\n"},
+ {note1, para2, "data2\n"},
+ {note2, para1, "data3\n"},
+ {note2, para2, "data4\n"}
+ };
+ loopForCompletingEvents(listener, 4, buffer);
+
+ verify(listener, times(4)).onOutputAppend(any(String.class), any(String.class), anyInt(), any(String.class));
+ verify(listener, times(1)).onOutputAppend(note1, para1, 0, "data1\n");
+ verify(listener, times(1)).onOutputAppend(note1, para2, 0, "data2\n");
+ verify(listener, times(1)).onOutputAppend(note2, para1, 0, "data3\n");
+ verify(listener, times(1)).onOutputAppend(note2, para2, 0, "data4\n");
+ }
+
+ @Test
+ public void testClubbedData() throws InterruptedException {
+ RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
+ AppendOutputRunner runner = new AppendOutputRunner(listener);
+ future = service.scheduleWithFixedDelay(runner, 0,
+ AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
+ Thread thread = new Thread(new BombardEvents(runner));
+ thread.start();
+ thread.join();
+ Thread.sleep(1000);
+
+ /* NUM_CLUBBED_EVENTS is a heuristic number.
+ * It has been observed that for 10,000 continuos event
+ * calls, 30-40 Web-socket calls are made. Keeping
+ * the unit-test to a pessimistic 100 web-socket calls.
+ */
+ verify(listener, atMost(NUM_CLUBBED_EVENTS)).onOutputAppend(any(String.class), any(String.class), anyInt(), any(String.class));
+ }
+
+ @Test
+ public void testWarnLoggerForLargeData() throws InterruptedException {
+ RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
+ AppendOutputRunner runner = new AppendOutputRunner(listener);
+ String data = "data\n";
+ int numEvents = 100000;
+
+ for (int i=0; i<numEvents; i++) {
+ runner.appendBuffer("noteId", "paraId", 0, data);
+ }
+
+ TestAppender appender = new TestAppender();
+ Logger logger = Logger.getRootLogger();
+ logger.addAppender(appender);
+ Logger.getLogger(RemoteInterpreterEventPoller.class);
+
+ runner.run();
+ List<LoggingEvent> log;
+
+ int warnLogCounter;
+ LoggingEvent sizeWarnLogEntry = null;
+ do {
+ warnLogCounter = 0;
+ log = appender.getLog();
+ for (LoggingEvent logEntry: log) {
+ if (Level.WARN.equals(logEntry.getLevel())) {
+ sizeWarnLogEntry = logEntry;
+ warnLogCounter += 1;
+ }
+ }
+ } while(warnLogCounter != 2);
+
+ String loggerString = "Processing size for buffered append-output is high: " +
+ (data.length() * numEvents) + " characters.";
+ assertTrue(loggerString.equals(sizeWarnLogEntry.getMessage()));
+ }
+
+ private class BombardEvents implements Runnable {
+
+ private final AppendOutputRunner runner;
+
+ private BombardEvents(AppendOutputRunner runner) {
+ this.runner = runner;
+ }
+
+ @Override
+ public void run() {
+ String noteId = "noteId";
+ String paraId = "paraId";
+ for (int i=0; i<NUM_EVENTS; i++) {
+ runner.appendBuffer(noteId, paraId, 0, "data\n");
+ }
+ }
+ }
+
+ private class TestAppender extends AppenderSkeleton {
+ private final List<LoggingEvent> log = new ArrayList<>();
+
+ @Override
+ public boolean requiresLayout() {
+ return false;
+ }
+
+ @Override
+ protected void append(final LoggingEvent loggingEvent) {
+ log.add(loggingEvent);
+ }
+
+ @Override
+ public void close() {
+ }
+
+ public List<LoggingEvent> getLog() {
+ return new ArrayList<>(log);
+ }
+ }
+
+ private void prepareInvocationCounts(RemoteInterpreterProcessListener listener) {
+ doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ numInvocations += 1;
+ return null;
+ }
+ }).when(listener).onOutputAppend(any(String.class), any(String.class), anyInt(), any(String.class));
+ }
+
+ private void loopForCompletingEvents(RemoteInterpreterProcessListener listener,
+ int numTimes, String[][] buffer) {
+ numInvocations = 0;
+ prepareInvocationCounts(listener);
+ AppendOutputRunner runner = new AppendOutputRunner(listener);
+ for (String[] bufferElement: buffer) {
+ runner.appendBuffer(bufferElement[0], bufferElement[1], 0, bufferElement[2]);
+ }
+ future = service.scheduleWithFixedDelay(runner, 0,
+ AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
+ long startTimeMs = System.currentTimeMillis();
+ while(numInvocations != numTimes) {
+ if (System.currentTimeMillis() - startTimeMs > 2000) {
+ fail("Buffered events were not sent for 2 seconds");
+ }
+ }
+ }
+}
\ No newline at end of file
[02/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
deleted file mode 100644
index 95235e5..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
+++ /dev/null
@@ -1,975 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.thrift.transport.TTransportException;
-import org.apache.zeppelin.display.AngularObject;
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterEnv;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
-import org.apache.zeppelin.user.AuthenticationInfo;
-import org.apache.zeppelin.display.GUI;
-import org.apache.zeppelin.interpreter.*;
-import org.apache.zeppelin.interpreter.InterpreterResult.Code;
-import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterA;
-import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterB;
-import org.apache.zeppelin.resource.LocalResourcePool;
-import org.apache.zeppelin.scheduler.Job;
-import org.apache.zeppelin.scheduler.Job.Status;
-import org.apache.zeppelin.scheduler.Scheduler;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-
-public class RemoteInterpreterTest {
-
-
- private static final String INTERPRETER_SCRIPT =
- System.getProperty("os.name").startsWith("Windows") ?
- "../bin/interpreter.cmd" :
- "../bin/interpreter.sh";
-
- private InterpreterGroup intpGroup;
- private HashMap<String, String> env;
-
- @Before
- public void setUp() throws Exception {
- intpGroup = new InterpreterGroup();
- env = new HashMap<>();
- env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
- }
-
- @After
- public void tearDown() throws Exception {
- intpGroup.close();
- }
-
- private RemoteInterpreter createMockInterpreterA(Properties p) {
- return createMockInterpreterA(p, "note");
- }
-
- private RemoteInterpreter createMockInterpreterA(Properties p, String noteId) {
- return new RemoteInterpreter(
- p,
- noteId,
- MockInterpreterA.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- null,
- null,
- "anonymous",
- false);
- }
-
- private RemoteInterpreter createMockInterpreterB(Properties p) {
- return createMockInterpreterB(p, "note");
- }
-
- private RemoteInterpreter createMockInterpreterB(Properties p, String noteId) {
- return new RemoteInterpreter(
- p,
- noteId,
- MockInterpreterB.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- null,
- null,
- "anonymous",
- false);
- }
-
- @Test
- public void testRemoteInterperterCall() throws TTransportException, IOException {
- Properties p = new Properties();
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.get("note").add(intpA);
-
- intpA.setInterpreterGroup(intpGroup);
-
- RemoteInterpreter intpB = createMockInterpreterB(p);
-
- intpGroup.get("note").add(intpB);
- intpB.setInterpreterGroup(intpGroup);
-
-
- RemoteInterpreterProcess process = intpA.getInterpreterProcess();
- process.equals(intpB.getInterpreterProcess());
-
- assertFalse(process.isRunning());
- assertEquals(0, process.getNumIdleClient());
- assertEquals(0, process.referenceCount());
-
- intpA.open(); // initializa all interpreters in the same group
- assertTrue(process.isRunning());
- assertEquals(1, process.getNumIdleClient());
- assertEquals(1, process.referenceCount());
-
- intpA.interpret("1",
- new InterpreterContext(
- "note",
- "id",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
-
- intpB.open();
- assertEquals(1, process.referenceCount());
-
- intpA.close();
- assertEquals(0, process.referenceCount());
- intpB.close();
- assertEquals(0, process.referenceCount());
-
- assertFalse(process.isRunning());
-
- }
-
- @Test
- public void testExecuteIncorrectPrecode() throws TTransportException, IOException {
- Properties p = new Properties();
- p.put("zeppelin.MockInterpreterA.precode", "fail test");
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.get("note").add(intpA);
-
- intpA.setInterpreterGroup(intpGroup);
-
- RemoteInterpreterProcess process = intpA.getInterpreterProcess();
-
- intpA.open();
-
- InterpreterResult result = intpA.interpret("1",
- new InterpreterContext(
- "note",
- "id",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
-
-
-
- intpA.close();
- assertEquals(Code.ERROR, result.code());
- }
-
- @Test
- public void testExecuteCorrectPrecode() throws TTransportException, IOException {
- Properties p = new Properties();
- p.put("zeppelin.MockInterpreterA.precode", "2");
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.get("note").add(intpA);
-
- intpA.setInterpreterGroup(intpGroup);
-
- RemoteInterpreterProcess process = intpA.getInterpreterProcess();
-
- intpA.open();
-
- InterpreterResult result = intpA.interpret("1",
- new InterpreterContext(
- "note",
- "id",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
-
-
-
- intpA.close();
- assertEquals(Code.SUCCESS, result.code());
- assertEquals("1", result.message().get(0).getData());
- }
-
- @Test
- public void testRemoteInterperterErrorStatus() throws TTransportException, IOException {
- Properties p = new Properties();
-
- RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.put("note", new LinkedList<Interpreter>());
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
- intpA.open();
- InterpreterResult ret = intpA.interpret("non numeric value",
- new InterpreterContext(
- "noteId",
- "id",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
-
- assertEquals(Code.ERROR, ret.code());
- }
-
- @Test
- public void testRemoteSchedulerSharing() throws TTransportException, IOException {
- Properties p = new Properties();
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- RemoteInterpreter intpA = new RemoteInterpreter(
- p,
- "note",
- MockInterpreterA.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- null,
- null,
- "anonymous",
- false);
-
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
- RemoteInterpreter intpB = new RemoteInterpreter(
- p,
- "note",
- MockInterpreterB.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- null,
- null,
- "anonymous",
- false);
-
- intpGroup.get("note").add(intpB);
- intpB.setInterpreterGroup(intpGroup);
-
- intpA.open();
- intpB.open();
-
- long start = System.currentTimeMillis();
- InterpreterResult ret = intpA.interpret("500",
- new InterpreterContext(
- "note",
- "id",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
- assertEquals("500", ret.message().get(0).getData());
-
- ret = intpB.interpret("500",
- new InterpreterContext(
- "note",
- "id",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
- assertEquals("1000", ret.message().get(0).getData());
- long end = System.currentTimeMillis();
- assertTrue(end - start >= 1000);
-
-
- intpA.close();
- intpB.close();
- }
-
- @Test
- public void testRemoteSchedulerSharingSubmit() throws TTransportException, IOException, InterruptedException {
- Properties p = new Properties();
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- final RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
- final RemoteInterpreter intpB = createMockInterpreterB(p);
-
- intpGroup.get("note").add(intpB);
- intpB.setInterpreterGroup(intpGroup);
-
- intpA.open();
- intpB.open();
-
- long start = System.currentTimeMillis();
- Job jobA = new Job("jobA", null) {
- private Object r;
-
- @Override
- public Object getReturn() {
- return r;
- }
-
- @Override
- public void setResult(Object results) {
- this.r = results;
- }
-
- @Override
- public int progress() {
- return 0;
- }
-
- @Override
- public Map<String, Object> info() {
- return null;
- }
-
- @Override
- protected Object jobRun() throws Throwable {
- return intpA.interpret("500",
- new InterpreterContext(
- "note",
- "jobA",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
- }
-
- @Override
- protected boolean jobAbort() {
- return false;
- }
-
- };
- intpA.getScheduler().submit(jobA);
-
- Job jobB = new Job("jobB", null) {
-
- private Object r;
-
- @Override
- public Object getReturn() {
- return r;
- }
-
- @Override
- public void setResult(Object results) {
- this.r = results;
- }
-
- @Override
- public int progress() {
- return 0;
- }
-
- @Override
- public Map<String, Object> info() {
- return null;
- }
-
- @Override
- protected Object jobRun() throws Throwable {
- return intpB.interpret("500",
- new InterpreterContext(
- "note",
- "jobB",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
- }
-
- @Override
- protected boolean jobAbort() {
- return false;
- }
-
- };
- intpB.getScheduler().submit(jobB);
- // wait until both job finished
- while (jobA.getStatus() != Status.FINISHED ||
- jobB.getStatus() != Status.FINISHED) {
- Thread.sleep(100);
- }
- long end = System.currentTimeMillis();
- assertTrue(end - start >= 1000);
-
- assertEquals("1000", ((InterpreterResult) jobB.getReturn()).message().get(0).getData());
-
- intpA.close();
- intpB.close();
- }
-
- @Test
- public void testRunOrderPreserved() throws InterruptedException {
- Properties p = new Properties();
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- final RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
- intpA.open();
-
- int concurrency = 3;
- final List<InterpreterResultMessage> results = new LinkedList<>();
-
- Scheduler scheduler = intpA.getScheduler();
- for (int i = 0; i < concurrency; i++) {
- final String jobId = Integer.toString(i);
- scheduler.submit(new Job(jobId, Integer.toString(i), null, 200) {
- private Object r;
-
- @Override
- public Object getReturn() {
- return r;
- }
-
- @Override
- public void setResult(Object results) {
- this.r = results;
- }
-
- @Override
- public int progress() {
- return 0;
- }
-
- @Override
- public Map<String, Object> info() {
- return null;
- }
-
- @Override
- protected Object jobRun() throws Throwable {
- InterpreterResult ret = intpA.interpret(getJobName(), new InterpreterContext(
- "note",
- jobId,
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
-
- synchronized (results) {
- results.addAll(ret.message());
- results.notify();
- }
- return null;
- }
-
- @Override
- protected boolean jobAbort() {
- return false;
- }
-
- });
- }
-
- // wait for job finished
- synchronized (results) {
- while (results.size() != concurrency) {
- results.wait(300);
- }
- }
-
- int i = 0;
- for (InterpreterResultMessage result : results) {
- assertEquals(Integer.toString(i++), result.getData());
- }
- assertEquals(concurrency, i);
-
- intpA.close();
- }
-
-
- @Test
- public void testRunParallel() throws InterruptedException {
- Properties p = new Properties();
- p.put("parallel", "true");
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- final RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
- intpA.open();
-
- int concurrency = 4;
- final int timeToSleep = 1000;
- final List<InterpreterResultMessage> results = new LinkedList<>();
- long start = System.currentTimeMillis();
-
- Scheduler scheduler = intpA.getScheduler();
- for (int i = 0; i < concurrency; i++) {
- final String jobId = Integer.toString(i);
- scheduler.submit(new Job(jobId, Integer.toString(i), null, 300) {
- private Object r;
-
- @Override
- public Object getReturn() {
- return r;
- }
-
- @Override
- public void setResult(Object results) {
- this.r = results;
- }
-
- @Override
- public int progress() {
- return 0;
- }
-
- @Override
- public Map<String, Object> info() {
- return null;
- }
-
- @Override
- protected Object jobRun() throws Throwable {
- String stmt = Integer.toString(timeToSleep);
- InterpreterResult ret = intpA.interpret(stmt, new InterpreterContext(
- "note",
- jobId,
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
-
- synchronized (results) {
- results.addAll(ret.message());
- results.notify();
- }
- return stmt;
- }
-
- @Override
- protected boolean jobAbort() {
- return false;
- }
-
- });
- }
-
- // wait for job finished
- synchronized (results) {
- while (results.size() != concurrency) {
- results.wait(300);
- }
- }
-
- long end = System.currentTimeMillis();
-
- assertTrue(end - start < timeToSleep * concurrency);
-
- intpA.close();
- }
-
- @Test
- public void testInterpreterGroupResetBeforeProcessStarts() {
- Properties p = new Properties();
-
- RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpA.setInterpreterGroup(intpGroup);
- RemoteInterpreterProcess processA = intpA.getInterpreterProcess();
-
- intpA.setInterpreterGroup(new InterpreterGroup(intpA.getInterpreterGroup().getId()));
- RemoteInterpreterProcess processB = intpA.getInterpreterProcess();
-
- assertNotSame(processA.hashCode(), processB.hashCode());
- }
-
- @Test
- public void testInterpreterGroupResetAfterProcessFinished() {
- Properties p = new Properties();
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpA.setInterpreterGroup(intpGroup);
- RemoteInterpreterProcess processA = intpA.getInterpreterProcess();
- intpA.open();
-
- processA.dereference(); // intpA.close();
-
- intpA.setInterpreterGroup(new InterpreterGroup(intpA.getInterpreterGroup().getId()));
- RemoteInterpreterProcess processB = intpA.getInterpreterProcess();
-
- assertNotSame(processA.hashCode(), processB.hashCode());
- }
-
- @Test
- public void testInterpreterGroupResetDuringProcessRunning() throws InterruptedException {
- Properties p = new Properties();
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- final RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
- intpA.open();
-
- Job jobA = new Job("jobA", null) {
- private Object r;
-
- @Override
- public Object getReturn() {
- return r;
- }
-
- @Override
- public void setResult(Object results) {
- this.r = results;
- }
-
- @Override
- public int progress() {
- return 0;
- }
-
- @Override
- public Map<String, Object> info() {
- return null;
- }
-
- @Override
- protected Object jobRun() throws Throwable {
- return intpA.interpret("2000",
- new InterpreterContext(
- "note",
- "jobA",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
- }
-
- @Override
- protected boolean jobAbort() {
- return false;
- }
-
- };
- intpA.getScheduler().submit(jobA);
-
- // wait for job started
- while (intpA.getScheduler().getJobsRunning().size() == 0) {
- Thread.sleep(100);
- }
-
- // restart interpreter
- RemoteInterpreterProcess processA = intpA.getInterpreterProcess();
- intpA.close();
-
- InterpreterGroup newInterpreterGroup =
- new InterpreterGroup(intpA.getInterpreterGroup().getId());
- newInterpreterGroup.put("note", new LinkedList<Interpreter>());
-
- intpA.setInterpreterGroup(newInterpreterGroup);
- intpA.open();
- RemoteInterpreterProcess processB = intpA.getInterpreterProcess();
-
- assertNotSame(processA.hashCode(), processB.hashCode());
-
- }
-
- @Test
- public void testRemoteInterpreterSharesTheSameSchedulerInstanceInTheSameGroup() {
- Properties p = new Properties();
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
- RemoteInterpreter intpB = createMockInterpreterB(p);
-
- intpGroup.get("note").add(intpB);
- intpB.setInterpreterGroup(intpGroup);
-
- intpA.open();
- intpB.open();
-
- assertEquals(intpA.getScheduler(), intpB.getScheduler());
- }
-
- @Test
- public void testMultiInterpreterSession() {
- Properties p = new Properties();
- intpGroup.put("sessionA", new LinkedList<Interpreter>());
- intpGroup.put("sessionB", new LinkedList<Interpreter>());
-
- RemoteInterpreter intpAsessionA = createMockInterpreterA(p, "sessionA");
- intpGroup.get("sessionA").add(intpAsessionA);
- intpAsessionA.setInterpreterGroup(intpGroup);
-
- RemoteInterpreter intpBsessionA = createMockInterpreterB(p, "sessionA");
- intpGroup.get("sessionA").add(intpBsessionA);
- intpBsessionA.setInterpreterGroup(intpGroup);
-
- intpAsessionA.open();
- intpBsessionA.open();
-
- assertEquals(intpAsessionA.getScheduler(), intpBsessionA.getScheduler());
-
- RemoteInterpreter intpAsessionB = createMockInterpreterA(p, "sessionB");
- intpGroup.get("sessionB").add(intpAsessionB);
- intpAsessionB.setInterpreterGroup(intpGroup);
-
- RemoteInterpreter intpBsessionB = createMockInterpreterB(p, "sessionB");
- intpGroup.get("sessionB").add(intpBsessionB);
- intpBsessionB.setInterpreterGroup(intpGroup);
-
- intpAsessionB.open();
- intpBsessionB.open();
-
- assertEquals(intpAsessionB.getScheduler(), intpBsessionB.getScheduler());
- assertNotEquals(intpAsessionA.getScheduler(), intpAsessionB.getScheduler());
- }
-
- @Test
- public void should_push_local_angular_repo_to_remote() throws Exception {
- //Given
- final Client client = Mockito.mock(Client.class);
- final RemoteInterpreter intr = new RemoteInterpreter(new Properties(), "noteId",
- MockInterpreterA.class.getName(), "runner", "path", "localRepo", env, 10 * 1000, null,
- null, "anonymous", false);
- final AngularObjectRegistry registry = new AngularObjectRegistry("spark", null);
- registry.add("name", "DuyHai DOAN", "nodeId", "paragraphId");
- final InterpreterGroup interpreterGroup = new InterpreterGroup("groupId");
- interpreterGroup.setAngularObjectRegistry(registry);
- intr.setInterpreterGroup(interpreterGroup);
-
- final java.lang.reflect.Type registryType = new TypeToken<Map<String,
- Map<String, AngularObject>>>() {}.getType();
- final Gson gson = new Gson();
- final String expected = gson.toJson(registry.getRegistry(), registryType);
-
- //When
- intr.pushAngularObjectRegistryToRemote(client);
-
- //Then
- Mockito.verify(client).angularRegistryPush(expected);
- }
-
- @Test
- public void testEnvStringPattern() {
- assertFalse(RemoteInterpreterUtils.isEnvString(null));
- assertFalse(RemoteInterpreterUtils.isEnvString(""));
- assertFalse(RemoteInterpreterUtils.isEnvString("abcDEF"));
- assertFalse(RemoteInterpreterUtils.isEnvString("ABC-DEF"));
- assertTrue(RemoteInterpreterUtils.isEnvString("ABCDEF"));
- assertTrue(RemoteInterpreterUtils.isEnvString("ABC_DEF"));
- assertTrue(RemoteInterpreterUtils.isEnvString("ABC_DEF123"));
- }
-
- @Test
- public void testEnvronmentAndPropertySet() {
- Properties p = new Properties();
- p.setProperty("MY_ENV1", "env value 1");
- p.setProperty("my.property.1", "property value 1");
-
- RemoteInterpreter intp = new RemoteInterpreter(
- p,
- "note",
- MockInterpreterEnv.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- null,
- null,
- "anonymous",
- false);
-
- intpGroup.put("note", new LinkedList<Interpreter>());
- intpGroup.get("note").add(intp);
- intp.setInterpreterGroup(intpGroup);
-
- intp.open();
-
- InterpreterContext context = new InterpreterContext(
- "noteId",
- "id",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null);
-
-
- assertEquals("env value 1", intp.interpret("getEnv MY_ENV1", context).message().get(0).getData());
- assertEquals(Code.ERROR, intp.interpret("getProperty MY_ENV1", context).code());
- assertEquals(Code.ERROR, intp.interpret("getEnv my.property.1", context).code());
- assertEquals("property value 1", intp.interpret("getProperty my.property.1", context).message().get(0).getData());
-
- intp.close();
- }
-
- @Test
- public void testSetProgress() throws InterruptedException {
- // given MockInterpreterA set progress through InterpreterContext
- Properties p = new Properties();
- p.setProperty("progress", "50");
- final RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.put("note", new LinkedList<Interpreter>());
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
- intpA.open();
-
- final InterpreterContext context1 = new InterpreterContext(
- "noteId",
- "id1",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null);
-
- InterpreterContext context2 = new InterpreterContext(
- "noteId",
- "id2",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null);
-
-
- assertEquals(0, intpA.getProgress(context1));
- assertEquals(0, intpA.getProgress(context2));
-
- // when interpreter update progress through InterpreterContext
- Thread t = new Thread() {
- public void run() {
- InterpreterResult ret = intpA.interpret("1000", context1);
- }
- };
- t.start();
-
- // then progress need to be updated in given context
- while(intpA.getProgress(context1) == 0) Thread.yield();
- assertEquals(50, intpA.getProgress(context1));
- assertEquals(0, intpA.getProgress(context2));
-
- t.join();
- assertEquals(0, intpA.getProgress(context1));
- assertEquals(0, intpA.getProgress(context2));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java
deleted file mode 100644
index 975d6ea..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils;
-import org.junit.Test;
-
-public class RemoteInterpreterUtilsTest {
-
- @Test
- public void testFindRandomAvailablePortOnAllLocalInterfaces() throws IOException {
- assertTrue(RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces() > 0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java
deleted file mode 100644
index 50d9888..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote.mock;
-
-import java.util.List;
-import java.util.Properties;
-
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterException;
-import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.InterpreterResult.Code;
-import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
-import org.apache.zeppelin.scheduler.Scheduler;
-import org.apache.zeppelin.scheduler.SchedulerFactory;
-
-public class MockInterpreterA extends Interpreter {
-
- private String lastSt;
-
- public MockInterpreterA(Properties property) {
- super(property);
- }
-
- @Override
- public void open() {
- //new RuntimeException().printStackTrace();
- }
-
- @Override
- public void close() {
- }
-
- public String getLastStatement() {
- return lastSt;
- }
-
- @Override
- public InterpreterResult interpret(String st, InterpreterContext context) {
- if (property.containsKey("progress")) {
- context.setProgress(Integer.parseInt(getProperty("progress")));
- }
- try {
- Thread.sleep(Long.parseLong(st));
- this.lastSt = st;
- } catch (NumberFormatException | InterruptedException e) {
- throw new InterpreterException(e);
- }
- return new InterpreterResult(Code.SUCCESS, st);
- }
-
- @Override
- public void cancel(InterpreterContext context) {
-
- }
-
- @Override
- public FormType getFormType() {
- return FormType.NATIVE;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-
- @Override
- public List<InterpreterCompletion> completion(String buf, int cursor,
- InterpreterContext interpreterContext) {
- return null;
- }
-
- @Override
- public Scheduler getScheduler() {
- if (getProperty("parallel") != null && getProperty("parallel").equals("true")) {
- return SchedulerFactory.singleton().createOrGetParallelScheduler("interpreter_" + this.hashCode(), 10);
- } else {
- return SchedulerFactory.singleton().createOrGetFIFOScheduler("interpreter_" + this.hashCode());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java
deleted file mode 100644
index d4b26ad..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote.mock;
-
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.display.AngularObjectWatcher;
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.InterpreterResult.Code;
-import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
-
-public class MockInterpreterAngular extends Interpreter {
-
- AtomicInteger numWatch = new AtomicInteger(0);
-
- public MockInterpreterAngular(Properties property) {
- super(property);
- }
-
- @Override
- public void open() {
- }
-
- @Override
- public void close() {
-
- }
-
- @Override
- public InterpreterResult interpret(String st, InterpreterContext context) {
- String[] stmt = st.split(" ");
- String cmd = stmt[0];
- String name = null;
- if (stmt.length >= 2) {
- name = stmt[1];
- }
- String value = null;
- if (stmt.length == 3) {
- value = stmt[2];
- }
-
- AngularObjectRegistry registry = context.getAngularObjectRegistry();
-
- if (cmd.equals("add")) {
- registry.add(name, value, context.getNoteId(), null);
- registry.get(name, context.getNoteId(), null).addWatcher(new AngularObjectWatcher
- (null) {
-
- @Override
- public void watch(Object oldObject, Object newObject,
- InterpreterContext context) {
- numWatch.incrementAndGet();
- }
-
- });
- } else if (cmd.equalsIgnoreCase("update")) {
- registry.get(name, context.getNoteId(), null).set(value);
- } else if (cmd.equals("remove")) {
- registry.remove(name, context.getNoteId(), null);
- }
-
- try {
- Thread.sleep(500); // wait for watcher executed
- } catch (InterruptedException e) {
- logger.error("Exception in MockInterpreterAngular while interpret Thread.sleep", e);
- }
-
- String msg = registry.getAll(context.getNoteId(), null).size() + " " + Integer.toString(numWatch
- .get());
- return new InterpreterResult(Code.SUCCESS, msg);
- }
-
- @Override
- public void cancel(InterpreterContext context) {
- }
-
- @Override
- public FormType getFormType() {
- return FormType.NATIVE;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-
- @Override
- public List<InterpreterCompletion> completion(String buf, int cursor,
- InterpreterContext interpreterContext) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterB.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterB.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterB.java
deleted file mode 100644
index 7103335..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterB.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote.mock;
-
-import java.util.List;
-import java.util.Properties;
-
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterException;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
-import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.InterpreterResult.Code;
-import org.apache.zeppelin.interpreter.WrappedInterpreter;
-import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
-import org.apache.zeppelin.scheduler.Scheduler;
-
-public class MockInterpreterB extends Interpreter {
-
- public MockInterpreterB(Properties property) {
- super(property);
- }
-
- @Override
- public void open() {
- //new RuntimeException().printStackTrace();
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public InterpreterResult interpret(String st, InterpreterContext context) {
- MockInterpreterA intpA = getInterpreterA();
- String intpASt = intpA.getLastStatement();
- long timeToSleep = Long.parseLong(st);
- if (intpASt != null) {
- timeToSleep += Long.parseLong(intpASt);
- }
- try {
- Thread.sleep(timeToSleep);
- } catch (NumberFormatException | InterruptedException e) {
- throw new InterpreterException(e);
- }
- return new InterpreterResult(Code.SUCCESS, Long.toString(timeToSleep));
- }
-
- @Override
- public void cancel(InterpreterContext context) {
-
- }
-
- @Override
- public FormType getFormType() {
- return FormType.NATIVE;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-
- @Override
- public List<InterpreterCompletion> completion(String buf, int cursor,
- InterpreterContext interpreterContext) {
- return null;
- }
-
- public MockInterpreterA getInterpreterA() {
- InterpreterGroup interpreterGroup = getInterpreterGroup();
- synchronized (interpreterGroup) {
- for (List<Interpreter> interpreters : interpreterGroup.values()) {
- boolean belongsToSameNoteGroup = false;
- MockInterpreterA a = null;
- for (Interpreter intp : interpreters) {
- if (intp.getClassName().equals(MockInterpreterA.class.getName())) {
- Interpreter p = intp;
- while (p instanceof WrappedInterpreter) {
- p = ((WrappedInterpreter) p).getInnerInterpreter();
- }
- a = (MockInterpreterA) p;
- }
-
- Interpreter p = intp;
- while (p instanceof WrappedInterpreter) {
- p = ((WrappedInterpreter) p).getInnerInterpreter();
- }
- if (this == p) {
- belongsToSameNoteGroup = true;
- }
- }
- if (belongsToSameNoteGroup) {
- return a;
- }
- }
- }
- return null;
- }
-
- @Override
- public Scheduler getScheduler() {
- MockInterpreterA intpA = getInterpreterA();
- if (intpA != null) {
- return intpA.getScheduler();
- }
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterEnv.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterEnv.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterEnv.java
deleted file mode 100644
index 12e11f7..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterEnv.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.zeppelin.interpreter.remote.mock;
-
-import org.apache.zeppelin.interpreter.*;
-import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
-import org.apache.zeppelin.scheduler.Scheduler;
-import org.apache.zeppelin.scheduler.SchedulerFactory;
-
-import java.util.List;
-import java.util.Properties;
-
-
-public class MockInterpreterEnv extends Interpreter {
-
- public MockInterpreterEnv(Properties property) {
- super(property);
- }
-
- @Override
- public void open() {
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public InterpreterResult interpret(String st, InterpreterContext context) {
- String[] cmd = st.split(" ");
- if (cmd[0].equals("getEnv")) {
- return new InterpreterResult(InterpreterResult.Code.SUCCESS, System.getenv(cmd[1]));
- } else if (cmd[0].equals("getProperty")){
- return new InterpreterResult(InterpreterResult.Code.SUCCESS, System.getProperty(cmd[1]));
- } else {
- return new InterpreterResult(InterpreterResult.Code.ERROR, cmd[0]);
- }
- }
-
- @Override
- public void cancel(InterpreterContext context) {
-
- }
-
- @Override
- public FormType getFormType() {
- return FormType.NATIVE;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-
- @Override
- public List<InterpreterCompletion> completion(String buf, int cursor,
- InterpreterContext interpreterContext) {
- return null;
- }
-
- @Override
- public Scheduler getScheduler() {
- return SchedulerFactory.singleton().createOrGetFIFOScheduler("interpreter_" + this.hashCode());
- }
-}
-
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java
deleted file mode 100644
index 349315c..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.zeppelin.interpreter.remote.mock;
-
-import org.apache.zeppelin.interpreter.*;
-import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
-import org.apache.zeppelin.scheduler.Scheduler;
-import org.apache.zeppelin.scheduler.SchedulerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Properties;
-
-/**
- * MockInterpreter to test outputstream
- */
-public class MockInterpreterOutputStream extends Interpreter {
- private String lastSt;
-
- public MockInterpreterOutputStream(Properties property) {
- super(property);
- }
-
- @Override
- public void open() {
- //new RuntimeException().printStackTrace();
- }
-
- @Override
- public void close() {
- }
-
- public String getLastStatement() {
- return lastSt;
- }
-
- @Override
- public InterpreterResult interpret(String st, InterpreterContext context) {
- String[] ret = st.split(":");
- try {
- if (ret[1] != null) {
- context.out.write(ret[1]);
- }
- } catch (IOException e) {
- throw new InterpreterException(e);
- }
- return new InterpreterResult(InterpreterResult.Code.valueOf(ret[0]), (ret.length > 2) ?
- ret[2] : "");
- }
-
- @Override
- public void cancel(InterpreterContext context) {
-
- }
-
- @Override
- public FormType getFormType() {
- return FormType.NATIVE;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-
- @Override
- public List<InterpreterCompletion> completion(String buf, int cursor,
- InterpreterContext interpreterContext) {
- return null;
- }
-
- @Override
- public Scheduler getScheduler() {
- return SchedulerFactory.singleton().createOrGetFIFOScheduler("interpreter_" + this.hashCode());
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterResourcePool.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterResourcePool.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterResourcePool.java
deleted file mode 100644
index c4ff6ab..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterResourcePool.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote.mock;
-
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.google.gson.Gson;
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.display.AngularObjectWatcher;
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.InterpreterResult.Code;
-import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
-import org.apache.zeppelin.resource.Resource;
-import org.apache.zeppelin.resource.ResourcePool;
-
-public class MockInterpreterResourcePool extends Interpreter {
-
- AtomicInteger numWatch = new AtomicInteger(0);
-
- public MockInterpreterResourcePool(Properties property) {
- super(property);
- }
-
- @Override
- public void open() {
- }
-
- @Override
- public void close() {
-
- }
-
- @Override
- public InterpreterResult interpret(String st, InterpreterContext context) {
- String[] stmt = st.split(" ");
- String cmd = stmt[0];
- String noteId = null;
- String paragraphId = null;
- String name = null;
- if (stmt.length >= 2) {
- String[] npn = stmt[1].split(":");
- if (npn.length >= 3) {
- noteId = npn[0];
- paragraphId = npn[1];
- name = npn[2];
- } else {
- name = stmt[1];
- }
- }
- String value = null;
- if (stmt.length >= 3) {
- value = stmt[2];
- }
-
- ResourcePool resourcePool = context.getResourcePool();
- Object ret = null;
- if (cmd.equals("put")) {
- resourcePool.put(noteId, paragraphId, name, value);
- } else if (cmd.equalsIgnoreCase("get")) {
- Resource resource = resourcePool.get(noteId, paragraphId, name);
- if (resource != null) {
- ret = resourcePool.get(noteId, paragraphId, name).get();
- } else {
- ret = "";
- }
- } else if (cmd.equals("remove")) {
- ret = resourcePool.remove(noteId, paragraphId, name);
- } else if (cmd.equals("getAll")) {
- ret = resourcePool.getAll();
- } else if (cmd.equals("invoke")) {
- Resource resource = resourcePool.get(noteId, paragraphId, name);
- if (stmt.length >=4) {
- Resource res = resource.invokeMethod(value, null, null, stmt[3]);
- ret = res.get();
- } else {
- ret = resource.invokeMethod(value, null, null);
- }
- }
-
- try {
- Thread.sleep(500); // wait for watcher executed
- } catch (InterruptedException e) {
- }
-
- Gson gson = new Gson();
- return new InterpreterResult(Code.SUCCESS, gson.toJson(ret));
- }
-
- @Override
- public void cancel(InterpreterContext context) {
- }
-
- @Override
- public FormType getFormType() {
- return FormType.NATIVE;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-
- @Override
- public List<InterpreterCompletion> completion(String buf, int cursor,
- InterpreterContext interpreterContext) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java
deleted file mode 100644
index 5632513..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.zeppelin.notebook;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
-import org.apache.zeppelin.dep.Dependency;
-import org.apache.zeppelin.dep.DependencyResolver;
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterFactory;
-import org.apache.zeppelin.interpreter.InterpreterInfo;
-import org.apache.zeppelin.interpreter.InterpreterOption;
-import org.apache.zeppelin.interpreter.DefaultInterpreterProperty;
-import org.apache.zeppelin.interpreter.InterpreterProperty;
-import org.apache.zeppelin.interpreter.InterpreterSettingManager;
-import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
-import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
-import org.apache.zeppelin.interpreter.mock.MockInterpreter11;
-import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-public class NoteInterpreterLoaderTest {
-
- private File tmpDir;
- private ZeppelinConfiguration conf;
- private InterpreterFactory factory;
- private InterpreterSettingManager interpreterSettingManager;
- private DependencyResolver depResolver;
-
- @Before
- public void setUp() throws Exception {
- tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
- tmpDir.mkdirs();
- new File(tmpDir, "conf").mkdirs();
-
- System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
-
- conf = ZeppelinConfiguration.create();
-
- depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
-
- ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
- interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, Maps.<String, Object>newHashMap()));
- interpreterInfos.add(new InterpreterInfo(MockInterpreter11.class.getName(), "mock11", false, Maps.<String, Object>newHashMap()));
- ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
- interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, Maps.<String, Object>newHashMap()));
-
- interpreterSettingManager.add("group1", interpreterInfos, Lists.<Dependency>newArrayList(), new InterpreterOption(), Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock", null);
- interpreterSettingManager.add("group2", interpreterInfos2, Lists.<Dependency>newArrayList(), new InterpreterOption(), Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock", null);
-
- interpreterSettingManager.createNewSetting("group1", "group1", Lists.<Dependency>newArrayList(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
- interpreterSettingManager.createNewSetting("group2", "group2", Lists.<Dependency>newArrayList(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
-
-
- }
-
- @After
- public void tearDown() throws Exception {
- delete(tmpDir);
- Interpreter.registeredInterpreters.clear();
- }
-
- @Test
- public void testGetInterpreter() throws IOException {
- interpreterSettingManager.setInterpreters("user", "note", interpreterSettingManager.getDefaultInterpreterSettingList());
-
- // when there're no interpreter selection directive
- assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", null).getClassName());
- assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", "").getClassName());
- assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", " ").getClassName());
-
- // when group name is omitted
- assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("user", "note", "mock11").getClassName());
-
- // when 'name' is ommitted
- assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", "group1").getClassName());
- assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("user", "note", "group2").getClassName());
-
- // when nothing is ommitted
- assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", "group1.mock1").getClassName());
- assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("user", "note", "group1.mock11").getClassName());
- assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("user", "note", "group2.mock2").getClassName());
-
- interpreterSettingManager.closeNote("user", "note");
- }
-
- @Test
- public void testNoteSession() throws IOException {
- interpreterSettingManager.setInterpreters("user", "noteA", interpreterSettingManager.getDefaultInterpreterSettingList());
- interpreterSettingManager.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
-
- interpreterSettingManager.setInterpreters("user", "noteB", interpreterSettingManager.getDefaultInterpreterSettingList());
- interpreterSettingManager.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
-
- // interpreters are not created before accessing it
- assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA"));
- assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB"));
-
- factory.getInterpreter("user", "noteA", null).open();
- factory.getInterpreter("user", "noteB", null).open();
-
- assertTrue(
- factory.getInterpreter("user", "noteA", null).getInterpreterGroup().getId().equals(
- factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId()));
-
- // interpreters are created after accessing it
- assertNotNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA"));
- assertNotNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB"));
-
- // invalid close
- interpreterSettingManager.closeNote("user", "note");
- assertNotNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA"));
- assertNotNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB"));
-
- // when
- interpreterSettingManager.closeNote("user", "noteA");
- interpreterSettingManager.closeNote("user", "noteB");
-
- // interpreters are destroyed after close
- assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA"));
- assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB"));
-
- }
-
- @Test
- public void testNotePerInterpreterProcess() throws IOException {
- interpreterSettingManager.setInterpreters("user", "noteA", interpreterSettingManager.getDefaultInterpreterSettingList());
- interpreterSettingManager.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
-
- interpreterSettingManager.setInterpreters("user", "noteB", interpreterSettingManager.getDefaultInterpreterSettingList());
- interpreterSettingManager.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
-
- // interpreters are not created before accessing it
- assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session"));
- assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session"));
-
- factory.getInterpreter("user", "noteA", null).open();
- factory.getInterpreter("user", "noteB", null).open();
-
- // per note interpreter process
- assertFalse(
- factory.getInterpreter("user", "noteA", null).getInterpreterGroup().getId().equals(
- factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId()));
-
- // interpreters are created after accessing it
- assertNotNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session"));
- assertNotNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session"));
-
- // when
- interpreterSettingManager.closeNote("user", "noteA");
- interpreterSettingManager.closeNote("user", "noteB");
-
- // interpreters are destroyed after close
- assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session"));
- assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session"));
- }
-
- @Test
- public void testNoteInterpreterCloseForAll() throws IOException {
- interpreterSettingManager.setInterpreters("user", "FitstNote", interpreterSettingManager.getDefaultInterpreterSettingList());
- interpreterSettingManager.getInterpreterSettings("FitstNote").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
-
- interpreterSettingManager.setInterpreters("user", "yourFirstNote", interpreterSettingManager.getDefaultInterpreterSettingList());
- interpreterSettingManager.getInterpreterSettings("yourFirstNote").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
-
- // interpreters are not created before accessing it
- assertNull(interpreterSettingManager.getInterpreterSettings("FitstNote").get(0).getInterpreterGroup("user", "FitstNote").get("FitstNote"));
- assertNull(interpreterSettingManager.getInterpreterSettings("yourFirstNote").get(0).getInterpreterGroup("user", "yourFirstNote").get("yourFirstNote"));
-
- Interpreter firstNoteIntp = factory.getInterpreter("user", "FitstNote", "group1.mock1");
- Interpreter yourFirstNoteIntp = factory.getInterpreter("user", "yourFirstNote", "group1.mock1");
-
- firstNoteIntp.open();
- yourFirstNoteIntp.open();
-
- assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
- assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
-
- interpreterSettingManager.closeNote("user", "FitstNote");
-
- assertFalse(((LazyOpenInterpreter)firstNoteIntp).isOpen());
- assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
-
- //reopen
- firstNoteIntp.open();
-
- assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
- assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
-
- // invalid check
- interpreterSettingManager.closeNote("invalid", "Note");
-
- assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
- assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
-
- // invalid contains value check
- interpreterSettingManager.closeNote("u", "Note");
-
- assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
- assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
- }
-
-
- private void delete(File file){
- if(file.isFile()) file.delete();
- else if(file.isDirectory()){
- File [] files = file.listFiles();
- if(files!=null && files.length>0){
- for(File f : files){
- delete(f);
- }
- }
- file.delete();
- }
- }
-}
[10/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
new file mode 100644
index 0000000..3f84cd0
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
@@ -0,0 +1,911 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.annotations.SerializedName;
+import com.google.gson.internal.StringMap;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.dep.Dependency;
+import org.apache.zeppelin.dep.DependencyResolver;
+import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.display.AngularObjectRegistryListener;
+import org.apache.zeppelin.helium.ApplicationEventListener;
+import org.apache.zeppelin.interpreter.remote.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE;
+import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT;
+import static org.apache.zeppelin.util.IdHashes.generateId;
+
+/**
+ * Represent one InterpreterSetting in the interpreter setting page
+ */
+public class InterpreterSetting {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterSetting.class);
+ private static final String SHARED_PROCESS = "shared_process";
+ private static final String SHARED_SESSION = "shared_session";
+ private static final Map<String, Object> DEFAULT_EDITOR = ImmutableMap.of(
+ "language", (Object) "text",
+ "editOnDblClick", false);
+
+ private String id;
+ private String name;
+ // the original interpreter setting template name where it is created from
+ private String group;
+
+ //TODO(zjffdu) make the interpreter.json consistent with interpreter-setting.json
+ /**
+ * properties can be either Properties or Map<String, InterpreterProperty>
+ * properties should be:
+ * - Properties when Interpreter instances are saved to `conf/interpreter.json` file
+ * - Map<String, InterpreterProperty> when Interpreters are registered
+ * : this is needed after https://github.com/apache/zeppelin/pull/1145
+ * which changed the way of getting default interpreter setting AKA interpreterSettingsRef
+ * Note(mina): In order to simplify the implementation, I chose to change properties
+ * from Properties to Object instead of creating new classes.
+ */
+ private Object properties = new Properties();
+
+ private Status status;
+ private String errorReason;
+
+ @SerializedName("interpreterGroup")
+ private List<InterpreterInfo> interpreterInfos;
+
+ private List<Dependency> dependencies = new ArrayList<>();
+ private InterpreterOption option = new InterpreterOption(true);
+
+ @SerializedName("runner")
+ private InterpreterRunner interpreterRunner;
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ private transient InterpreterSettingManager interpreterSettingManager;
+ private transient String interpreterDir;
+ private final transient Map<String, InterpreterGroup> interpreterGroups =
+ new ConcurrentHashMap<>();
+
+ private final transient ReentrantReadWriteLock.ReadLock interpreterGroupReadLock;
+ private final transient ReentrantReadWriteLock.WriteLock interpreterGroupWriteLock;
+
+ private transient AngularObjectRegistryListener angularObjectRegistryListener;
+ private transient RemoteInterpreterProcessListener remoteInterpreterProcessListener;
+ private transient ApplicationEventListener appEventListener;
+ private transient DependencyResolver dependencyResolver;
+
+ private transient Map<String, String> infos;
+
+ // Map of the note and paragraphs which has runtime infos generated by this interpreter setting.
+ // This map is used to clear the infos in paragraph when the interpretersetting is restarted
+ private transient Map<String, Set<String>> runtimeInfosToBeCleared;
+
+ private transient ZeppelinConfiguration conf = new ZeppelinConfiguration();
+
+ private transient Map<String, URLClassLoader> cleanCl =
+ Collections.synchronizedMap(new HashMap<String, URLClassLoader>());
+ ///////////////////////////////////////////////////////////////////////////////////////////
+
+
+ /**
+ * Builder class for InterpreterSetting
+ */
+ public static class Builder {
+ private InterpreterSetting interpreterSetting;
+
+ public Builder() {
+ this.interpreterSetting = new InterpreterSetting();
+ }
+
+ public Builder setId(String id) {
+ interpreterSetting.id = id;
+ return this;
+ }
+
+ public Builder setName(String name) {
+ interpreterSetting.name = name;
+ return this;
+ }
+
+ public Builder setGroup(String group) {
+ interpreterSetting.group = group;
+ return this;
+ }
+
+ public Builder setInterpreterInfos(List<InterpreterInfo> interpreterInfos) {
+ interpreterSetting.interpreterInfos = interpreterInfos;
+ return this;
+ }
+
+ public Builder setProperties(Object properties) {
+ interpreterSetting.properties = properties;
+ return this;
+ }
+
+ public Builder setOption(InterpreterOption option) {
+ interpreterSetting.option = option;
+ return this;
+ }
+
+ public Builder setInterpreterDir(String interpreterDir) {
+ interpreterSetting.interpreterDir = interpreterDir;
+ return this;
+ }
+
+ public Builder setRunner(InterpreterRunner runner) {
+ interpreterSetting.interpreterRunner = runner;
+ return this;
+ }
+
+ public Builder setDependencies(List<Dependency> dependencies) {
+ interpreterSetting.dependencies = dependencies;
+ return this;
+ }
+
+ public Builder setConf(ZeppelinConfiguration conf) {
+ interpreterSetting.conf = conf;
+ return this;
+ }
+
+ public Builder setDependencyResolver(DependencyResolver dependencyResolver) {
+ interpreterSetting.dependencyResolver = dependencyResolver;
+ return this;
+ }
+
+// public Builder setInterpreterRunner(InterpreterRunner runner) {
+// interpreterSetting.interpreterRunner = runner;
+// return this;
+// }
+
+ public Builder setIntepreterSettingManager(
+ InterpreterSettingManager interpreterSettingManager) {
+ interpreterSetting.interpreterSettingManager = interpreterSettingManager;
+ return this;
+ }
+
+ public Builder setRemoteInterpreterProcessListener(RemoteInterpreterProcessListener
+ remoteInterpreterProcessListener) {
+ interpreterSetting.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
+ return this;
+ }
+
+ public Builder setAngularObjectRegistryListener(
+ AngularObjectRegistryListener angularObjectRegistryListener) {
+ interpreterSetting.angularObjectRegistryListener = angularObjectRegistryListener;
+ return this;
+ }
+
+ public Builder setApplicationEventListener(ApplicationEventListener applicationEventListener) {
+ interpreterSetting.appEventListener = applicationEventListener;
+ return this;
+ }
+
+ public InterpreterSetting create() {
+ // post processing
+ interpreterSetting.postProcessing();
+ return interpreterSetting;
+ }
+ }
+
+ public InterpreterSetting() {
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ this.id = generateId();
+ interpreterGroupReadLock = lock.readLock();
+ interpreterGroupWriteLock = lock.writeLock();
+ }
+
+ void postProcessing() {
+ this.status = Status.READY;
+ }
+
+ /**
+ * Create interpreter from InterpreterSettingTemplate
+ *
+ * @param o interpreterSetting from InterpreterSettingTemplate
+ */
+ public InterpreterSetting(InterpreterSetting o) {
+ this();
+ this.id = generateId();
+ this.name = o.name;
+ this.group = o.group;
+ this.properties = convertInterpreterProperties(
+ (Map<String, DefaultInterpreterProperty>) o.getProperties());
+ this.interpreterInfos = new ArrayList<>(o.getInterpreterInfos());
+ this.option = InterpreterOption.fromInterpreterOption(o.getOption());
+ this.dependencies = new ArrayList<>(o.getDependencies());
+ this.interpreterDir = o.getInterpreterDir();
+ this.interpreterRunner = o.getInterpreterRunner();
+ this.conf = o.getConf();
+ }
+
+ public AngularObjectRegistryListener getAngularObjectRegistryListener() {
+ return angularObjectRegistryListener;
+ }
+
+ public RemoteInterpreterProcessListener getRemoteInterpreterProcessListener() {
+ return remoteInterpreterProcessListener;
+ }
+
+ public ApplicationEventListener getAppEventListener() {
+ return appEventListener;
+ }
+
+ public DependencyResolver getDependencyResolver() {
+ return dependencyResolver;
+ }
+
+ public InterpreterSettingManager getInterpreterSettingManager() {
+ return interpreterSettingManager;
+ }
+
+ public void setAngularObjectRegistryListener(AngularObjectRegistryListener
+ angularObjectRegistryListener) {
+ this.angularObjectRegistryListener = angularObjectRegistryListener;
+ }
+
+ public void setAppEventListener(ApplicationEventListener appEventListener) {
+ this.appEventListener = appEventListener;
+ }
+
+ public void setRemoteInterpreterProcessListener(RemoteInterpreterProcessListener
+ remoteInterpreterProcessListener) {
+ this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
+ }
+
+ public void setDependencyResolver(DependencyResolver dependencyResolver) {
+ this.dependencyResolver = dependencyResolver;
+ }
+
+ public void setInterpreterSettingManager(InterpreterSettingManager interpreterSettingManager) {
+ this.interpreterSettingManager = interpreterSettingManager;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getGroup() {
+ return group;
+ }
+
+ private String getInterpreterGroupId(String user, String noteId) {
+ String key;
+ if (option.isExistingProcess) {
+ key = Constants.EXISTING_PROCESS;
+ } else if (getOption().isProcess()) {
+ key = (option.perUserIsolated() ? user : "") + ":" + (option.perNoteIsolated() ? noteId : "");
+ } else {
+ key = SHARED_PROCESS;
+ }
+
+ //TODO(zjffdu) we encode interpreter setting id into groupId, this is not a good design
+ return id + ":" + key;
+ }
+
+ private String getInterpreterSessionId(String user, String noteId) {
+ String key;
+ if (option.isExistingProcess()) {
+ key = Constants.EXISTING_PROCESS;
+ } else if (option.perNoteScoped() && option.perUserScoped()) {
+ key = user + ":" + noteId;
+ } else if (option.perUserScoped()) {
+ key = user;
+ } else if (option.perNoteScoped()) {
+ key = noteId;
+ } else {
+ key = SHARED_SESSION;
+ }
+
+ return key;
+ }
+
+ public InterpreterGroup getOrCreateInterpreterGroup(String user, String noteId) {
+ String groupId = getInterpreterGroupId(user, noteId);
+ try {
+ interpreterGroupWriteLock.lock();
+ if (!interpreterGroups.containsKey(groupId)) {
+ LOGGER.info("Create InterpreterGroup with groupId {} for user {} and note {}",
+ groupId, user, noteId);
+ InterpreterGroup intpGroup = createInterpreterGroup(groupId);
+ interpreterGroups.put(groupId, intpGroup);
+ }
+ return interpreterGroups.get(groupId);
+ } finally {
+ interpreterGroupWriteLock.unlock();;
+ }
+ }
+
+ void removeInterpreterGroup(String groupId) {
+ this.interpreterGroups.remove(groupId);
+ }
+
+ InterpreterGroup getInterpreterGroup(String user, String noteId) {
+ String groupId = getInterpreterGroupId(user, noteId);
+ try {
+ interpreterGroupReadLock.lock();
+ return interpreterGroups.get(groupId);
+ } finally {
+ interpreterGroupReadLock.unlock();;
+ }
+ }
+
+ InterpreterGroup getInterpreterGroup(String groupId) {
+ return interpreterGroups.get(groupId);
+ }
+
+ @VisibleForTesting
+ public ArrayList<InterpreterGroup> getAllInterpreterGroups() {
+ try {
+ interpreterGroupReadLock.lock();
+ return new ArrayList(interpreterGroups.values());
+ } finally {
+ interpreterGroupReadLock.unlock();
+ }
+ }
+
+ Map<String, Object> getEditorFromSettingByClassName(String className) {
+ for (InterpreterInfo intpInfo : interpreterInfos) {
+ if (className.equals(intpInfo.getClassName())) {
+ if (intpInfo.getEditor() == null) {
+ break;
+ }
+ return intpInfo.getEditor();
+ }
+ }
+ return DEFAULT_EDITOR;
+ }
+
+ void closeInterpreters(String user, String noteId) {
+ InterpreterGroup interpreterGroup = getInterpreterGroup(user, noteId);
+ if (interpreterGroup != null) {
+ String sessionId = getInterpreterSessionId(user, noteId);
+ interpreterGroup.close(sessionId);
+ }
+ }
+
+ public void close() {
+ LOGGER.info("Close InterpreterSetting: " + name);
+ for (InterpreterGroup intpGroup : interpreterGroups.values()) {
+ intpGroup.close();
+ }
+ interpreterGroups.clear();
+ this.runtimeInfosToBeCleared = null;
+ this.infos = null;
+ }
+
+ public void setProperties(Object object) {
+ if (object instanceof StringMap) {
+ StringMap<String> map = (StringMap) properties;
+ Properties newProperties = new Properties();
+ for (String key : map.keySet()) {
+ newProperties.put(key, map.get(key));
+ }
+ this.properties = newProperties;
+ } else {
+ this.properties = object;
+ }
+ }
+
+
+ public Object getProperties() {
+ return properties;
+ }
+
+ @VisibleForTesting
+ public void setProperty(String name, String value) {
+ ((Map<String, InterpreterProperty>) properties).put(name, new InterpreterProperty(name, value));
+ }
+
+ // This method is supposed to be only called by InterpreterSetting
+ // but not InterpreterSetting Template
+ public Properties getJavaProperties() {
+ Properties jProperties = new Properties();
+ Map<String, InterpreterProperty> iProperties = (Map<String, InterpreterProperty>) properties;
+ for (Map.Entry<String, InterpreterProperty> entry : iProperties.entrySet()) {
+ jProperties.setProperty(entry.getKey(), entry.getValue().getValue().toString());
+ }
+
+ if (!jProperties.containsKey("zeppelin.interpreter.output.limit")) {
+ jProperties.setProperty("zeppelin.interpreter.output.limit",
+ conf.getInt(ZEPPELIN_INTERPRETER_OUTPUT_LIMIT) + "");
+ }
+
+ if (!jProperties.containsKey("zeppelin.interpreter.max.poolsize")) {
+ jProperties.setProperty("zeppelin.interpreter.max.poolsize",
+ conf.getInt(ZEPPELIN_INTERPRETER_MAX_POOL_SIZE) + "");
+ }
+
+ String interpreterLocalRepoPath = conf.getInterpreterLocalRepoPath();
+ //TODO(zjffdu) change it to interpreterDir/{interpreter_name}
+ jProperties.setProperty("zeppelin.interpreter.localRepo",
+ interpreterLocalRepoPath + "/" + id);
+ return jProperties;
+ }
+
+ public ZeppelinConfiguration getConf() {
+ return conf;
+ }
+
+ public void setConf(ZeppelinConfiguration conf) {
+ this.conf = conf;
+ }
+
+ public List<Dependency> getDependencies() {
+ return dependencies;
+ }
+
+ public void setDependencies(List<Dependency> dependencies) {
+ this.dependencies = dependencies;
+ loadInterpreterDependencies();
+ }
+
+ public InterpreterOption getOption() {
+ return option;
+ }
+
+ public void setOption(InterpreterOption option) {
+ this.option = option;
+ }
+
+ public String getInterpreterDir() {
+ return interpreterDir;
+ }
+
+ public void setInterpreterDir(String interpreterDir) {
+ this.interpreterDir = interpreterDir;
+ }
+
+ public List<InterpreterInfo> getInterpreterInfos() {
+ return interpreterInfos;
+ }
+
+ void appendDependencies(List<Dependency> dependencies) {
+ for (Dependency dependency : dependencies) {
+ if (!this.dependencies.contains(dependency)) {
+ this.dependencies.add(dependency);
+ }
+ }
+ loadInterpreterDependencies();
+ }
+
+ void setInterpreterOption(InterpreterOption interpreterOption) {
+ this.option = interpreterOption;
+ }
+
+ public void setProperties(Properties p) {
+ this.properties = p;
+ }
+
+ void setGroup(String group) {
+ this.group = group;
+ }
+
+ void setName(String name) {
+ this.name = name;
+ }
+
+ /***
+ * Interpreter status
+ */
+ public enum Status {
+ DOWNLOADING_DEPENDENCIES,
+ ERROR,
+ READY
+ }
+
+ public Status getStatus() {
+ return status;
+ }
+
+ public void setStatus(Status status) {
+ this.status = status;
+ }
+
+ public String getErrorReason() {
+ return errorReason;
+ }
+
+ public void setErrorReason(String errorReason) {
+ this.errorReason = errorReason;
+ }
+
+ public void setInterpreterInfos(List<InterpreterInfo> interpreterInfos) {
+ this.interpreterInfos = interpreterInfos;
+ }
+
+ public void setInfos(Map<String, String> infos) {
+ this.infos = infos;
+ }
+
+ public Map<String, String> getInfos() {
+ return infos;
+ }
+
+ public InterpreterRunner getInterpreterRunner() {
+ return interpreterRunner;
+ }
+
+ public void setInterpreterRunner(InterpreterRunner interpreterRunner) {
+ this.interpreterRunner = interpreterRunner;
+ }
+
+ public void addNoteToPara(String noteId, String paraId) {
+ if (runtimeInfosToBeCleared == null) {
+ runtimeInfosToBeCleared = new HashMap<>();
+ }
+ Set<String> paraIdSet = runtimeInfosToBeCleared.get(noteId);
+ if (paraIdSet == null) {
+ paraIdSet = new HashSet<>();
+ runtimeInfosToBeCleared.put(noteId, paraIdSet);
+ }
+ paraIdSet.add(paraId);
+ }
+
+ public Map<String, Set<String>> getNoteIdAndParaMap() {
+ return runtimeInfosToBeCleared;
+ }
+
+ public void clearNoteIdAndParaMap() {
+ runtimeInfosToBeCleared = null;
+ }
+
+
+ //////////////////////////// IMPORTANT ////////////////////////////////////////////////
+ ///////////////////////////////////////////////////////////////////////////////////////
+ ///////////////////////////////////////////////////////////////////////////////////////
+ // This is the only place to create interpreters. For now we always create multiple interpreter
+ // together (one session). We don't support to create single interpreter yet.
+ List<Interpreter> createInterpreters(String user, String sessionId) {
+ List<Interpreter> interpreters = new ArrayList<>();
+ List<InterpreterInfo> interpreterInfos = getInterpreterInfos();
+ for (InterpreterInfo info : interpreterInfos) {
+ Interpreter interpreter = null;
+ if (option.isRemote()) {
+ interpreter = new RemoteInterpreter(getJavaProperties(), sessionId,
+ info.getClassName(), user);
+ } else {
+ interpreter = createLocalInterpreter(info.getClassName());
+ }
+
+ if (info.isDefaultInterpreter()) {
+ interpreters.add(0, interpreter);
+ } else {
+ interpreters.add(interpreter);
+ }
+ LOGGER.info("Interpreter {} created for user: {}, sessionId: {}",
+ interpreter.getClassName(), user, sessionId);
+ }
+ return interpreters;
+ }
+
+ // Create Interpreter in ZeppelinServer for non-remote mode
+ private Interpreter createLocalInterpreter(String className)
+ throws InterpreterException {
+ LOGGER.info("Create Local Interpreter {} from {}", className, interpreterDir);
+
+ ClassLoader oldcl = Thread.currentThread().getContextClassLoader();
+ try {
+
+ URLClassLoader ccl = cleanCl.get(interpreterDir);
+ if (ccl == null) {
+ // classloader fallback
+ ccl = URLClassLoader.newInstance(new URL[]{}, oldcl);
+ }
+
+ boolean separateCL = true;
+ try { // check if server's classloader has driver already.
+ Class cls = this.getClass().forName(className);
+ if (cls != null) {
+ separateCL = false;
+ }
+ } catch (Exception e) {
+ LOGGER.error("exception checking server classloader driver", e);
+ }
+
+ URLClassLoader cl;
+
+ if (separateCL == true) {
+ cl = URLClassLoader.newInstance(new URL[]{}, ccl);
+ } else {
+ cl = ccl;
+ }
+ Thread.currentThread().setContextClassLoader(cl);
+
+ Class<Interpreter> replClass = (Class<Interpreter>) cl.loadClass(className);
+ Constructor<Interpreter> constructor =
+ replClass.getConstructor(new Class[]{Properties.class});
+ Interpreter repl = constructor.newInstance(getJavaProperties());
+ repl.setClassloaderUrls(ccl.getURLs());
+ LazyOpenInterpreter intp = new LazyOpenInterpreter(new ClassloaderInterpreter(repl, cl));
+ return intp;
+ } catch (SecurityException e) {
+ throw new InterpreterException(e);
+ } catch (NoSuchMethodException e) {
+ throw new InterpreterException(e);
+ } catch (IllegalArgumentException e) {
+ throw new InterpreterException(e);
+ } catch (InstantiationException e) {
+ throw new InterpreterException(e);
+ } catch (IllegalAccessException e) {
+ throw new InterpreterException(e);
+ } catch (InvocationTargetException e) {
+ throw new InterpreterException(e);
+ } catch (ClassNotFoundException e) {
+ throw new InterpreterException(e);
+ } finally {
+ Thread.currentThread().setContextClassLoader(oldcl);
+ }
+ }
+
+ RemoteInterpreterProcess createInterpreterProcess() {
+ RemoteInterpreterProcess remoteInterpreterProcess = null;
+ int connectTimeout =
+ conf.getInt(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
+ String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + id;
+ if (option.isExistingProcess()) {
+ // TODO(zjffdu) remove the existing process approach seems no one is using this.
+ // use the existing process
+ remoteInterpreterProcess = new RemoteInterpreterRunningProcess(
+ connectTimeout,
+ remoteInterpreterProcessListener,
+ appEventListener,
+ option.getHost(),
+ option.getPort());
+ } else {
+ // create new remote process
+ remoteInterpreterProcess = new RemoteInterpreterManagedProcess(
+ interpreterRunner != null ? interpreterRunner.getPath() :
+ conf.getInterpreterRemoteRunnerPath(), interpreterDir, localRepoPath,
+ getEnvFromInterpreterProperty(getJavaProperties()), connectTimeout,
+ remoteInterpreterProcessListener, appEventListener, group);
+ }
+ return remoteInterpreterProcess;
+ }
+
+ private Map<String, String> getEnvFromInterpreterProperty(Properties property) {
+ Map<String, String> env = new HashMap<String, String>();
+ StringBuilder sparkConfBuilder = new StringBuilder();
+ for (String key : property.stringPropertyNames()) {
+ if (RemoteInterpreterUtils.isEnvString(key)) {
+ env.put(key, property.getProperty(key));
+ }
+ if (key.equals("master")) {
+ sparkConfBuilder.append(" --master " + property.getProperty("master"));
+ }
+ if (isSparkConf(key, property.getProperty(key))) {
+ sparkConfBuilder.append(" --conf " + key + "=" +
+ toShellFormat(property.getProperty(key)));
+ }
+ }
+ env.put("ZEPPELIN_SPARK_CONF", sparkConfBuilder.toString());
+ return env;
+ }
+
+ private String toShellFormat(String value) {
+ if (value.contains("\'") && value.contains("\"")) {
+ throw new RuntimeException("Spark property value could not contain both \" and '");
+ } else if (value.contains("\'")) {
+ return "\"" + value + "\"";
+ } else {
+ return "\'" + value + "\'";
+ }
+ }
+
+ static boolean isSparkConf(String key, String value) {
+ return !StringUtils.isEmpty(key) && key.startsWith("spark.") && !StringUtils.isEmpty(value);
+ }
+
+ private List<Interpreter> getOrCreateSession(String user, String noteId) {
+ InterpreterGroup interpreterGroup = getOrCreateInterpreterGroup(user, noteId);
+ Preconditions.checkNotNull(interpreterGroup, "No InterpreterGroup existed for user {}, " +
+ "noteId {}", user, noteId);
+ String sessionId = getInterpreterSessionId(user, noteId);
+ return interpreterGroup.getOrCreateSession(user, sessionId);
+ }
+
+ public Interpreter getDefaultInterpreter(String user, String noteId) {
+ return getOrCreateSession(user, noteId).get(0);
+ }
+
+ public Interpreter getInterpreter(String user, String noteId, String replName) {
+ Preconditions.checkNotNull(noteId, "noteId should be not null");
+ Preconditions.checkNotNull(replName, "replName should be not null");
+
+ String className = getInterpreterClassFromInterpreterSetting(replName);
+ if (className == null) {
+ return null;
+ }
+ List<Interpreter> interpreters = getOrCreateSession(user, noteId);
+ for (Interpreter interpreter : interpreters) {
+ if (className.equals(interpreter.getClassName())) {
+ return interpreter;
+ }
+ }
+ return null;
+ }
+
+ private String getInterpreterClassFromInterpreterSetting(String replName) {
+ Preconditions.checkNotNull(replName, "replName should be not null");
+
+ for (InterpreterInfo info : interpreterInfos) {
+ String infoName = info.getName();
+ if (null != info.getName() && replName.equals(infoName)) {
+ return info.getClassName();
+ }
+ }
+ return null;
+ }
+
+ private InterpreterGroup createInterpreterGroup(String groupId) throws InterpreterException {
+ AngularObjectRegistry angularObjectRegistry;
+ InterpreterGroup interpreterGroup = new InterpreterGroup(groupId, this);
+ if (option.isRemote()) {
+ angularObjectRegistry =
+ new RemoteAngularObjectRegistry(groupId, angularObjectRegistryListener, interpreterGroup);
+ } else {
+ angularObjectRegistry = new AngularObjectRegistry(id, angularObjectRegistryListener);
+ // TODO(moon) : create distributed resource pool for local interpreters and set
+ }
+
+ interpreterGroup.setAngularObjectRegistry(angularObjectRegistry);
+ return interpreterGroup;
+ }
+
+ private void loadInterpreterDependencies() {
+ setStatus(Status.DOWNLOADING_DEPENDENCIES);
+ setErrorReason(null);
+ Thread t = new Thread() {
+ public void run() {
+ try {
+ // dependencies to prevent library conflict
+ File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + getId());
+ if (localRepoDir.exists()) {
+ try {
+ FileUtils.forceDelete(localRepoDir);
+ } catch (FileNotFoundException e) {
+ LOGGER.info("A file that does not exist cannot be deleted, nothing to worry", e);
+ }
+ }
+
+ // load dependencies
+ List<Dependency> deps = getDependencies();
+ if (deps != null) {
+ for (Dependency d : deps) {
+ File destDir = new File(
+ conf.getRelativeDir(ZeppelinConfiguration.ConfVars.ZEPPELIN_DEP_LOCALREPO));
+
+ if (d.getExclusions() != null) {
+ dependencyResolver.load(d.getGroupArtifactVersion(), d.getExclusions(),
+ new File(destDir, id));
+ } else {
+ dependencyResolver
+ .load(d.getGroupArtifactVersion(), new File(destDir, id));
+ }
+ }
+ }
+
+ setStatus(Status.READY);
+ setErrorReason(null);
+ } catch (Exception e) {
+ LOGGER.error(String.format("Error while downloading repos for interpreter group : %s," +
+ " go to interpreter setting page click on edit and save it again to make " +
+ "this interpreter work properly. : %s",
+ getGroup(), e.getLocalizedMessage()), e);
+ setErrorReason(e.getLocalizedMessage());
+ setStatus(Status.ERROR);
+ }
+ }
+ };
+
+ t.start();
+ }
+
+ //TODO(zjffdu) ugly code, should not use JsonObject as parameter. not readable
+ public void convertPermissionsFromUsersToOwners(JsonObject jsonObject) {
+ if (jsonObject != null) {
+ JsonObject option = jsonObject.getAsJsonObject("option");
+ if (option != null) {
+ JsonArray users = option.getAsJsonArray("users");
+ if (users != null) {
+ if (this.option.getOwners() == null) {
+ this.option.owners = new LinkedList<>();
+ }
+ for (JsonElement user : users) {
+ this.option.getOwners().add(user.getAsString());
+ }
+ }
+ }
+ }
+ }
+
+ // For backward compatibility of interpreter.json format after ZEPPELIN-2403
+ static Map<String, InterpreterProperty> convertInterpreterProperties(Object properties) {
+ if (properties != null && properties instanceof StringMap) {
+ Map<String, InterpreterProperty> newProperties = new HashMap<>();
+ StringMap p = (StringMap) properties;
+ for (Object o : p.entrySet()) {
+ Map.Entry entry = (Map.Entry) o;
+ if (!(entry.getValue() instanceof StringMap)) {
+ InterpreterProperty newProperty = new InterpreterProperty(
+ entry.getKey().toString(),
+ entry.getValue(),
+ InterpreterPropertyType.STRING.getValue());
+ newProperties.put(entry.getKey().toString(), newProperty);
+ } else {
+ // already converted
+ return (Map<String, InterpreterProperty>) properties;
+ }
+ }
+ return newProperties;
+
+ } else if (properties instanceof Map) {
+ Map<String, Object> dProperties =
+ (Map<String, Object>) properties;
+ Map<String, InterpreterProperty> newProperties = new HashMap<>();
+ for (String key : dProperties.keySet()) {
+ Object value = dProperties.get(key);
+ if (value instanceof InterpreterProperty) {
+ return (Map<String, InterpreterProperty>) properties;
+ } else if (value instanceof StringMap) {
+ StringMap stringMap = (StringMap) value;
+ InterpreterProperty newProperty = new InterpreterProperty(
+ key,
+ stringMap.get("value"),
+ stringMap.get("type").toString());
+
+ newProperties.put(newProperty.getName(), newProperty);
+ } else if (value instanceof DefaultInterpreterProperty){
+ DefaultInterpreterProperty dProperty = (DefaultInterpreterProperty) value;
+ InterpreterProperty property = new InterpreterProperty(
+ key,
+ dProperty.getValue(),
+ dProperty.getType() != null ? dProperty.getType() : "string"
+ // in case user forget to specify type in interpreter-setting.json
+ );
+ newProperties.put(key, property);
+ } else {
+ throw new RuntimeException("Can not convert this type of property: " + value.getClass());
+ }
+ }
+ return newProperties;
+ }
+ throw new RuntimeException("Can not convert this type: " + properties.getClass());
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
new file mode 100644
index 0000000..ed3ebd8
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
@@ -0,0 +1,886 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.reflect.TypeToken;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.dep.Dependency;
+import org.apache.zeppelin.dep.DependencyResolver;
+import org.apache.zeppelin.display.AngularObjectRegistryListener;
+import org.apache.zeppelin.helium.ApplicationEventListener;
+import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
+import org.apache.zeppelin.resource.Resource;
+import org.apache.zeppelin.resource.ResourcePool;
+import org.apache.zeppelin.resource.ResourceSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonatype.aether.repository.Authentication;
+import org.sonatype.aether.repository.Proxy;
+import org.sonatype.aether.repository.RemoteRepository;
+
+import java.io.*;
+import java.lang.reflect.Type;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.file.DirectoryStream.Filter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+
+/**
+ * InterpreterSettingManager is the component which manage all the interpreter settings.
+ * (load/create/update/remove/get)
+ * Besides that InterpreterSettingManager also manage the interpreter setting binding.
+ * TODO(zjffdu) We could move it into another separated component.
+ */
+public class InterpreterSettingManager {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterSettingManager.class);
+ private static final Map<String, Object> DEFAULT_EDITOR = ImmutableMap.of(
+ "language", (Object) "text",
+ "editOnDblClick", false);
+
+ private final ZeppelinConfiguration conf;
+ private final Path interpreterDirPath;
+ private final Path interpreterSettingPath;
+
+ /**
+ * This is only InterpreterSetting templates with default name and properties
+ * name --> InterpreterSetting
+ */
+ private final Map<String, InterpreterSetting> interpreterSettingTemplates =
+ Maps.newConcurrentMap();
+ /**
+ * This is used by creating and running Interpreters
+ * id --> InterpreterSetting
+ * TODO(zjffdu) change it to name --> InterpreterSetting
+ */
+ private final Map<String, InterpreterSetting> interpreterSettings =
+ Maps.newConcurrentMap();
+
+ /**
+ * noteId --> list of InterpreterSettingId
+ */
+ private final Map<String, List<String>> interpreterBindings =
+ Maps.newConcurrentMap();
+
+ private final List<RemoteRepository> interpreterRepositories;
+ private InterpreterOption defaultOption;
+ private List<String> interpreterGroupOrderList;
+ private final Gson gson;
+
+ private AngularObjectRegistryListener angularObjectRegistryListener;
+ private RemoteInterpreterProcessListener remoteInterpreterProcessListener;
+ private ApplicationEventListener appEventListener;
+ private DependencyResolver dependencyResolver;
+
+
+ public InterpreterSettingManager(ZeppelinConfiguration zeppelinConfiguration,
+ AngularObjectRegistryListener angularObjectRegistryListener,
+ RemoteInterpreterProcessListener
+ remoteInterpreterProcessListener,
+ ApplicationEventListener appEventListener)
+ throws IOException {
+ this(zeppelinConfiguration, new InterpreterOption(true),
+ angularObjectRegistryListener,
+ remoteInterpreterProcessListener,
+ appEventListener);
+ }
+
+ @VisibleForTesting
+ public InterpreterSettingManager(ZeppelinConfiguration conf,
+ InterpreterOption defaultOption,
+ AngularObjectRegistryListener angularObjectRegistryListener,
+ RemoteInterpreterProcessListener
+ remoteInterpreterProcessListener,
+ ApplicationEventListener appEventListener) throws IOException {
+ this.conf = conf;
+ this.defaultOption = defaultOption;
+ this.interpreterDirPath = Paths.get(conf.getInterpreterDir());
+ LOGGER.debug("InterpreterRootPath: {}", interpreterDirPath);
+ this.interpreterSettingPath = Paths.get(conf.getInterpreterSettingPath());
+ LOGGER.debug("InterpreterBindingPath: {}", interpreterSettingPath);
+ this.dependencyResolver = new DependencyResolver(
+ conf.getString(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO));
+ this.interpreterRepositories = dependencyResolver.getRepos();
+ this.interpreterGroupOrderList = Arrays.asList(conf.getString(
+ ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER).split(","));
+ this.gson = new GsonBuilder().setPrettyPrinting().create();
+
+ this.angularObjectRegistryListener = angularObjectRegistryListener;
+ this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
+ this.appEventListener = appEventListener;
+ init();
+ }
+
+ /**
+ * Load interpreter setting from interpreter-setting.json
+ */
+ private void loadFromFile() {
+ if (!Files.exists(interpreterSettingPath)) {
+ // nothing to read
+ LOGGER.warn("Interpreter Setting file {} doesn't exist", interpreterSettingPath);
+ return;
+ }
+
+ try {
+ InterpreterInfoSaving infoSaving = InterpreterInfoSaving.loadFromFile(interpreterSettingPath);
+ //TODO(zjffdu) still ugly (should move all to InterpreterInfoSaving)
+ for (InterpreterSetting savedInterpreterSetting : infoSaving.interpreterSettings.values()) {
+ savedInterpreterSetting.setConf(conf);
+ savedInterpreterSetting.setInterpreterSettingManager(this);
+ savedInterpreterSetting.setAngularObjectRegistryListener(angularObjectRegistryListener);
+ savedInterpreterSetting.setRemoteInterpreterProcessListener(
+ remoteInterpreterProcessListener);
+ savedInterpreterSetting.setAppEventListener(appEventListener);
+ savedInterpreterSetting.setDependencyResolver(dependencyResolver);
+ savedInterpreterSetting.setProperties(InterpreterSetting.convertInterpreterProperties(
+ savedInterpreterSetting.getProperties()
+ ));
+
+ InterpreterSetting interpreterSettingTemplate =
+ interpreterSettingTemplates.get(savedInterpreterSetting.getGroup());
+ // InterpreterSettingTemplate is from interpreter-setting.json which represent the latest
+ // InterpreterSetting, while InterpreterSetting is from interpreter.json which represent
+ // the user saved interpreter setting
+ if (interpreterSettingTemplate != null) {
+ savedInterpreterSetting.setInterpreterDir(interpreterSettingTemplate.getInterpreterDir());
+ // merge properties from interpreter-setting.json and interpreter.json
+ Map<String, InterpreterProperty> mergedProperties =
+ new HashMap<>(InterpreterSetting.convertInterpreterProperties(
+ interpreterSettingTemplate.getProperties()));
+ mergedProperties.putAll(InterpreterSetting.convertInterpreterProperties(
+ savedInterpreterSetting.getProperties()));
+ savedInterpreterSetting.setProperties(mergedProperties);
+ // merge InterpreterInfo
+ savedInterpreterSetting.setInterpreterInfos(
+ interpreterSettingTemplate.getInterpreterInfos());
+ } else {
+ LOGGER.warn("No InterpreterSetting Template found for InterpreterSetting: "
+ + savedInterpreterSetting.getGroup());
+ }
+
+ // Overwrite the default InterpreterSetting we registered from InterpreterSetting Templates
+ // remove it first
+ for (InterpreterSetting setting : interpreterSettings.values()) {
+ if (setting.getName().equals(savedInterpreterSetting.getName())) {
+ interpreterSettings.remove(setting.getId());
+ }
+ }
+ savedInterpreterSetting.postProcessing();
+ LOGGER.info("Create Interpreter Setting {} from interpreter.json",
+ savedInterpreterSetting.getName());
+ interpreterSettings.put(savedInterpreterSetting.getId(), savedInterpreterSetting);
+ }
+
+ interpreterBindings.putAll(infoSaving.interpreterBindings);
+
+ if (infoSaving.interpreterRepositories != null) {
+ for (RemoteRepository repo : infoSaving.interpreterRepositories) {
+ if (!dependencyResolver.getRepos().contains(repo)) {
+ this.interpreterRepositories.add(repo);
+ }
+ }
+ }
+ } catch (IOException e) {
+ LOGGER.error("Fail to load interpreter setting configuration file: "
+ + interpreterSettingPath, e);
+ }
+ }
+
+ public void saveToFile() throws IOException {
+ synchronized (interpreterSettings) {
+ InterpreterInfoSaving info = new InterpreterInfoSaving();
+ info.interpreterBindings = interpreterBindings;
+ info.interpreterSettings = interpreterSettings;
+ info.interpreterRepositories = interpreterRepositories;
+ info.saveToFile(interpreterSettingPath);
+ }
+ }
+
+ private void init() throws IOException {
+
+ // 1. detect interpreter setting via interpreter-setting.json in each interpreter folder
+ // 2. detect interpreter setting in interpreter.json that is saved before
+ String interpreterJson = conf.getInterpreterJson();
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ if (Files.exists(interpreterDirPath)) {
+ for (Path interpreterDir : Files
+ .newDirectoryStream(interpreterDirPath, new Filter<Path>() {
+ @Override
+ public boolean accept(Path entry) throws IOException {
+ return Files.exists(entry) && Files.isDirectory(entry);
+ }
+ })) {
+ String interpreterDirString = interpreterDir.toString();
+
+ /**
+ * Register interpreter by the following ordering
+ * 1. Register it from path {ZEPPELIN_HOME}/interpreter/{interpreter_name}/
+ * interpreter-setting.json
+ * 2. Register it from interpreter-setting.json in classpath
+ * {ZEPPELIN_HOME}/interpreter/{interpreter_name}
+ */
+ if (!registerInterpreterFromPath(interpreterDirString, interpreterJson)) {
+ if (!registerInterpreterFromResource(cl, interpreterDirString, interpreterJson)) {
+ LOGGER.warn("No interpreter-setting.json found in " + interpreterDirPath);
+ }
+ }
+ }
+ } else {
+ LOGGER.warn("InterpreterDir {} doesn't exist", interpreterDirPath);
+ }
+
+ loadFromFile();
+ saveToFile();
+ }
+
+ private boolean registerInterpreterFromResource(ClassLoader cl, String interpreterDir,
+ String interpreterJson) throws IOException {
+ URL[] urls = recursiveBuildLibList(new File(interpreterDir));
+ ClassLoader tempClassLoader = new URLClassLoader(urls, cl);
+
+ URL url = tempClassLoader.getResource(interpreterJson);
+ if (url == null) {
+ return false;
+ }
+
+ LOGGER.debug("Reading interpreter-setting.json from {} as Resource", url);
+ List<RegisteredInterpreter> registeredInterpreterList =
+ getInterpreterListFromJson(url.openStream());
+ registerInterpreterSetting(registeredInterpreterList, interpreterDir);
+ return true;
+ }
+
+ private boolean registerInterpreterFromPath(String interpreterDir, String interpreterJson)
+ throws IOException {
+
+ Path interpreterJsonPath = Paths.get(interpreterDir, interpreterJson);
+ if (Files.exists(interpreterJsonPath)) {
+ LOGGER.debug("Reading interpreter-setting.json from file {}", interpreterJsonPath);
+ List<RegisteredInterpreter> registeredInterpreterList =
+ getInterpreterListFromJson(new FileInputStream(interpreterJsonPath.toFile()));
+ registerInterpreterSetting(registeredInterpreterList, interpreterDir);
+ return true;
+ }
+ return false;
+ }
+
+ private List<RegisteredInterpreter> getInterpreterListFromJson(InputStream stream) {
+ Type registeredInterpreterListType = new TypeToken<List<RegisteredInterpreter>>() {
+ }.getType();
+ return gson.fromJson(new InputStreamReader(stream), registeredInterpreterListType);
+ }
+
+ private void registerInterpreterSetting(List<RegisteredInterpreter> registeredInterpreters,
+ String interpreterDir) throws IOException {
+
+ Map<String, DefaultInterpreterProperty> properties = new HashMap<>();
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ InterpreterOption option = defaultOption;
+ String group = null;
+ InterpreterRunner runner = null;
+ for (RegisteredInterpreter registeredInterpreter : registeredInterpreters) {
+ //TODO(zjffdu) merge RegisteredInterpreter & InterpreterInfo
+ InterpreterInfo interpreterInfo =
+ new InterpreterInfo(registeredInterpreter.getClassName(), registeredInterpreter.getName(),
+ registeredInterpreter.isDefaultInterpreter(), registeredInterpreter.getEditor());
+ group = registeredInterpreter.getGroup();
+ runner = registeredInterpreter.getRunner();
+ // use defaultOption if it is not specified in interpreter-setting.json
+ if (registeredInterpreter.getOption() != null) {
+ option = registeredInterpreter.getOption();
+ }
+ properties.putAll(registeredInterpreter.getProperties());
+ interpreterInfos.add(interpreterInfo);
+ }
+
+ InterpreterSetting interpreterSettingTemplate = new InterpreterSetting.Builder()
+ .setGroup(group)
+ .setName(group)
+ .setInterpreterInfos(interpreterInfos)
+ .setProperties(properties)
+ .setDependencies(new ArrayList<Dependency>())
+ .setOption(option)
+ .setRunner(runner)
+ .setInterpreterDir(interpreterDir)
+ .setRunner(runner)
+ .setConf(conf)
+ .setIntepreterSettingManager(this)
+ .create();
+
+ LOGGER.info("Register InterpreterSettingTemplate & InterpreterSetting: {}",
+ interpreterSettingTemplate.getName());
+ interpreterSettingTemplates.put(interpreterSettingTemplate.getName(),
+ interpreterSettingTemplate);
+
+ InterpreterSetting interpreterSetting = new InterpreterSetting(interpreterSettingTemplate);
+ interpreterSetting.setAngularObjectRegistryListener(angularObjectRegistryListener);
+ interpreterSetting.setRemoteInterpreterProcessListener(remoteInterpreterProcessListener);
+ interpreterSetting.setAppEventListener(appEventListener);
+ interpreterSetting.setDependencyResolver(dependencyResolver);
+ interpreterSetting.setInterpreterSettingManager(this);
+ interpreterSetting.postProcessing();
+ interpreterSettings.put(interpreterSetting.getId(), interpreterSetting);
+ }
+
+ @VisibleForTesting
+ public InterpreterSetting getDefaultInterpreterSetting(String noteId) {
+ return getInterpreterSettings(noteId).get(0);
+ }
+
+ public List<InterpreterSetting> getInterpreterSettings(String noteId) {
+ List<InterpreterSetting> settings = new ArrayList<>();
+ synchronized (interpreterSettings) {
+ List<String> interpreterSettingIds = interpreterBindings.get(noteId);
+ if (interpreterSettingIds != null) {
+ for (String settingId : interpreterSettingIds) {
+ if (interpreterSettings.containsKey(settingId)) {
+ settings.add(interpreterSettings.get(settingId));
+ } else {
+ LOGGER.warn("InterpreterSetting {} has been removed, but note {} still bind to it.",
+ settingId, noteId);
+ }
+ }
+ }
+ }
+ return settings;
+ }
+
+ public InterpreterGroup getInterpreterGroupById(String groupId) {
+ for (InterpreterSetting setting : interpreterSettings.values()) {
+ InterpreterGroup interpreterGroup = setting.getInterpreterGroup(groupId);
+ if (interpreterGroup != null) {
+ return interpreterGroup;
+ }
+ }
+ return null;
+ }
+
+ //TODO(zjffdu) logic here is a little ugly
+ public Map<String, Object> getEditorSetting(Interpreter interpreter, String user, String noteId,
+ String replName) {
+ Map<String, Object> editor = DEFAULT_EDITOR;
+ String group = StringUtils.EMPTY;
+ try {
+ String defaultSettingName = getDefaultInterpreterSetting(noteId).getName();
+ List<InterpreterSetting> intpSettings = getInterpreterSettings(noteId);
+ for (InterpreterSetting intpSetting : intpSettings) {
+ String[] replNameSplit = replName.split("\\.");
+ if (replNameSplit.length == 2) {
+ group = replNameSplit[0];
+ }
+ // when replName is 'name' of interpreter
+ if (defaultSettingName.equals(intpSetting.getName())) {
+ editor = intpSetting.getEditorFromSettingByClassName(interpreter.getClassName());
+ }
+ // when replName is 'alias name' of interpreter or 'group' of interpreter
+ if (replName.equals(intpSetting.getName()) || group.equals(intpSetting.getName())) {
+ editor = intpSetting.getEditorFromSettingByClassName(interpreter.getClassName());
+ break;
+ }
+ }
+ } catch (NullPointerException e) {
+ // Use `debug` level because this log occurs frequently
+ LOGGER.debug("Couldn't get interpreter editor setting");
+ }
+ return editor;
+ }
+
+ public List<InterpreterGroup> getAllInterpreterGroup() {
+ List<InterpreterGroup> interpreterGroups = new ArrayList<>();
+ for (InterpreterSetting interpreterSetting : interpreterSettings.values()) {
+ interpreterGroups.addAll(interpreterSetting.getAllInterpreterGroups());
+ }
+ return interpreterGroups;
+ }
+
+ //TODO(zjffdu) move Resource related api to ResourceManager
+ public ResourceSet getAllResources() {
+ return getAllResourcesExcept(null);
+ }
+
+ private ResourceSet getAllResourcesExcept(String interpreterGroupExcludsion) {
+ ResourceSet resourceSet = new ResourceSet();
+ for (InterpreterGroup intpGroup : getAllInterpreterGroup()) {
+ if (interpreterGroupExcludsion != null &&
+ intpGroup.getId().equals(interpreterGroupExcludsion)) {
+ continue;
+ }
+
+ RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
+ if (remoteInterpreterProcess == null) {
+ ResourcePool localPool = intpGroup.getResourcePool();
+ if (localPool != null) {
+ resourceSet.addAll(localPool.getAll());
+ }
+ } else if (remoteInterpreterProcess.isRunning()) {
+ List<String> resourceList = remoteInterpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<List<String>>() {
+ @Override
+ public List<String> call(RemoteInterpreterService.Client client) throws Exception {
+ return client.resourcePoolGetAll();
+ }
+ });
+ for (String res : resourceList) {
+ resourceSet.add(Resource.fromJson(res));
+ }
+ }
+ }
+ return resourceSet;
+ }
+
+ public void removeResourcesBelongsToParagraph(String noteId, String paragraphId) {
+ for (InterpreterGroup intpGroup : getAllInterpreterGroup()) {
+ ResourceSet resourceSet = new ResourceSet();
+ RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
+ if (remoteInterpreterProcess == null) {
+ ResourcePool localPool = intpGroup.getResourcePool();
+ if (localPool != null) {
+ resourceSet.addAll(localPool.getAll());
+ }
+ if (noteId != null) {
+ resourceSet = resourceSet.filterByNoteId(noteId);
+ }
+ if (paragraphId != null) {
+ resourceSet = resourceSet.filterByParagraphId(paragraphId);
+ }
+
+ for (Resource r : resourceSet) {
+ localPool.remove(
+ r.getResourceId().getNoteId(),
+ r.getResourceId().getParagraphId(),
+ r.getResourceId().getName());
+ }
+ } else if (remoteInterpreterProcess.isRunning()) {
+ List<String> resourceList = remoteInterpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<List<String>>() {
+ @Override
+ public List<String> call(RemoteInterpreterService.Client client) throws Exception {
+ return client.resourcePoolGetAll();
+ }
+ });
+ for (String res : resourceList) {
+ resourceSet.add(Resource.fromJson(res));
+ }
+
+ if (noteId != null) {
+ resourceSet = resourceSet.filterByNoteId(noteId);
+ }
+ if (paragraphId != null) {
+ resourceSet = resourceSet.filterByParagraphId(paragraphId);
+ }
+
+ for (final Resource r : resourceSet) {
+ remoteInterpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<Void>() {
+
+ @Override
+ public Void call(RemoteInterpreterService.Client client) throws Exception {
+ client.resourceRemove(
+ r.getResourceId().getNoteId(),
+ r.getResourceId().getParagraphId(),
+ r.getResourceId().getName());
+ return null;
+ }
+ });
+ }
+ }
+ }
+ }
+
+ public void removeResourcesBelongsToNote(String noteId) {
+ removeResourcesBelongsToParagraph(noteId, null);
+ }
+
+ /**
+ * Overwrite dependency jar under local-repo/{interpreterId}
+ * if jar file in original path is changed
+ */
+ private void copyDependenciesFromLocalPath(final InterpreterSetting setting) {
+ setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
+ synchronized (interpreterSettings) {
+ final Thread t = new Thread() {
+ public void run() {
+ try {
+ List<Dependency> deps = setting.getDependencies();
+ if (deps != null) {
+ for (Dependency d : deps) {
+ File destDir = new File(
+ conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
+
+ int numSplits = d.getGroupArtifactVersion().split(":").length;
+ if (!(numSplits >= 3 && numSplits <= 6)) {
+ dependencyResolver.copyLocalDependency(d.getGroupArtifactVersion(),
+ new File(destDir, setting.getId()));
+ }
+ }
+ }
+ setting.setStatus(InterpreterSetting.Status.READY);
+ } catch (Exception e) {
+ LOGGER.error(String.format("Error while copying deps for interpreter group : %s," +
+ " go to interpreter setting page click on edit and save it again to make " +
+ "this interpreter work properly.",
+ setting.getGroup()), e);
+ setting.setErrorReason(e.getLocalizedMessage());
+ setting.setStatus(InterpreterSetting.Status.ERROR);
+ } finally {
+
+ }
+ }
+ };
+ t.start();
+ }
+ }
+
+ /**
+ * Return ordered interpreter setting list.
+ * The list does not contain more than one setting from the same interpreter class.
+ * Order by InterpreterClass (order defined by ZEPPELIN_INTERPRETERS), Interpreter setting name
+ */
+ public List<String> getInterpreterSettingIds() {
+ List<String> settingIdList = new ArrayList<>();
+ for (InterpreterSetting interpreterSetting : get()) {
+ settingIdList.add(interpreterSetting.getId());
+ }
+ return settingIdList;
+ }
+
+ public InterpreterSetting createNewSetting(String name, String group,
+ List<Dependency> dependencies, InterpreterOption option, Map<String, InterpreterProperty> p)
+ throws IOException {
+
+ if (name.indexOf(".") >= 0) {
+ throw new IOException("'.' is invalid for InterpreterSetting name.");
+ }
+ // check if name is existed
+ for (InterpreterSetting interpreterSetting : interpreterSettings.values()) {
+ if (interpreterSetting.getName().equals(name)) {
+ throw new IOException("Interpreter " + name + " already existed");
+ }
+ }
+ InterpreterSetting setting = new InterpreterSetting(interpreterSettingTemplates.get(group));
+ setting.setName(name);
+ setting.setGroup(group);
+ //TODO(zjffdu) Should use setDependencies
+ setting.appendDependencies(dependencies);
+ setting.setInterpreterOption(option);
+ setting.setProperties(p);
+ setting.setAppEventListener(appEventListener);
+ setting.setRemoteInterpreterProcessListener(remoteInterpreterProcessListener);
+ setting.setDependencyResolver(dependencyResolver);
+ setting.setAngularObjectRegistryListener(angularObjectRegistryListener);
+ setting.setInterpreterSettingManager(this);
+ setting.postProcessing();
+ interpreterSettings.put(setting.getId(), setting);
+ saveToFile();
+ return setting;
+ }
+
+ @VisibleForTesting
+ public void addInterpreterSetting(InterpreterSetting interpreterSetting) {
+ interpreterSettingTemplates.put(interpreterSetting.getName(), interpreterSetting);
+ interpreterSetting.setAppEventListener(appEventListener);
+ interpreterSetting.setDependencyResolver(dependencyResolver);
+ interpreterSetting.setAngularObjectRegistryListener(angularObjectRegistryListener);
+ interpreterSetting.setRemoteInterpreterProcessListener(remoteInterpreterProcessListener);
+ interpreterSetting.setInterpreterSettingManager(this);
+ interpreterSettings.put(interpreterSetting.getId(), interpreterSetting);
+ }
+
+ /**
+ * map interpreter ids into noteId
+ *
+ * @param user user name
+ * @param noteId note id
+ * @param settingIdList InterpreterSetting id list
+ */
+ public void setInterpreterBinding(String user, String noteId, List<String> settingIdList)
+ throws IOException {
+ List<String> unBindedSettingIdList = new LinkedList<>();
+
+ synchronized (interpreterSettings) {
+ List<String> oldSettingIdList = interpreterBindings.get(noteId);
+ if (oldSettingIdList != null) {
+ for (String oldSettingId : oldSettingIdList) {
+ if (!settingIdList.contains(oldSettingId)) {
+ unBindedSettingIdList.add(oldSettingId);
+ }
+ }
+ }
+ interpreterBindings.put(noteId, settingIdList);
+ saveToFile();
+
+ for (String settingId : unBindedSettingIdList) {
+ InterpreterSetting interpreterSetting = interpreterSettings.get(settingId);
+ //TODO(zjffdu) Add test for this scenario
+ //only close Interpreters when it is note scoped
+ if (interpreterSetting.getOption().perNoteIsolated() ||
+ interpreterSetting.getOption().perNoteScoped()) {
+ interpreterSetting.closeInterpreters(user, noteId);
+ }
+ }
+ }
+ }
+
+ public List<String> getInterpreterBinding(String noteId) {
+ return interpreterBindings.get(noteId);
+ }
+
+ @VisibleForTesting
+ public void closeNote(String user, String noteId) {
+ // close interpreters in this note session
+ LOGGER.info("Close Note: {}", noteId);
+ List<InterpreterSetting> settings = getInterpreterSettings(noteId);
+ for (InterpreterSetting setting : settings) {
+ setting.closeInterpreters(user, noteId);
+ }
+ }
+
+ public Map<String, InterpreterSetting> getInterpreterSettingTemplates() {
+ return interpreterSettingTemplates;
+ }
+
+ private URL[] recursiveBuildLibList(File path) throws MalformedURLException {
+ URL[] urls = new URL[0];
+ if (path == null || !path.exists()) {
+ return urls;
+ } else if (path.getName().startsWith(".")) {
+ return urls;
+ } else if (path.isDirectory()) {
+ File[] files = path.listFiles();
+ if (files != null) {
+ for (File f : files) {
+ urls = (URL[]) ArrayUtils.addAll(urls, recursiveBuildLibList(f));
+ }
+ }
+ return urls;
+ } else {
+ return new URL[]{path.toURI().toURL()};
+ }
+ }
+
+ public List<RemoteRepository> getRepositories() {
+ return this.interpreterRepositories;
+ }
+
+ public void addRepository(String id, String url, boolean snapshot, Authentication auth,
+ Proxy proxy) throws IOException {
+ dependencyResolver.addRepo(id, url, snapshot, auth, proxy);
+ saveToFile();
+ }
+
+ public void removeRepository(String id) throws IOException {
+ dependencyResolver.delRepo(id);
+ saveToFile();
+ }
+
+ public void removeNoteInterpreterSettingBinding(String user, String noteId) throws IOException {
+ setInterpreterBinding(user, noteId, new ArrayList<String>());
+ interpreterBindings.remove(noteId);
+ }
+
+ /**
+ * Change interpreter property and restart
+ */
+ public void setPropertyAndRestart(String id, InterpreterOption option,
+ Map<String, InterpreterProperty> properties,
+ List<Dependency> dependencies) throws IOException {
+ synchronized (interpreterSettings) {
+ InterpreterSetting intpSetting = interpreterSettings.get(id);
+ if (intpSetting != null) {
+ try {
+ intpSetting.close();
+ intpSetting.setOption(option);
+ intpSetting.setProperties(properties);
+ intpSetting.setDependencies(dependencies);
+ intpSetting.postProcessing();
+ saveToFile();
+ } catch (Exception e) {
+ loadFromFile();
+ throw e;
+ }
+ } else {
+ throw new InterpreterException("Interpreter setting id " + id + " not found");
+ }
+ }
+ }
+
+ // restart in note page
+ public void restart(String settingId, String noteId, String user) {
+ InterpreterSetting intpSetting = interpreterSettings.get(settingId);
+ Preconditions.checkNotNull(intpSetting);
+ synchronized (interpreterSettings) {
+ intpSetting = interpreterSettings.get(settingId);
+ // Check if dependency in specified path is changed
+ // If it did, overwrite old dependency jar with new one
+ if (intpSetting != null) {
+ //clean up metaInfos
+ intpSetting.setInfos(null);
+ copyDependenciesFromLocalPath(intpSetting);
+
+ if (user.equals("anonymous")) {
+ intpSetting.close();
+ } else {
+ intpSetting.closeInterpreters(user, noteId);
+ }
+
+ } else {
+ throw new InterpreterException("Interpreter setting id " + settingId + " not found");
+ }
+ }
+ }
+
+ public void restart(String id) {
+ restart(id, "", "anonymous");
+ }
+
+ public InterpreterSetting get(String id) {
+ synchronized (interpreterSettings) {
+ return interpreterSettings.get(id);
+ }
+ }
+
+ @VisibleForTesting
+ public InterpreterSetting getByName(String name) {
+ for (InterpreterSetting interpreterSetting : interpreterSettings.values()) {
+ if (interpreterSetting.getName().equals(name)) {
+ return interpreterSetting;
+ }
+ }
+ throw new RuntimeException("No InterpreterSetting: " + name);
+ }
+
+ public void remove(String id) throws IOException {
+ // 1. close interpreter groups of this interpreter setting
+ // 2. remove this interpreter setting
+ // 3. remove this interpreter setting from note binding
+ // 4. clean local repo directory
+ LOGGER.info("Remove interpreter setting: " + id);
+ synchronized (interpreterSettings) {
+ if (interpreterSettings.containsKey(id)) {
+
+ InterpreterSetting intp = interpreterSettings.get(id);
+ intp.close();
+ interpreterSettings.remove(id);
+ for (List<String> settings : interpreterBindings.values()) {
+ Iterator<String> it = settings.iterator();
+ while (it.hasNext()) {
+ String settingId = it.next();
+ if (settingId.equals(id)) {
+ it.remove();
+ }
+ }
+ }
+ saveToFile();
+ }
+ }
+
+ File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + id);
+ FileUtils.deleteDirectory(localRepoDir);
+ }
+
+ /**
+ * Get interpreter settings
+ */
+ public List<InterpreterSetting> get() {
+ synchronized (interpreterSettings) {
+ List<InterpreterSetting> orderedSettings = new ArrayList<>(interpreterSettings.values());
+ Collections.sort(orderedSettings, new Comparator<InterpreterSetting>() {
+ @Override
+ public int compare(InterpreterSetting o1, InterpreterSetting o2) {
+ int i = interpreterGroupOrderList.indexOf(o1.getGroup());
+ int j = interpreterGroupOrderList.indexOf(o2.getGroup());
+ if (i < 0) {
+ LOGGER.warn("InterpreterGroup " + o1.getGroup()
+ + " is not specified in " + ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName());
+ // move the unknown interpreter to last
+ i = Integer.MAX_VALUE;
+ }
+ if (j < 0) {
+ LOGGER.warn("InterpreterGroup " + o2.getGroup()
+ + " is not specified in " + ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName());
+ // move the unknown interpreter to last
+ j = Integer.MAX_VALUE;
+ }
+ if (i < j) {
+ return -1;
+ } else if (i > j) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+ });
+ return orderedSettings;
+ }
+ }
+
+ @VisibleForTesting
+ public List<String> getSettingIds() {
+ List<String> settingIds = new ArrayList<>();
+ for (InterpreterSetting interpreterSetting : get()) {
+ settingIds.add(interpreterSetting.getId());
+ }
+ return settingIds;
+ }
+
+ public void close(String settingId) {
+ get(settingId).close();
+ }
+
+ public void close() {
+ List<Thread> closeThreads = new LinkedList<>();
+ synchronized (interpreterSettings) {
+ Collection<InterpreterSetting> intpSettings = interpreterSettings.values();
+ for (final InterpreterSetting intpSetting : intpSettings) {
+ Thread t = new Thread() {
+ public void run() {
+ intpSetting.close();
+ }
+ };
+ t.start();
+ closeThreads.add(t);
+ }
+ }
+
+ for (Thread t : closeThreads) {
+ try {
+ t.join();
+ } catch (InterruptedException e) {
+ LOGGER.error("Can't close interpreterGroup", e);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java
new file mode 100644
index 0000000..0817595
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.interpreter.install;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.dep.DependencyResolver;
+import org.apache.zeppelin.util.Util;
+import org.sonatype.aether.RepositoryException;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Commandline utility to install interpreter from maven repository
+ */
+public class InstallInterpreter {
+ private final File interpreterListFile;
+ private final File interpreterBaseDir;
+ private final List<AvailableInterpreterInfo> availableInterpreters;
+ private final String localRepoDir;
+ private URL proxyUrl;
+ private String proxyUser;
+ private String proxyPassword;
+
+ /**
+ *
+ * @param interpreterListFile
+ * @param interpreterBaseDir interpreter directory for installing binaries
+ * @throws IOException
+ */
+ public InstallInterpreter(File interpreterListFile, File interpreterBaseDir, String localRepoDir)
+ throws IOException {
+ this.interpreterListFile = interpreterListFile;
+ this.interpreterBaseDir = interpreterBaseDir;
+ this.localRepoDir = localRepoDir;
+ availableInterpreters = new LinkedList<>();
+ readAvailableInterpreters();
+ }
+
+
+ /**
+ * Information for available informations
+ */
+ private static class AvailableInterpreterInfo {
+ public final String name;
+ public final String artifact;
+ public final String description;
+
+ public AvailableInterpreterInfo(String name, String artifact, String description) {
+ this.name = name;
+ this.artifact = artifact;
+ this.description = description;
+ }
+ }
+
+ private void readAvailableInterpreters() throws IOException {
+ if (!interpreterListFile.isFile()) {
+ System.err.println("Can't find interpreter list " + interpreterListFile.getAbsolutePath());
+ return;
+ }
+ String text = FileUtils.readFileToString(interpreterListFile);
+ String[] lines = text.split("\n");
+
+ Pattern pattern = Pattern.compile("(\\S+)\\s+(\\S+)\\s+(.*)");
+
+ int lineNo = 0;
+ for (String line : lines) {
+ lineNo++;
+ if (line == null || line.length() == 0 || line.startsWith("#")) {
+ continue;
+ }
+
+ Matcher match = pattern.matcher(line);
+ if (match.groupCount() != 3) {
+ System.err.println("Error on line " + lineNo + ", " + line);
+ continue;
+ }
+
+ match.find();
+
+ String name = match.group(1);
+ String artifact = match.group(2);
+ String description = match.group(3);
+
+ availableInterpreters.add(new AvailableInterpreterInfo(name, artifact, description));
+ }
+ }
+
+ public List<AvailableInterpreterInfo> list() {
+ for (AvailableInterpreterInfo info : availableInterpreters) {
+ System.out.println(info.name + "\t\t\t" + info.description);
+ }
+
+ return availableInterpreters;
+ }
+
+ public void installAll() {
+ for (AvailableInterpreterInfo info : availableInterpreters) {
+ install(info.name, info.artifact);
+ }
+ }
+
+ public void install(String [] names) {
+ for (String name : names) {
+ install(name);
+ }
+ }
+
+ public void install(String name) {
+ // find artifact name
+ for (AvailableInterpreterInfo info : availableInterpreters) {
+ if (name.equals(info.name)) {
+ install(name, info.artifact);
+ return;
+ }
+ }
+
+ throw new RuntimeException("Can't find interpreter '" + name + "'");
+ }
+
+ public void install(String [] names, String [] artifacts) {
+ if (names.length != artifacts.length) {
+ throw new RuntimeException("Length of given names and artifacts are different");
+ }
+
+ for (int i = 0; i < names.length; i++) {
+ install(names[i], artifacts[i]);
+ }
+ }
+
+ public void install(String name, String artifact) {
+ DependencyResolver depResolver = new DependencyResolver(localRepoDir);
+ if (proxyUrl != null) {
+ depResolver.setProxy(proxyUrl, proxyUser, proxyPassword);
+ }
+
+ File installDir = new File(interpreterBaseDir, name);
+ if (installDir.exists()) {
+ System.err.println("Directory " + installDir.getAbsolutePath()
+ + " already exists"
+ + "\n\nSkipped");
+ return;
+ }
+
+ System.out.println("Install " + name + "(" + artifact + ") to "
+ + installDir.getAbsolutePath() + " ... ");
+
+ try {
+ depResolver.load(artifact, installDir);
+ System.out.println("Interpreter " + name + " installed under " +
+ installDir.getAbsolutePath() + ".");
+ startTip();
+ } catch (RepositoryException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public void setProxy(URL proxyUrl, String proxyUser, String proxyPassword) {
+ this.proxyUrl = proxyUrl;
+ this.proxyUser = proxyUser;
+ this.proxyPassword = proxyPassword;
+ }
+
+ public static void usage() {
+ System.out.println("Options");
+ System.out.println(" -l, --list List available interpreters");
+ System.out.println(" -a, --all Install all available interpreters");
+ System.out.println(" -n, --name [NAMES] Install interpreters (comma separated " +
+ "list)" +
+ "e.g. md,shell,jdbc,python,angular");
+ System.out.println(" -t, --artifact [ARTIFACTS] (Optional with -n) custom artifact names" +
+ ". " +
+ "(comma separated list correspond to --name) " +
+ "e.g. customGroup:customArtifact:customVersion");
+ System.out.println(" --proxy-url [url] (Optional) proxy url. http(s)://host:port");
+ System.out.println(" --proxy-user [user] (Optional) proxy user");
+ System.out.println(" --proxy-password [password] (Optional) proxy password");
+ }
+
+ public static void main(String [] args) throws IOException {
+ if (args.length == 0) {
+ usage();
+ return;
+ }
+
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create();
+ InstallInterpreter installer = new InstallInterpreter(
+ new File(conf.getInterpreterListPath()),
+ new File(conf.getInterpreterDir()),
+ conf.getInterpreterLocalRepoPath());
+
+ String names = null;
+ String artifacts = null;
+ URL proxyUrl = null;
+ String proxyUser = null;
+ String proxyPassword = null;
+ boolean all = false;
+
+ for (int i = 0; i < args.length; i++) {
+ String arg = args[i].toLowerCase(Locale.US);
+ switch (arg) {
+ case "--list":
+ case "-l":
+ installer.list();
+ System.exit(0);
+ break;
+ case "--all":
+ case "-a":
+ all = true;
+ break;
+ case "--name":
+ case "-n":
+ names = args[++i];
+ break;
+ case "--artifact":
+ case "-t":
+ artifacts = args[++i];
+ break;
+ case "--version":
+ case "-v":
+ Util.getVersion();
+ break;
+ case "--proxy-url":
+ proxyUrl = new URL(args[++i]);
+ break;
+ case "--proxy-user":
+ proxyUser = args[++i];
+ break;
+ case "--proxy-password":
+ proxyPassword = args[++i];
+ break;
+ case "--help":
+ case "-h":
+ usage();
+ System.exit(0);
+ break;
+ default:
+ System.out.println("Unknown option " + arg);
+ }
+ }
+
+ if (proxyUrl != null) {
+ installer.setProxy(proxyUrl, proxyUser, proxyPassword);
+ }
+
+ if (all) {
+ installer.installAll();
+ System.exit(0);
+ }
+
+ if (names != null) {
+ if (artifacts != null) {
+ installer.install(names.split(","), artifacts.split(","));
+ } else {
+ installer.install(names.split(","));
+ }
+ }
+ }
+
+ private static void startTip() {
+ System.out.println("\n1. Restart Zeppelin"
+ + "\n2. Create interpreter setting in 'Interpreter' menu on Zeppelin GUI"
+ + "\n3. Then you can bind the interpreter on your note");
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java
new file mode 100644
index 0000000..74a2da2
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote;
+
+import com.google.gson.Gson;
+import org.apache.thrift.TException;
+import org.apache.zeppelin.display.AngularObject;
+import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.display.AngularObjectRegistryListener;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+/**
+ * Proxy for AngularObjectRegistry that exists in remote interpreter process
+ */
+public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
+ Logger logger = LoggerFactory.getLogger(RemoteAngularObjectRegistry.class);
+ private InterpreterGroup interpreterGroup;
+
+ public RemoteAngularObjectRegistry(String interpreterId,
+ AngularObjectRegistryListener listener,
+ InterpreterGroup interpreterGroup) {
+ super(interpreterId, listener);
+ this.interpreterGroup = interpreterGroup;
+ }
+
+ private RemoteInterpreterProcess getRemoteInterpreterProcess() {
+ return interpreterGroup.getRemoteInterpreterProcess();
+ }
+
+ /**
+ * When ZeppelinServer side code want to add angularObject to the registry,
+ * this method should be used instead of add()
+ * @param name
+ * @param o
+ * @param noteId
+ * @return
+ */
+ public AngularObject addAndNotifyRemoteProcess(final String name,
+ final Object o,
+ final String noteId,
+ final String paragraphId) {
+
+ RemoteInterpreterProcess remoteInterpreterProcess = getRemoteInterpreterProcess();
+ if (!remoteInterpreterProcess.isRunning()) {
+ return super.add(name, o, noteId, paragraphId, true);
+ }
+
+ remoteInterpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ Gson gson = new Gson();
+ client.angularObjectAdd(name, noteId, paragraphId, gson.toJson(o));
+ return null;
+ }
+ }
+ );
+
+ return super.add(name, o, noteId, paragraphId, true);
+
+ }
+
+ /**
+ * When ZeppelinServer side code want to remove angularObject from the registry,
+ * this method should be used instead of remove()
+ * @param name
+ * @param noteId
+ * @param paragraphId
+ * @return
+ */
+ public AngularObject removeAndNotifyRemoteProcess(final String name,
+ final String noteId,
+ final String paragraphId) {
+ RemoteInterpreterProcess remoteInterpreterProcess = getRemoteInterpreterProcess();
+ if (remoteInterpreterProcess == null || !remoteInterpreterProcess.isRunning()) {
+ return super.remove(name, noteId, paragraphId);
+ }
+ remoteInterpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ client.angularObjectRemove(name, noteId, paragraphId);
+ return null;
+ }
+ }
+ );
+
+ return super.remove(name, noteId, paragraphId);
+ }
+
+ public void removeAllAndNotifyRemoteProcess(String noteId, String paragraphId) {
+ List<AngularObject> all = getAll(noteId, paragraphId);
+ for (AngularObject ao : all) {
+ removeAndNotifyRemoteProcess(ao.getName(), noteId, paragraphId);
+ }
+ }
+
+ @Override
+ protected AngularObject createNewAngularObject(String name, Object o, String noteId, String
+ paragraphId) {
+ return new RemoteAngularObject(name, o, noteId, paragraphId, interpreterGroup,
+ getAngularObjectListener());
+ }
+}
[03/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java
deleted file mode 100644
index 1aab757..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java
+++ /dev/null
@@ -1,327 +0,0 @@
-package org.apache.zeppelin.interpreter;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-
-import org.junit.Test;
-
-import org.apache.zeppelin.dep.Dependency;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.mock;
-
-public class InterpreterSettingTest {
-
- @Test
- public void sharedModeCloseandRemoveInterpreterGroupTest() {
- InterpreterOption interpreterOption = new InterpreterOption();
- interpreterOption.setPerUser(InterpreterOption.SHARED);
- InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
-
- interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
- @Override
- public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
- InterpreterOption option) {
- return new InterpreterGroup(interpreterGroupId);
- }
- });
-
- Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList1 = new ArrayList<>();
- interpreterList1.add(mockInterpreter1);
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
-
- // This won't effect anything
- Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList2 = new ArrayList<>();
- interpreterList2.add(mockInterpreter2);
- interpreterGroup = interpreterSetting.getInterpreterGroup("user2", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user2", "note1"), interpreterList2);
-
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
-
- interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user2");
- assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
- }
-
- @Test
- public void perUserScopedModeCloseAndRemoveInterpreterGroupTest() {
- InterpreterOption interpreterOption = new InterpreterOption();
- interpreterOption.setPerUser(InterpreterOption.SCOPED);
- InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
-
- interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
- @Override
- public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
- InterpreterOption option) {
- return new InterpreterGroup(interpreterGroupId);
- }
- });
-
- Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList1 = new ArrayList<>();
- interpreterList1.add(mockInterpreter1);
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
-
- Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList2 = new ArrayList<>();
- interpreterList2.add(mockInterpreter2);
- interpreterGroup = interpreterSetting.getInterpreterGroup("user2", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user2", "note1"), interpreterList2);
-
- assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
- assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note1").size());
- assertEquals(2, interpreterSetting.getInterpreterGroup("user2", "note1").size());
-
- interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
- assertEquals(1, interpreterSetting.getInterpreterGroup("user2","note1").size());
-
- // Check if non-existed key works or not
- interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
- assertEquals(1, interpreterSetting.getInterpreterGroup("user2","note1").size());
-
- interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user2");
- assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
- }
-
- @Test
- public void perUserIsolatedModeCloseAndRemoveInterpreterGroupTest() {
- InterpreterOption interpreterOption = new InterpreterOption();
- interpreterOption.setPerUser(InterpreterOption.ISOLATED);
- InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
-
- interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
- @Override
- public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
- InterpreterOption option) {
- return new InterpreterGroup(interpreterGroupId);
- }
- });
-
- Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList1 = new ArrayList<>();
- interpreterList1.add(mockInterpreter1);
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
-
- Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList2 = new ArrayList<>();
- interpreterList2.add(mockInterpreter2);
- interpreterGroup = interpreterSetting.getInterpreterGroup("user2", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user2", "note1"), interpreterList2);
-
- assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
- assertEquals(1, interpreterSetting.getInterpreterGroup("user2", "note1").size());
-
- interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
- assertEquals(1, interpreterSetting.getInterpreterGroup("user2","note1").size());
- assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
-
- interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user2");
- assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
- }
-
- @Test
- public void perNoteScopedModeCloseAndRemoveInterpreterGroupTest() {
- InterpreterOption interpreterOption = new InterpreterOption();
- interpreterOption.setPerNote(InterpreterOption.SCOPED);
- InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
-
- interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
- @Override
- public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
- InterpreterOption option) {
- return new InterpreterGroup(interpreterGroupId);
- }
- });
-
- Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList1 = new ArrayList<>();
- interpreterList1.add(mockInterpreter1);
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
-
- Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList2 = new ArrayList<>();
- interpreterList2.add(mockInterpreter2);
- interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note2");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note2"), interpreterList2);
-
- assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
- assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note1").size());
- assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note2").size());
-
- interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note2").size());
-
- // Check if non-existed key works or not
- interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note2").size());
-
- interpreterSetting.closeAndRemoveInterpreterGroup("note2", "user1");
- assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
- }
-
- @Test
- public void perNoteIsolatedModeCloseAndRemoveInterpreterGroupTest() {
- InterpreterOption interpreterOption = new InterpreterOption();
- interpreterOption.setPerNote(InterpreterOption.ISOLATED);
- InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
-
- interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
- @Override
- public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
- InterpreterOption option) {
- return new InterpreterGroup(interpreterGroupId);
- }
- });
-
- Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList1 = new ArrayList<>();
- interpreterList1.add(mockInterpreter1);
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
-
- Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList2 = new ArrayList<>();
- interpreterList2.add(mockInterpreter2);
- interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note2");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note2"), interpreterList2);
-
- assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note2").size());
-
- interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note2").size());
- assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
-
- interpreterSetting.closeAndRemoveInterpreterGroup("note2", "user1");
- assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
- }
-
- @Test
- public void perNoteScopedModeRemoveInterpreterGroupWhenNoteIsRemoved() {
- InterpreterOption interpreterOption = new InterpreterOption();
- interpreterOption.setPerNote(InterpreterOption.SCOPED);
- InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
-
- interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
- @Override
- public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
- InterpreterOption option) {
- return new InterpreterGroup(interpreterGroupId);
- }
- });
-
- Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList1 = new ArrayList<>();
- interpreterList1.add(mockInterpreter1);
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
-
- assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
-
- // This method will be called when remove note
- interpreterSetting.closeAndRemoveInterpreterGroup("note1","");
- assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
- // Be careful that getInterpreterGroup makes interpreterGroup if it doesn't exist
- assertEquals(0, interpreterSetting.getInterpreterGroup("user1","note1").size());
- }
-
- @Test
- public void perNoteIsolatedModeRemoveInterpreterGroupWhenNoteIsRemoved() {
- InterpreterOption interpreterOption = new InterpreterOption();
- interpreterOption.setPerNote(InterpreterOption.ISOLATED);
- InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
-
- interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
- @Override
- public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
- InterpreterOption option) {
- return new InterpreterGroup(interpreterGroupId);
- }
- });
-
- Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList1 = new ArrayList<>();
- interpreterList1.add(mockInterpreter1);
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
-
- assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
-
- // This method will be called when remove note
- interpreterSetting.closeAndRemoveInterpreterGroup("note1","");
- assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
- // Be careful that getInterpreterGroup makes interpreterGroup if it doesn't exist
- assertEquals(0, interpreterSetting.getInterpreterGroup("user1","note1").size());
- }
-
- @Test
- public void perUserScopedModeNeverRemoveInterpreterGroupWhenNoteIsRemoved() {
- InterpreterOption interpreterOption = new InterpreterOption();
- interpreterOption.setPerUser(InterpreterOption.SCOPED);
- InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
-
- interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
- @Override
- public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
- InterpreterOption option) {
- return new InterpreterGroup(interpreterGroupId);
- }
- });
-
- Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList1 = new ArrayList<>();
- interpreterList1.add(mockInterpreter1);
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
-
- assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
-
- // This method will be called when remove note
- interpreterSetting.closeAndRemoveInterpreterGroup("note1","");
- assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
- // Be careful that getInterpreterGroup makes interpreterGroup if it doesn't exist
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note1").size());
- }
-
- @Test
- public void perUserIsolatedModeNeverRemoveInterpreterGroupWhenNoteIsRemoved() {
- InterpreterOption interpreterOption = new InterpreterOption();
- interpreterOption.setPerUser(InterpreterOption.ISOLATED);
- InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
-
- interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
- @Override
- public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
- InterpreterOption option) {
- return new InterpreterGroup(interpreterGroupId);
- }
- });
-
- Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList1 = new ArrayList<>();
- interpreterList1.add(mockInterpreter1);
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
-
- assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
-
- // This method will be called when remove note
- interpreterSetting.closeAndRemoveInterpreterGroup("note1","");
- assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
- // Be careful that getInterpreterGroup makes interpreterGroup if it doesn't exist
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note1").size());
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java
deleted file mode 100644
index e934f1a..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java
+++ /dev/null
@@ -1,86 +0,0 @@
-package org.apache.zeppelin.interpreter.install;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-public class InstallInterpreterTest {
- private File tmpDir;
- private InstallInterpreter installer;
- private File interpreterBaseDir;
-
- @Before
- public void setUp() throws IOException {
- tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
- new File(tmpDir, "conf").mkdirs();
- interpreterBaseDir = new File(tmpDir, "interpreter");
- File localRepoDir = new File(tmpDir, "local-repo");
- interpreterBaseDir.mkdir();
- localRepoDir.mkdir();
-
- File interpreterListFile = new File(tmpDir, "conf/interpreter-list");
-
-
- // create interpreter list file
- System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
-
- String interpreterList = "";
- interpreterList += "intp1 org.apache.commons:commons-csv:1.1 test interpreter 1\n";
- interpreterList += "intp2 org.apache.commons:commons-math3:3.6.1 test interpreter 2\n";
-
- FileUtils.writeStringToFile(new File(tmpDir, "conf/interpreter-list"), interpreterList);
-
- installer = new InstallInterpreter(interpreterListFile, interpreterBaseDir, localRepoDir
- .getAbsolutePath());
- }
-
- @After
- public void tearDown() throws IOException {
- FileUtils.deleteDirectory(tmpDir);
- }
-
-
- @Test
- public void testList() {
- assertEquals(2, installer.list().size());
- }
-
- @Test
- public void install() {
- assertEquals(0, interpreterBaseDir.listFiles().length);
-
- installer.install("intp1");
- assertTrue(new File(interpreterBaseDir, "intp1").isDirectory());
- }
-
- @Test
- public void installAll() {
- installer.installAll();
- assertTrue(new File(interpreterBaseDir, "intp1").isDirectory());
- assertTrue(new File(interpreterBaseDir, "intp2").isDirectory());
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
deleted file mode 100644
index b16e937..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.zeppelin.interpreter.mock;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
-import org.apache.zeppelin.scheduler.Scheduler;
-import org.apache.zeppelin.scheduler.SchedulerFactory;
-
-public class MockInterpreter1 extends Interpreter{
-Map<String, Object> vars = new HashMap<>();
-
- public MockInterpreter1(Properties property) {
- super(property);
- }
- boolean open;
-
-
- @Override
- public void open() {
- open = true;
- }
-
- @Override
- public void close() {
- open = false;
- }
-
-
- public boolean isOpen() {
- return open;
- }
-
- @Override
- public InterpreterResult interpret(String st, InterpreterContext context) {
- InterpreterResult result;
-
- if ("getId".equals(st)) {
- // get unique id of this interpreter instance
- result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "" + this.hashCode());
- } else if (st.startsWith("sleep")) {
- try {
- Thread.sleep(Integer.parseInt(st.split(" ")[1]));
- } catch (InterruptedException e) {
- // nothing to do
- }
- result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl1: " + st);
- } else {
- result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl1: " + st);
- }
-
- if (context.getResourcePool() != null) {
- context.getResourcePool().put(context.getNoteId(), context.getParagraphId(), "result", result);
- }
-
- return result;
- }
-
- @Override
- public void cancel(InterpreterContext context) {
- }
-
- @Override
- public FormType getFormType() {
- return FormType.SIMPLE;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-
- @Override
- public Scheduler getScheduler() {
- return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_"+this.hashCode());
- }
-
- @Override
- public List<InterpreterCompletion> completion(String buf, int cursor,
- InterpreterContext interpreterContext) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter11.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter11.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter11.java
deleted file mode 100644
index 5b9e802..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter11.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.mock;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
-import org.apache.zeppelin.scheduler.Scheduler;
-import org.apache.zeppelin.scheduler.SchedulerFactory;
-
-public class MockInterpreter11 extends Interpreter{
- Map<String, Object> vars = new HashMap<>();
-
- public MockInterpreter11(Properties property) {
- super(property);
- }
- boolean open;
-
- @Override
- public void open() {
- open = true;
- }
-
- @Override
- public void close() {
- open = false;
- }
-
- public boolean isOpen() {
- return open;
- }
-
- @Override
- public InterpreterResult interpret(String st, InterpreterContext context) {
- return new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl11: "+st);
- }
-
- @Override
- public void cancel(InterpreterContext context) {
- }
-
- @Override
- public FormType getFormType() {
- return FormType.SIMPLE;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-
- @Override
- public Scheduler getScheduler() {
- return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_"+this.hashCode());
- }
-
- @Override
- public List<InterpreterCompletion> completion(String buf, int cursor,
- InterpreterContext interpreterContext) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java
deleted file mode 100644
index 7a52f7d..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.mock;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
-import org.apache.zeppelin.scheduler.Scheduler;
-import org.apache.zeppelin.scheduler.SchedulerFactory;
-
-public class MockInterpreter2 extends Interpreter{
- Map<String, Object> vars = new HashMap<>();
-
- public MockInterpreter2(Properties property) {
- super(property);
- }
-
- boolean open;
-
- @Override
- public void open() {
- open = true;
- }
-
- @Override
- public void close() {
- open = false;
- }
-
- public boolean isOpen() {
- return open;
- }
-
-
- @Override
- public InterpreterResult interpret(String st, InterpreterContext context) {
- InterpreterResult result;
-
- if ("getId".equals(st)) {
- // get unique id of this interpreter instance
- result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "" + this.hashCode());
- } else if (st.startsWith("sleep")) {
- try {
- Thread.sleep(Integer.parseInt(st.split(" ")[1]));
- } catch (InterruptedException e) {
- // nothing to do
- }
- result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl2: " + st);
- } else {
- result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl2: " + st);
- }
-
- if (context.getResourcePool() != null) {
- context.getResourcePool().put(context.getNoteId(), context.getParagraphId(), "result", result);
- }
- return result;
- }
-
- @Override
- public void cancel(InterpreterContext context) {
- }
-
- @Override
- public FormType getFormType() {
- return FormType.SIMPLE;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-
- @Override
- public Scheduler getScheduler() {
- return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_"+this.hashCode());
- }
-
- @Override
- public List<InterpreterCompletion> completion(String buf, int cursor,
- InterpreterContext interpreterContext) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java
deleted file mode 100644
index c8c64ea..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyInt;
-import static org.mockito.Mockito.atMost;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.log4j.AppenderSkeleton;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.log4j.spi.LoggingEvent;
-import org.junit.After;
-import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-public class AppendOutputRunnerTest {
-
- private static final int NUM_EVENTS = 10000;
- private static final int NUM_CLUBBED_EVENTS = 100;
- private static final ScheduledExecutorService service = Executors.newSingleThreadScheduledExecutor();
- private static ScheduledFuture<?> future = null;
- /* It is being accessed by multiple threads.
- * While loop for 'loopForBufferCompletion' could
- * run for-ever.
- */
- private volatile static int numInvocations = 0;
-
- @After
- public void afterEach() {
- if (future != null) {
- future.cancel(true);
- }
- }
-
- @Test
- public void testSingleEvent() throws InterruptedException {
- RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
- String[][] buffer = {{"note", "para", "data\n"}};
-
- loopForCompletingEvents(listener, 1, buffer);
- verify(listener, times(1)).onOutputAppend(any(String.class), any(String.class), anyInt(), any(String.class));
- verify(listener, times(1)).onOutputAppend("note", "para", 0, "data\n");
- }
-
- @Test
- public void testMultipleEventsOfSameParagraph() throws InterruptedException {
- RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
- String note1 = "note1";
- String para1 = "para1";
- String[][] buffer = {
- {note1, para1, "data1\n"},
- {note1, para1, "data2\n"},
- {note1, para1, "data3\n"}
- };
-
- loopForCompletingEvents(listener, 1, buffer);
- verify(listener, times(1)).onOutputAppend(any(String.class), any(String.class), anyInt(), any(String.class));
- verify(listener, times(1)).onOutputAppend(note1, para1, 0, "data1\ndata2\ndata3\n");
- }
-
- @Test
- public void testMultipleEventsOfDifferentParagraphs() throws InterruptedException {
- RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
- String note1 = "note1";
- String note2 = "note2";
- String para1 = "para1";
- String para2 = "para2";
- String[][] buffer = {
- {note1, para1, "data1\n"},
- {note1, para2, "data2\n"},
- {note2, para1, "data3\n"},
- {note2, para2, "data4\n"}
- };
- loopForCompletingEvents(listener, 4, buffer);
-
- verify(listener, times(4)).onOutputAppend(any(String.class), any(String.class), anyInt(), any(String.class));
- verify(listener, times(1)).onOutputAppend(note1, para1, 0, "data1\n");
- verify(listener, times(1)).onOutputAppend(note1, para2, 0, "data2\n");
- verify(listener, times(1)).onOutputAppend(note2, para1, 0, "data3\n");
- verify(listener, times(1)).onOutputAppend(note2, para2, 0, "data4\n");
- }
-
- @Test
- public void testClubbedData() throws InterruptedException {
- RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
- AppendOutputRunner runner = new AppendOutputRunner(listener);
- future = service.scheduleWithFixedDelay(runner, 0,
- AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
- Thread thread = new Thread(new BombardEvents(runner));
- thread.start();
- thread.join();
- Thread.sleep(1000);
-
- /* NUM_CLUBBED_EVENTS is a heuristic number.
- * It has been observed that for 10,000 continuos event
- * calls, 30-40 Web-socket calls are made. Keeping
- * the unit-test to a pessimistic 100 web-socket calls.
- */
- verify(listener, atMost(NUM_CLUBBED_EVENTS)).onOutputAppend(any(String.class), any(String.class), anyInt(), any(String.class));
- }
-
- @Test
- public void testWarnLoggerForLargeData() throws InterruptedException {
- RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
- AppendOutputRunner runner = new AppendOutputRunner(listener);
- String data = "data\n";
- int numEvents = 100000;
-
- for (int i=0; i<numEvents; i++) {
- runner.appendBuffer("noteId", "paraId", 0, data);
- }
-
- TestAppender appender = new TestAppender();
- Logger logger = Logger.getRootLogger();
- logger.addAppender(appender);
- Logger.getLogger(RemoteInterpreterEventPoller.class);
-
- runner.run();
- List<LoggingEvent> log;
-
- int warnLogCounter;
- LoggingEvent sizeWarnLogEntry = null;
- do {
- warnLogCounter = 0;
- log = appender.getLog();
- for (LoggingEvent logEntry: log) {
- if (Level.WARN.equals(logEntry.getLevel())) {
- sizeWarnLogEntry = logEntry;
- warnLogCounter += 1;
- }
- }
- } while(warnLogCounter != 2);
-
- String loggerString = "Processing size for buffered append-output is high: " +
- (data.length() * numEvents) + " characters.";
- assertTrue(loggerString.equals(sizeWarnLogEntry.getMessage()));
- }
-
- private class BombardEvents implements Runnable {
-
- private final AppendOutputRunner runner;
-
- private BombardEvents(AppendOutputRunner runner) {
- this.runner = runner;
- }
-
- @Override
- public void run() {
- String noteId = "noteId";
- String paraId = "paraId";
- for (int i=0; i<NUM_EVENTS; i++) {
- runner.appendBuffer(noteId, paraId, 0, "data\n");
- }
- }
- }
-
- private class TestAppender extends AppenderSkeleton {
- private final List<LoggingEvent> log = new ArrayList<>();
-
- @Override
- public boolean requiresLayout() {
- return false;
- }
-
- @Override
- protected void append(final LoggingEvent loggingEvent) {
- log.add(loggingEvent);
- }
-
- @Override
- public void close() {
- }
-
- public List<LoggingEvent> getLog() {
- return new ArrayList<>(log);
- }
- }
-
- private void prepareInvocationCounts(RemoteInterpreterProcessListener listener) {
- doAnswer(new Answer<Void>() {
- @Override
- public Void answer(InvocationOnMock invocation) throws Throwable {
- numInvocations += 1;
- return null;
- }
- }).when(listener).onOutputAppend(any(String.class), any(String.class), anyInt(), any(String.class));
- }
-
- private void loopForCompletingEvents(RemoteInterpreterProcessListener listener,
- int numTimes, String[][] buffer) {
- numInvocations = 0;
- prepareInvocationCounts(listener);
- AppendOutputRunner runner = new AppendOutputRunner(listener);
- for (String[] bufferElement: buffer) {
- runner.appendBuffer(bufferElement[0], bufferElement[1], 0, bufferElement[2]);
- }
- future = service.scheduleWithFixedDelay(runner, 0,
- AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
- long startTimeMs = System.currentTimeMillis();
- while(numInvocations != numTimes) {
- if (System.currentTimeMillis() - startTimeMs > 2000) {
- fail("Buffered events were not sent for 2 seconds");
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
deleted file mode 100644
index f7404e3..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.Properties;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.zeppelin.display.*;
-import org.apache.zeppelin.interpreter.*;
-import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterAngular;
-import org.apache.zeppelin.resource.LocalResourcePool;
-import org.apache.zeppelin.user.AuthenticationInfo;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
- private static final String INTERPRETER_SCRIPT =
- System.getProperty("os.name").startsWith("Windows") ?
- "../bin/interpreter.cmd" :
- "../bin/interpreter.sh";
-
- private InterpreterGroup intpGroup;
- private HashMap<String, String> env;
- private RemoteInterpreter intp;
- private InterpreterContext context;
- private RemoteAngularObjectRegistry localRegistry;
-
- private AtomicInteger onAdd;
- private AtomicInteger onUpdate;
- private AtomicInteger onRemove;
-
- @Before
- public void setUp() throws Exception {
- onAdd = new AtomicInteger(0);
- onUpdate = new AtomicInteger(0);
- onRemove = new AtomicInteger(0);
-
- intpGroup = new InterpreterGroup("intpId");
- localRegistry = new RemoteAngularObjectRegistry("intpId", this, intpGroup);
- intpGroup.setAngularObjectRegistry(localRegistry);
- env = new HashMap<>();
- env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
-
- Properties p = new Properties();
-
- intp = new RemoteInterpreter(
- p,
- "note",
- MockInterpreterAngular.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- null,
- null,
- "anonymous",
- false
- );
-
- intpGroup.put("note", new LinkedList<Interpreter>());
- intpGroup.get("note").add(intp);
- intp.setInterpreterGroup(intpGroup);
-
- context = new InterpreterContext(
- "note",
- "id",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null);
-
- intp.open();
- }
-
- @After
- public void tearDown() throws Exception {
- intp.close();
- intpGroup.close();
- }
-
- @Test
- public void testAngularObjectInterpreterSideCRUD() throws InterruptedException {
- InterpreterResult ret = intp.interpret("get", context);
- Thread.sleep(500); // waitFor eventpoller pool event
- String[] result = ret.message().get(0).getData().split(" ");
- assertEquals("0", result[0]); // size of registry
- assertEquals("0", result[1]); // num watcher called
-
- // create object
- ret = intp.interpret("add n1 v1", context);
- Thread.sleep(500);
- result = ret.message().get(0).getData().split(" ");
- assertEquals("1", result[0]); // size of registry
- assertEquals("0", result[1]); // num watcher called
- assertEquals("v1", localRegistry.get("n1", "note", null).get());
-
- // update object
- ret = intp.interpret("update n1 v11", context);
- result = ret.message().get(0).getData().split(" ");
- Thread.sleep(500);
- assertEquals("1", result[0]); // size of registry
- assertEquals("1", result[1]); // num watcher called
- assertEquals("v11", localRegistry.get("n1", "note", null).get());
-
- // remove object
- ret = intp.interpret("remove n1", context);
- result = ret.message().get(0).getData().split(" ");
- Thread.sleep(500);
- assertEquals("0", result[0]); // size of registry
- assertEquals("1", result[1]); // num watcher called
- assertEquals(null, localRegistry.get("n1", "note", null));
- }
-
- @Test
- public void testAngularObjectRemovalOnZeppelinServerSide() throws InterruptedException {
- // test if angularobject removal from server side propagate to interpreter process's registry.
- // will happen when notebook is removed.
-
- InterpreterResult ret = intp.interpret("get", context);
- Thread.sleep(500); // waitFor eventpoller pool event
- String[] result = ret.message().get(0).getData().split(" ");
- assertEquals("0", result[0]); // size of registry
-
- // create object
- ret = intp.interpret("add n1 v1", context);
- Thread.sleep(500);
- result = ret.message().get(0).getData().split(" ");
- assertEquals("1", result[0]); // size of registry
- assertEquals("v1", localRegistry.get("n1", "note", null).get());
-
- // remove object in local registry.
- localRegistry.removeAndNotifyRemoteProcess("n1", "note", null);
- ret = intp.interpret("get", context);
- Thread.sleep(500); // waitFor eventpoller pool event
- result = ret.message().get(0).getData().split(" ");
- assertEquals("0", result[0]); // size of registry
- }
-
- @Test
- public void testAngularObjectAddOnZeppelinServerSide() throws InterruptedException {
- // test if angularobject add from server side propagate to interpreter process's registry.
- // will happen when zeppelin server loads notebook and restore the object into registry
-
- InterpreterResult ret = intp.interpret("get", context);
- Thread.sleep(500); // waitFor eventpoller pool event
- String[] result = ret.message().get(0).getData().split(" ");
- assertEquals("0", result[0]); // size of registry
-
- // create object
- localRegistry.addAndNotifyRemoteProcess("n1", "v1", "note", null);
-
- // get from remote registry
- ret = intp.interpret("get", context);
- Thread.sleep(500); // waitFor eventpoller pool event
- result = ret.message().get(0).getData().split(" ");
- assertEquals("1", result[0]); // size of registry
- }
-
- @Override
- public void onAdd(String interpreterGroupId, AngularObject object) {
- onAdd.incrementAndGet();
- }
-
- @Override
- public void onUpdate(String interpreterGroupId, AngularObject object) {
- onUpdate.incrementAndGet();
- }
-
- @Override
- public void onRemove(String interpreterGroupId, String name, String noteId, String paragraphId) {
- onRemove.incrementAndGet();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPollerTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPollerTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPollerTest.java
deleted file mode 100644
index 49aa7aa..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPollerTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote;
-
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
-import org.junit.Test;
-
-import static org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType.NO_OP;
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class RemoteInterpreterEventPollerTest {
-
- @Test
- public void shouldClearUnreadEventsOnShutdown() throws Exception {
- RemoteInterpreterProcess interpreterProc = getMockEventsInterpreterProcess();
- RemoteInterpreterEventPoller eventPoller = new RemoteInterpreterEventPoller(null, null);
-
- eventPoller.setInterpreterProcess(interpreterProc);
- eventPoller.shutdown();
- eventPoller.start();
- eventPoller.join();
-
- assertEquals(NO_OP, interpreterProc.getClient().getEvent().getType());
- }
-
- private RemoteInterpreterProcess getMockEventsInterpreterProcess() throws Exception {
- RemoteInterpreterEvent fakeEvent = new RemoteInterpreterEvent();
- RemoteInterpreterEvent noMoreEvents = new RemoteInterpreterEvent(NO_OP, "");
- RemoteInterpreterService.Client client = mock(RemoteInterpreterService.Client.class);
- RemoteInterpreterProcess intProc = mock(RemoteInterpreterProcess.class);
-
- when(client.getEvent()).thenReturn(fakeEvent, fakeEvent, noMoreEvents);
- when(intProc.getClient()).thenReturn(client);
-
- return intProc;
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
deleted file mode 100644
index 3f865cb..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote;
-
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.user.AuthenticationInfo;
-import org.apache.zeppelin.display.GUI;
-import org.apache.zeppelin.interpreter.*;
-import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterOutputStream;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.Properties;
-
-import static org.junit.Assert.assertEquals;
-
-
-/**
- * Test for remote interpreter output stream
- */
-public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProcessListener {
- private static final String INTERPRETER_SCRIPT =
- System.getProperty("os.name").startsWith("Windows") ?
- "../bin/interpreter.cmd" :
- "../bin/interpreter.sh";
- private InterpreterGroup intpGroup;
- private HashMap<String, String> env;
-
- @Before
- public void setUp() throws Exception {
- intpGroup = new InterpreterGroup();
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- env = new HashMap<>();
- env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
- }
-
- @After
- public void tearDown() throws Exception {
- intpGroup.close();
- }
-
- private RemoteInterpreter createMockInterpreter() {
- RemoteInterpreter intp = new RemoteInterpreter(
- new Properties(),
- "note",
- MockInterpreterOutputStream.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- this,
- null,
- "anonymous",
- false);
-
- intpGroup.get("note").add(intp);
- intp.setInterpreterGroup(intpGroup);
- return intp;
- }
-
- private InterpreterContext createInterpreterContext() {
- return new InterpreterContext(
- "noteId",
- "id",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- null,
- new LinkedList<InterpreterContextRunner>(), null);
- }
-
- @Test
- public void testInterpreterResultOnly() {
- RemoteInterpreter intp = createMockInterpreter();
- InterpreterResult ret = intp.interpret("SUCCESS::staticresult", createInterpreterContext());
- assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
- assertEquals("staticresult", ret.message().get(0).getData());
-
- ret = intp.interpret("SUCCESS::staticresult2", createInterpreterContext());
- assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
- assertEquals("staticresult2", ret.message().get(0).getData());
-
- ret = intp.interpret("ERROR::staticresult3", createInterpreterContext());
- assertEquals(InterpreterResult.Code.ERROR, ret.code());
- assertEquals("staticresult3", ret.message().get(0).getData());
- }
-
- @Test
- public void testInterpreterOutputStreamOnly() {
- RemoteInterpreter intp = createMockInterpreter();
- InterpreterResult ret = intp.interpret("SUCCESS:streamresult:", createInterpreterContext());
- assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
- assertEquals("streamresult", ret.message().get(0).getData());
-
- ret = intp.interpret("ERROR:streamresult2:", createInterpreterContext());
- assertEquals(InterpreterResult.Code.ERROR, ret.code());
- assertEquals("streamresult2", ret.message().get(0).getData());
- }
-
- @Test
- public void testInterpreterResultOutputStreamMixed() {
- RemoteInterpreter intp = createMockInterpreter();
- InterpreterResult ret = intp.interpret("SUCCESS:stream:static", createInterpreterContext());
- assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
- assertEquals("stream", ret.message().get(0).getData());
- assertEquals("static", ret.message().get(1).getData());
- }
-
- @Test
- public void testOutputType() {
- RemoteInterpreter intp = createMockInterpreter();
-
- InterpreterResult ret = intp.interpret("SUCCESS:%html hello:", createInterpreterContext());
- assertEquals(InterpreterResult.Type.HTML, ret.message().get(0).getType());
- assertEquals("hello", ret.message().get(0).getData());
-
- ret = intp.interpret("SUCCESS:%html\nhello:", createInterpreterContext());
- assertEquals(InterpreterResult.Type.HTML, ret.message().get(0).getType());
- assertEquals("hello", ret.message().get(0).getData());
-
- ret = intp.interpret("SUCCESS:%html hello:%angular world", createInterpreterContext());
- assertEquals(InterpreterResult.Type.HTML, ret.message().get(0).getType());
- assertEquals("hello", ret.message().get(0).getData());
- assertEquals(InterpreterResult.Type.ANGULAR, ret.message().get(1).getType());
- assertEquals("world", ret.message().get(1).getData());
- }
-
- @Override
- public void onOutputAppend(String noteId, String paragraphId, int index, String output) {
-
- }
-
- @Override
- public void onOutputUpdated(String noteId, String paragraphId, int index, InterpreterResult.Type type, String output) {
-
- }
-
- @Override
- public void onOutputClear(String noteId, String paragraphId) {
-
- }
-
- @Override
- public void onMetaInfosReceived(String settingId, Map<String, String> metaInfos) {
-
- }
-
- @Override
- public void onGetParagraphRunners(String noteId, String paragraphId, RemoteWorksEventListener callback) {
- if (callback != null) {
- callback.onFinished(new LinkedList<>());
- }
- }
-
- @Override
- public void onRemoteRunParagraph(String noteId, String ParagraphID) throws Exception {
-
- }
-
- @Override
- public void onParaInfosReceived(String noteId, String paragraphId,
- String interpreterSettingId, Map<String, String> metaInfos) {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
deleted file mode 100644
index b85d7ef..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.mockito.Mockito.*;
-
-import java.util.HashMap;
-import java.util.Properties;
-
-import org.apache.thrift.TException;
-import org.apache.thrift.transport.TTransportException;
-import org.apache.zeppelin.interpreter.Constants;
-import org.apache.zeppelin.interpreter.InterpreterException;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
-import org.junit.Test;
-
-public class RemoteInterpreterProcessTest {
- private static final String INTERPRETER_SCRIPT =
- System.getProperty("os.name").startsWith("Windows") ?
- "../bin/interpreter.cmd" :
- "../bin/interpreter.sh";
- private static final int DUMMY_PORT=3678;
-
- @Test
- public void testStartStop() {
- InterpreterGroup intpGroup = new InterpreterGroup();
- RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
- INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap<String, String>(),
- 10 * 1000, null, null,"fakeName");
- assertFalse(rip.isRunning());
- assertEquals(0, rip.referenceCount());
- assertEquals(1, rip.reference(intpGroup, "anonymous", false));
- assertEquals(2, rip.reference(intpGroup, "anonymous", false));
- assertEquals(true, rip.isRunning());
- assertEquals(1, rip.dereference());
- assertEquals(true, rip.isRunning());
- assertEquals(0, rip.dereference());
- assertEquals(false, rip.isRunning());
- }
-
- @Test
- public void testClientFactory() throws Exception {
- InterpreterGroup intpGroup = new InterpreterGroup();
- RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
- INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap<String, String>(),
- mock(RemoteInterpreterEventPoller.class), 10 * 1000, "fakeName");
- rip.reference(intpGroup, "anonymous", false);
- assertEquals(0, rip.getNumActiveClient());
- assertEquals(0, rip.getNumIdleClient());
-
- Client client = rip.getClient();
- assertEquals(1, rip.getNumActiveClient());
- assertEquals(0, rip.getNumIdleClient());
-
- rip.releaseClient(client);
- assertEquals(0, rip.getNumActiveClient());
- assertEquals(1, rip.getNumIdleClient());
-
- rip.dereference();
- }
-
- @Test
- public void testStartStopRemoteInterpreter() throws TException, InterruptedException {
- RemoteInterpreterServer server = new RemoteInterpreterServer(3678);
- server.start();
- boolean running = false;
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() - startTime < 10 * 1000) {
- if (server.isRunning()) {
- running = true;
- break;
- } else {
- Thread.sleep(200);
- }
- }
- Properties properties = new Properties();
- properties.setProperty(Constants.ZEPPELIN_INTERPRETER_PORT, "3678");
- properties.setProperty(Constants.ZEPPELIN_INTERPRETER_HOST, "localhost");
- InterpreterGroup intpGroup = mock(InterpreterGroup.class);
- when(intpGroup.getProperty()).thenReturn(properties);
- when(intpGroup.containsKey(Constants.EXISTING_PROCESS)).thenReturn(true);
-
- RemoteInterpreterProcess rip = new RemoteInterpreterManagedProcess(
- INTERPRETER_SCRIPT,
- "nonexists",
- "fakeRepo",
- new HashMap<String, String>(),
- mock(RemoteInterpreterEventPoller.class)
- , 10 * 1000,
- "fakeName");
- assertFalse(rip.isRunning());
- assertEquals(0, rip.referenceCount());
- assertEquals(1, rip.reference(intpGroup, "anonymous", false));
- assertEquals(true, rip.isRunning());
- }
-
-
- @Test
- public void testPropagateError() throws TException, InterruptedException {
- InterpreterGroup intpGroup = new InterpreterGroup();
- RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
- "echo hello_world", "nonexists", "fakeRepo", new HashMap<String, String>(),
- 10 * 1000, null, null, "fakeName");
- assertFalse(rip.isRunning());
- assertEquals(0, rip.referenceCount());
- try {
- assertEquals(1, rip.reference(intpGroup, "anonymous", false));
- } catch (InterpreterException e) {
- e.getMessage().contains("hello_world");
- }
- assertEquals(0, rip.referenceCount());
- }
-}
[11/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
Posted by zj...@apache.org.
[ZEPPELIN-2627] Interpreter refactor
### What is this PR for?
I didn't intended to make such large change at the beginning, but found many things are coupled together that I have to make such large change. Several suggestions for you how to review and read it.
* I move the interpreter package from zeppelin-zengine to zeppelin-interpreter, this is needed for this refactoring.
* The overall change is the same as I described in the design doc. I would suggest you to read the unit test first. These unit test is very readable and easy to understand what the code is doing now. `InterpreterFactoryTest`, `InterpreterGroupTest`, `InterpreterSettingTest`, `InterpreterSettingManagerTest`, `RemoteInterpreterTest`.
* Remove the referent counting logic. Now I will kill the interpreter process as long as all the sessions in the same interpreter group is closed. (I plan to add another kind of policy for the interpreter process lifecycle control, ZEPPELIN-2197)
* The `RemoteFunction` I introduced is for reducing code duplicates when we use RPC.
* The changes in Job.java and RemoteScheduler is for fixing the race issue bug. This bug cause the flaky test we see often in `ZeppelinSparkClusterTest.pySparkTest`
### What type of PR is it?
[Bug Fix | Refactoring]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-2627
### How should this be tested?
Unit test is added
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #2422 from zjffdu/interpreter_refactor and squashes the following commits:
4724c98 [Jeff Zhang] [ZEPPELIN-2627] Interpreter Component Refactoring
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/8d4902e7
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/8d4902e7
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/8d4902e7
Branch: refs/heads/master
Commit: 8d4902e717ba2932ddb0f7110f2a669811f5d1af
Parents: 32517c9
Author: Jeff Zhang <zj...@apache.org>
Authored: Tue May 23 15:52:15 2017 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Mon Aug 28 09:14:11 2017 +0800
----------------------------------------------------------------------
.travis.yml | 1 +
.../zeppelin/helium/ZeppelinDevServer.java | 8 +-
zeppelin-interpreter/pom.xml | 22 +
.../zeppelin/conf/ZeppelinConfiguration.java | 835 +++++++++++++
.../zeppelin/interpreter/Interpreter.java | 1 -
.../interpreter/InterpreterFactory.java | 115 ++
.../zeppelin/interpreter/InterpreterGroup.java | 272 ++---
.../zeppelin/interpreter/InterpreterInfo.java | 82 ++
.../interpreter/InterpreterInfoSaving.java | 101 ++
.../interpreter/InterpreterProperty.java | 1 +
.../zeppelin/interpreter/InterpreterRunner.java | 11 +
.../interpreter/InterpreterSetting.java | 911 ++++++++++++++
.../interpreter/InterpreterSettingManager.java | 886 ++++++++++++++
.../interpreter/install/InstallInterpreter.java | 288 +++++
.../remote/RemoteAngularObjectRegistry.java | 125 ++
.../interpreter/remote/RemoteInterpreter.java | 371 ++++++
.../remote/RemoteInterpreterEventPoller.java | 279 ++---
.../remote/RemoteInterpreterManagedProcess.java | 260 ++++
.../remote/RemoteInterpreterProcess.java | 154 +--
.../remote/RemoteInterpreterRunningProcess.java | 67 ++
.../remote/RemoteInterpreterServer.java | 58 +-
.../zeppelin/resource/ResourcePoolUtils.java | 138 ---
.../java/org/apache/zeppelin/scheduler/Job.java | 11 +-
.../zeppelin/scheduler/RemoteScheduler.java | 134 +--
.../zeppelin/scheduler/SchedulerFactory.java | 49 +-
.../zeppelin/tabledata/TableDataProxy.java | 1 -
.../java/org/apache/zeppelin/util/IdHashes.java | 76 ++
.../java/org/apache/zeppelin/util/Util.java | 76 ++
.../interpreter/AbstractInterpreterTest.java | 74 ++
.../interpreter/DoubleEchoInterpreter.java | 61 +
.../zeppelin/interpreter/DummyInterpreter.java | 43 -
.../zeppelin/interpreter/EchoInterpreter.java | 65 +
.../interpreter/InterpreterFactoryTest.java | 66 +
.../interpreter/InterpreterGroupTest.java | 90 ++
.../InterpreterOutputChangeWatcherTest.java | 11 +-
.../InterpreterSettingManagerTest.java | 270 +++++
.../interpreter/InterpreterSettingTest.java | 411 +++++++
.../zeppelin/interpreter/InterpreterTest.java | 7 +-
.../zeppelin/interpreter/SleepInterpreter.java | 60 +
.../install/InstallInterpreterTest.java | 86 ++
.../interpreter/mock/MockInterpreter1.java | 106 ++
.../interpreter/mock/MockInterpreter11.java | 85 ++
.../interpreter/mock/MockInterpreter2.java | 104 ++
.../remote/AppendOutputRunnerTest.java | 232 ++++
.../remote/RemoteAngularObjectTest.java | 196 +++
.../RemoteInterpreterEventPollerTest.java | 55 +
.../RemoteInterpreterOutputTestStream.java | 178 +++
.../remote/RemoteInterpreterTest.java | 520 ++++++++
.../remote/RemoteInterpreterUtilsTest.java | 33 +
.../remote/mock/GetEnvPropertyInterpreter.java | 82 ++
.../remote/mock/MockInterpreterA.java | 96 ++
.../remote/mock/MockInterpreterAngular.java | 112 ++
.../remote/mock/MockInterpreterB.java | 120 ++
.../mock/MockInterpreterOutputStream.java | 93 ++
.../mock/MockInterpreterResourcePool.java | 125 ++
.../zeppelin/scheduler/RemoteSchedulerTest.java | 343 ++++++
.../src/test/resources/conf/interpreter.json | 115 ++
.../interpreter/test/interpreter-setting.json | 42 +
.../src/test/resources/log4j.properties | 4 +-
.../zeppelin/rest/InterpreterRestApi.java | 4 +-
.../apache/zeppelin/server/ZeppelinServer.java | 35 +-
.../apache/zeppelin/socket/NotebookServer.java | 32 +-
.../interpreter/mock/MockInterpreter1.java | 75 --
.../zeppelin/rest/AbstractTestRestApi.java | 7 +-
.../zeppelin/rest/InterpreterRestApiTest.java | 6 +-
.../zeppelin/socket/NotebookServerTest.java | 13 +-
.../src/test/resources/log4j.properties | 1 -
zeppelin-zengine/pom.xml | 13 +-
.../zeppelin/conf/ZeppelinConfiguration.java | 835 -------------
.../java/org/apache/zeppelin/helium/Helium.java | 51 +-
.../helium/HeliumApplicationFactory.java | 120 +-
.../interpreter/InterpreterFactory.java | 423 -------
.../interpreter/InterpreterGroupFactory.java | 26 -
.../zeppelin/interpreter/InterpreterInfo.java | 82 --
.../interpreter/InterpreterInfoSaving.java | 46 -
.../interpreter/InterpreterSetting.java | 459 -------
.../interpreter/InterpreterSettingManager.java | 1136 ------------------
.../interpreter/install/InstallInterpreter.java | 290 -----
.../remote/RemoteAngularObjectRegistry.java | 133 --
.../interpreter/remote/RemoteInterpreter.java | 597 ---------
.../remote/RemoteInterpreterManagedProcess.java | 247 ----
.../remote/RemoteInterpreterRunningProcess.java | 67 --
.../zeppelin/notebook/ApplicationState.java | 1 -
.../java/org/apache/zeppelin/notebook/Note.java | 12 +-
.../org/apache/zeppelin/notebook/Notebook.java | 23 +-
.../org/apache/zeppelin/notebook/Paragraph.java | 21 +-
.../java/org/apache/zeppelin/util/Util.java | 76 --
.../helium/HeliumApplicationFactoryTest.java | 79 +-
.../org/apache/zeppelin/helium/HeliumTest.java | 8 +-
.../interpreter/InterpreterFactoryTest.java | 497 --------
.../interpreter/InterpreterSettingTest.java | 327 -----
.../install/InstallInterpreterTest.java | 86 --
.../interpreter/mock/MockInterpreter1.java | 105 --
.../interpreter/mock/MockInterpreter11.java | 83 --
.../interpreter/mock/MockInterpreter2.java | 104 --
.../remote/AppendOutputRunnerTest.java | 236 ----
.../remote/RemoteAngularObjectTest.java | 201 ----
.../RemoteInterpreterEventPollerTest.java | 55 -
.../RemoteInterpreterOutputTestStream.java | 191 ---
.../remote/RemoteInterpreterProcessTest.java | 131 --
.../remote/RemoteInterpreterTest.java | 975 ---------------
.../remote/RemoteInterpreterUtilsTest.java | 34 -
.../remote/mock/MockInterpreterA.java | 97 --
.../remote/mock/MockInterpreterAngular.java | 113 --
.../remote/mock/MockInterpreterB.java | 126 --
.../remote/mock/MockInterpreterEnv.java | 80 --
.../mock/MockInterpreterOutputStream.java | 90 --
.../mock/MockInterpreterResourcePool.java | 128 --
.../notebook/NoteInterpreterLoaderTest.java | 243 ----
.../apache/zeppelin/notebook/NotebookTest.java | 181 ++-
.../apache/zeppelin/notebook/ParagraphTest.java | 9 +-
.../notebook/repo/NotebookRepoSyncTest.java | 74 +-
.../notebook/repo/VFSNotebookRepoTest.java | 55 +-
.../resource/DistributedResourcePoolTest.java | 96 +-
.../zeppelin/scheduler/RemoteSchedulerTest.java | 364 ------
.../src/test/resources/conf/interpreter.json | 1 +
.../interpreter/mock/interpreter-setting.json | 12 -
.../interpreter/mock1/interpreter-setting.json | 19 +
.../interpreter/mock2/interpreter-setting.json | 19 +
.../mock_resource_pool/interpreter-setting.json | 19 +
.../src/test/resources/log4j.properties | 3 +-
121 files changed, 8861 insertions(+), 10029 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 97ca60a..64ea559 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -162,6 +162,7 @@ after_success:
after_failure:
- echo "Travis exited with ${TRAVIS_TEST_RESULT}"
- find . -name rat.txt | xargs cat
+ - cat logs/*
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.log
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
- cat zeppelin-web/npm-debug.log
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java
----------------------------------------------------------------------
diff --git a/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java b/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java
index 21ce283..2484469 100644
--- a/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java
+++ b/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java
@@ -43,13 +43,13 @@ public class ZeppelinDevServer extends
}
@Override
- protected Interpreter getInterpreter(String sessionKey, String className) throws TException {
+ protected Interpreter getInterpreter(String sessionId, String className) throws TException {
synchronized (this) {
InterpreterGroup interpreterGroup = getInterpreterGroup();
if (interpreterGroup == null || interpreterGroup.isEmpty()) {
createInterpreter(
"dev",
- sessionKey,
+ sessionId,
DevInterpreter.class.getName(),
new HashMap<String, String>(),
"anonymous");
@@ -57,11 +57,11 @@ public class ZeppelinDevServer extends
}
}
- Interpreter intp = super.getInterpreter(sessionKey, className);
+ Interpreter intp = super.getInterpreter(sessionId, className);
interpreter = (DevInterpreter) (
((LazyOpenInterpreter) intp).getInnerInterpreter());
interpreter.setInterpreterEvent(this);
- return super.getInterpreter(sessionKey, className);
+ return super.getInterpreter(sessionId, className);
}
@Override
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/pom.xml
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml
index 384b9d1..d8f9c43 100644
--- a/zeppelin-interpreter/pom.xml
+++ b/zeppelin-interpreter/pom.xml
@@ -52,6 +52,11 @@
<dependencies>
<dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
</dependency>
@@ -226,4 +231,21 @@
<scope>test</scope>
</dependency>
</dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>3.0.2</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</project>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
new file mode 100644
index 0000000..03cc069
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -0,0 +1,835 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.conf;
+
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.XMLConfiguration;
+import org.apache.commons.configuration.tree.ConfigurationNode;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.util.Util;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Zeppelin configuration.
+ *
+ */
+public class ZeppelinConfiguration extends XMLConfiguration {
+ private static final String ZEPPELIN_SITE_XML = "zeppelin-site.xml";
+ private static final long serialVersionUID = 4749305895693848035L;
+ private static final Logger LOG = LoggerFactory.getLogger(ZeppelinConfiguration.class);
+
+ private static final String HELIUM_PACKAGE_DEFAULT_URL =
+ "https://s3.amazonaws.com/helium-package/helium.json";
+ private static ZeppelinConfiguration conf;
+
+ public ZeppelinConfiguration(URL url) throws ConfigurationException {
+ setDelimiterParsingDisabled(true);
+ load(url);
+ }
+
+ public ZeppelinConfiguration() {
+ ConfVars[] vars = ConfVars.values();
+ for (ConfVars v : vars) {
+ if (v.getType() == ConfVars.VarType.BOOLEAN) {
+ this.setProperty(v.getVarName(), v.getBooleanValue());
+ } else if (v.getType() == ConfVars.VarType.LONG) {
+ this.setProperty(v.getVarName(), v.getLongValue());
+ } else if (v.getType() == ConfVars.VarType.INT) {
+ this.setProperty(v.getVarName(), v.getIntValue());
+ } else if (v.getType() == ConfVars.VarType.FLOAT) {
+ this.setProperty(v.getVarName(), v.getFloatValue());
+ } else if (v.getType() == ConfVars.VarType.STRING) {
+ this.setProperty(v.getVarName(), v.getStringValue());
+ } else {
+ throw new RuntimeException("Unsupported VarType");
+ }
+ }
+
+ }
+
+
+ /**
+ * Load from resource.
+ *url = ZeppelinConfiguration.class.getResource(ZEPPELIN_SITE_XML);
+ * @throws ConfigurationException
+ */
+ public static synchronized ZeppelinConfiguration create() {
+ if (conf != null) {
+ return conf;
+ }
+
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ URL url;
+
+ url = ZeppelinConfiguration.class.getResource(ZEPPELIN_SITE_XML);
+ if (url == null) {
+ ClassLoader cl = ZeppelinConfiguration.class.getClassLoader();
+ if (cl != null) {
+ url = cl.getResource(ZEPPELIN_SITE_XML);
+ }
+ }
+ if (url == null) {
+ url = classLoader.getResource(ZEPPELIN_SITE_XML);
+ }
+
+ if (url == null) {
+ LOG.warn("Failed to load configuration, proceeding with a default");
+ conf = new ZeppelinConfiguration();
+ } else {
+ try {
+ LOG.info("Load configuration from " + url);
+ conf = new ZeppelinConfiguration(url);
+ } catch (ConfigurationException e) {
+ LOG.warn("Failed to load configuration from " + url + " proceeding with a default", e);
+ conf = new ZeppelinConfiguration();
+ }
+ }
+
+ LOG.info("Server Host: " + conf.getServerAddress());
+ if (conf.useSsl() == false) {
+ LOG.info("Server Port: " + conf.getServerPort());
+ } else {
+ LOG.info("Server SSL Port: " + conf.getServerSslPort());
+ }
+ LOG.info("Context Path: " + conf.getServerContextPath());
+ LOG.info("Zeppelin Version: " + Util.getVersion());
+
+ return conf;
+ }
+
+
+ private String getStringValue(String name, String d) {
+ List<ConfigurationNode> properties = getRootNode().getChildren();
+ if (properties == null || properties.isEmpty()) {
+ return d;
+ }
+ for (ConfigurationNode p : properties) {
+ if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
+ && name.equals(p.getChildren("name").get(0).getValue())) {
+ return (String) p.getChildren("value").get(0).getValue();
+ }
+ }
+ return d;
+ }
+
+ private int getIntValue(String name, int d) {
+ List<ConfigurationNode> properties = getRootNode().getChildren();
+ if (properties == null || properties.isEmpty()) {
+ return d;
+ }
+ for (ConfigurationNode p : properties) {
+ if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
+ && name.equals(p.getChildren("name").get(0).getValue())) {
+ return Integer.parseInt((String) p.getChildren("value").get(0).getValue());
+ }
+ }
+ return d;
+ }
+
+ private long getLongValue(String name, long d) {
+ List<ConfigurationNode> properties = getRootNode().getChildren();
+ if (properties == null || properties.isEmpty()) {
+ return d;
+ }
+ for (ConfigurationNode p : properties) {
+ if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
+ && name.equals(p.getChildren("name").get(0).getValue())) {
+ return Long.parseLong((String) p.getChildren("value").get(0).getValue());
+ }
+ }
+ return d;
+ }
+
+ private float getFloatValue(String name, float d) {
+ List<ConfigurationNode> properties = getRootNode().getChildren();
+ if (properties == null || properties.isEmpty()) {
+ return d;
+ }
+ for (ConfigurationNode p : properties) {
+ if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
+ && name.equals(p.getChildren("name").get(0).getValue())) {
+ return Float.parseFloat((String) p.getChildren("value").get(0).getValue());
+ }
+ }
+ return d;
+ }
+
+ private boolean getBooleanValue(String name, boolean d) {
+ List<ConfigurationNode> properties = getRootNode().getChildren();
+ if (properties == null || properties.isEmpty()) {
+ return d;
+ }
+ for (ConfigurationNode p : properties) {
+ if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
+ && name.equals(p.getChildren("name").get(0).getValue())) {
+ return Boolean.parseBoolean((String) p.getChildren("value").get(0).getValue());
+ }
+ }
+ return d;
+ }
+
+ public String getString(ConfVars c) {
+ return getString(c.name(), c.getVarName(), c.getStringValue());
+ }
+
+ public String getString(String envName, String propertyName, String defaultValue) {
+ if (System.getenv(envName) != null) {
+ return System.getenv(envName);
+ }
+ if (System.getProperty(propertyName) != null) {
+ return System.getProperty(propertyName);
+ }
+
+ return getStringValue(propertyName, defaultValue);
+ }
+
+ public int getInt(ConfVars c) {
+ return getInt(c.name(), c.getVarName(), c.getIntValue());
+ }
+
+ public int getInt(String envName, String propertyName, int defaultValue) {
+ if (System.getenv(envName) != null) {
+ return Integer.parseInt(System.getenv(envName));
+ }
+
+ if (System.getProperty(propertyName) != null) {
+ return Integer.parseInt(System.getProperty(propertyName));
+ }
+ return getIntValue(propertyName, defaultValue);
+ }
+
+ public long getLong(ConfVars c) {
+ return getLong(c.name(), c.getVarName(), c.getLongValue());
+ }
+
+ public long getLong(String envName, String propertyName, long defaultValue) {
+ if (System.getenv(envName) != null) {
+ return Long.parseLong(System.getenv(envName));
+ }
+
+ if (System.getProperty(propertyName) != null) {
+ return Long.parseLong(System.getProperty(propertyName));
+ }
+ return getLongValue(propertyName, defaultValue);
+ }
+
+ public float getFloat(ConfVars c) {
+ return getFloat(c.name(), c.getVarName(), c.getFloatValue());
+ }
+
+ public float getFloat(String envName, String propertyName, float defaultValue) {
+ if (System.getenv(envName) != null) {
+ return Float.parseFloat(System.getenv(envName));
+ }
+ if (System.getProperty(propertyName) != null) {
+ return Float.parseFloat(System.getProperty(propertyName));
+ }
+ return getFloatValue(propertyName, defaultValue);
+ }
+
+ public boolean getBoolean(ConfVars c) {
+ return getBoolean(c.name(), c.getVarName(), c.getBooleanValue());
+ }
+
+ public boolean getBoolean(String envName, String propertyName, boolean defaultValue) {
+ if (System.getenv(envName) != null) {
+ return Boolean.parseBoolean(System.getenv(envName));
+ }
+
+ if (System.getProperty(propertyName) != null) {
+ return Boolean.parseBoolean(System.getProperty(propertyName));
+ }
+ return getBooleanValue(propertyName, defaultValue);
+ }
+
+ public boolean useSsl() {
+ return getBoolean(ConfVars.ZEPPELIN_SSL);
+ }
+
+ public int getServerSslPort() {
+ return getInt(ConfVars.ZEPPELIN_SSL_PORT);
+ }
+
+ public boolean useClientAuth() {
+ return getBoolean(ConfVars.ZEPPELIN_SSL_CLIENT_AUTH);
+ }
+
+ public String getServerAddress() {
+ return getString(ConfVars.ZEPPELIN_ADDR);
+ }
+
+ public int getServerPort() {
+ return getInt(ConfVars.ZEPPELIN_PORT);
+ }
+
+ public String getServerContextPath() {
+ return getString(ConfVars.ZEPPELIN_SERVER_CONTEXT_PATH);
+ }
+
+ public String getKeyStorePath() {
+ String path = getString(ConfVars.ZEPPELIN_SSL_KEYSTORE_PATH);
+ if (path != null && path.startsWith("/") || isWindowsPath(path)) {
+ return path;
+ } else {
+ return getRelativeDir(
+ String.format("%s/%s",
+ getConfDir(),
+ path));
+ }
+ }
+
+ public String getKeyStoreType() {
+ return getString(ConfVars.ZEPPELIN_SSL_KEYSTORE_TYPE);
+ }
+
+ public String getKeyStorePassword() {
+ return getString(ConfVars.ZEPPELIN_SSL_KEYSTORE_PASSWORD);
+ }
+
+ public String getKeyManagerPassword() {
+ String password = getString(ConfVars.ZEPPELIN_SSL_KEY_MANAGER_PASSWORD);
+ if (password == null) {
+ return getKeyStorePassword();
+ } else {
+ return password;
+ }
+ }
+
+ public String getTrustStorePath() {
+ String path = getString(ConfVars.ZEPPELIN_SSL_TRUSTSTORE_PATH);
+ if (path == null) {
+ path = getKeyStorePath();
+ }
+ if (path != null && path.startsWith("/") || isWindowsPath(path)) {
+ return path;
+ } else {
+ return getRelativeDir(
+ String.format("%s/%s",
+ getConfDir(),
+ path));
+ }
+ }
+
+ public String getTrustStoreType() {
+ String type = getString(ConfVars.ZEPPELIN_SSL_TRUSTSTORE_TYPE);
+ if (type == null) {
+ return getKeyStoreType();
+ } else {
+ return type;
+ }
+ }
+
+ public String getTrustStorePassword() {
+ String password = getString(ConfVars.ZEPPELIN_SSL_TRUSTSTORE_PASSWORD);
+ if (password == null) {
+ return getKeyStorePassword();
+ } else {
+ return password;
+ }
+ }
+
+ public String getNotebookDir() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_DIR);
+ }
+
+ public String getUser() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_USER);
+ }
+
+ public String getBucketName() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_BUCKET);
+ }
+
+ public String getEndpoint() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_ENDPOINT);
+ }
+
+ public String getS3KMSKeyID() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID);
+ }
+
+ public String getS3KMSKeyRegion() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION);
+ }
+
+ public String getS3EncryptionMaterialsProviderClass() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_EMP);
+ }
+
+ public boolean isS3ServerSideEncryption() {
+ return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_S3_SSE);
+ }
+
+ public String getMongoUri() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_MONGO_URI);
+ }
+
+ public String getMongoDatabase() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_MONGO_DATABASE);
+ }
+
+ public String getMongoCollection() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_MONGO_COLLECTION);
+ }
+
+ public boolean getMongoAutoimport() {
+ return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT);
+ }
+
+ public String getInterpreterListPath() {
+ return getRelativeDir(String.format("%s/interpreter-list", getConfDir()));
+ }
+
+ public String getInterpreterDir() {
+ return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_DIR);
+ }
+
+ public String getInterpreterJson() {
+ return getString(ConfVars.ZEPPELIN_INTERPRETER_JSON);
+ }
+
+ public String getInterpreterSettingPath() {
+ return getRelativeDir(String.format("%s/interpreter.json", getConfDir()));
+ }
+
+ public String getHeliumConfPath() {
+ return getRelativeDir(String.format("%s/helium.json", getConfDir()));
+ }
+
+ public String getHeliumRegistry() {
+ return getRelativeDir(ConfVars.ZEPPELIN_HELIUM_REGISTRY);
+ }
+
+ public String getHeliumNodeInstallerUrl() {
+ return getString(ConfVars.ZEPPELIN_HELIUM_NODE_INSTALLER_URL);
+ }
+
+ public String getHeliumNpmInstallerUrl() {
+ return getString(ConfVars.ZEPPELIN_HELIUM_NPM_INSTALLER_URL);
+ }
+
+ public String getHeliumYarnInstallerUrl() {
+ return getString(ConfVars.ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL);
+ }
+
+ public String getNotebookAuthorizationPath() {
+ return getRelativeDir(String.format("%s/notebook-authorization.json", getConfDir()));
+ }
+
+ public Boolean credentialsPersist() {
+ return getBoolean(ConfVars.ZEPPELIN_CREDENTIALS_PERSIST);
+ }
+
+ public String getCredentialsPath() {
+ return getRelativeDir(String.format("%s/credentials.json", getConfDir()));
+ }
+
+ public String getShiroPath() {
+ String shiroPath = getRelativeDir(String.format("%s/shiro.ini", getConfDir()));
+ return new File(shiroPath).exists() ? shiroPath : StringUtils.EMPTY;
+ }
+
+ public String getInterpreterRemoteRunnerPath() {
+ return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER);
+ }
+
+ public String getInterpreterLocalRepoPath() {
+ return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO);
+ }
+
+ public String getInterpreterMvnRepoPath() {
+ return getString(ConfVars.ZEPPELIN_INTERPRETER_DEP_MVNREPO);
+ }
+
+ public String getRelativeDir(ConfVars c) {
+ return getRelativeDir(getString(c));
+ }
+
+ public String getRelativeDir(String path) {
+ if (path != null && path.startsWith("/") || isWindowsPath(path)) {
+ return path;
+ } else {
+ return getString(ConfVars.ZEPPELIN_HOME) + "/" + path;
+ }
+ }
+
+ public boolean isWindowsPath(String path){
+ return path.matches("^[A-Za-z]:\\\\.*");
+ }
+
+ public boolean isAnonymousAllowed() {
+ return getBoolean(ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED);
+ }
+
+ public boolean isNotebokPublic() {
+ return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC);
+ }
+
+ public String getConfDir() {
+ return getString(ConfVars.ZEPPELIN_CONF_DIR);
+ }
+
+ public List<String> getAllowedOrigins()
+ {
+ if (getString(ConfVars.ZEPPELIN_ALLOWED_ORIGINS).isEmpty()) {
+ return Arrays.asList(new String[0]);
+ }
+
+ return Arrays.asList(getString(ConfVars.ZEPPELIN_ALLOWED_ORIGINS).toLowerCase().split(","));
+ }
+
+ public String getWebsocketMaxTextMessageSize() {
+ return getString(ConfVars.ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE);
+ }
+
+ public String getJettyName() {
+ return getString(ConfVars.ZEPPELIN_SERVER_JETTY_NAME);
+ }
+
+
+ public String getXFrameOptions() {
+ return getString(ConfVars.ZEPPELIN_SERVER_XFRAME_OPTIONS);
+ }
+
+ public String getXxssProtection() {
+ return getString(ConfVars.ZEPPELIN_SERVER_X_XSS_PROTECTION);
+ }
+
+ public String getStrictTransport() {
+ return getString(ConfVars.ZEPPELIN_SERVER_STRICT_TRANSPORT);
+ }
+
+
+ public Map<String, String> dumpConfigurations(ZeppelinConfiguration conf,
+ ConfigurationKeyPredicate predicate) {
+ Map<String, String> configurations = new HashMap<>();
+
+ for (ConfVars v : ConfVars.values()) {
+ String key = v.getVarName();
+
+ if (!predicate.apply(key)) {
+ continue;
+ }
+
+ ConfVars.VarType type = v.getType();
+ Object value = null;
+ if (type == ConfVars.VarType.BOOLEAN) {
+ value = conf.getBoolean(v);
+ } else if (type == ConfVars.VarType.LONG) {
+ value = conf.getLong(v);
+ } else if (type == ConfVars.VarType.INT) {
+ value = conf.getInt(v);
+ } else if (type == ConfVars.VarType.FLOAT) {
+ value = conf.getFloat(v);
+ } else if (type == ConfVars.VarType.STRING) {
+ value = conf.getString(v);
+ }
+
+ if (value != null) {
+ configurations.put(key, value.toString());
+ }
+ }
+ return configurations;
+ }
+
+ /**
+ * Predication whether key/value pair should be included or not
+ */
+ public interface ConfigurationKeyPredicate {
+ boolean apply(String key);
+ }
+
+ /**
+ * Wrapper class.
+ */
+ public static enum ConfVars {
+ ZEPPELIN_HOME("zeppelin.home", "./"),
+ ZEPPELIN_ADDR("zeppelin.server.addr", "0.0.0.0"),
+ ZEPPELIN_PORT("zeppelin.server.port", 8080),
+ ZEPPELIN_SERVER_CONTEXT_PATH("zeppelin.server.context.path", "/"),
+ ZEPPELIN_SSL("zeppelin.ssl", false),
+ ZEPPELIN_SSL_PORT("zeppelin.server.ssl.port", 8443),
+ ZEPPELIN_SSL_CLIENT_AUTH("zeppelin.ssl.client.auth", false),
+ ZEPPELIN_SSL_KEYSTORE_PATH("zeppelin.ssl.keystore.path", "keystore"),
+ ZEPPELIN_SSL_KEYSTORE_TYPE("zeppelin.ssl.keystore.type", "JKS"),
+ ZEPPELIN_SSL_KEYSTORE_PASSWORD("zeppelin.ssl.keystore.password", ""),
+ ZEPPELIN_SSL_KEY_MANAGER_PASSWORD("zeppelin.ssl.key.manager.password", null),
+ ZEPPELIN_SSL_TRUSTSTORE_PATH("zeppelin.ssl.truststore.path", null),
+ ZEPPELIN_SSL_TRUSTSTORE_TYPE("zeppelin.ssl.truststore.type", null),
+ ZEPPELIN_SSL_TRUSTSTORE_PASSWORD("zeppelin.ssl.truststore.password", null),
+ ZEPPELIN_WAR("zeppelin.war", "zeppelin-web/dist"),
+ ZEPPELIN_WAR_TEMPDIR("zeppelin.war.tempdir", "webapps"),
+ ZEPPELIN_INTERPRETERS("zeppelin.interpreters", "org.apache.zeppelin.spark.SparkInterpreter,"
+ + "org.apache.zeppelin.spark.PySparkInterpreter,"
+ + "org.apache.zeppelin.rinterpreter.RRepl,"
+ + "org.apache.zeppelin.rinterpreter.KnitR,"
+ + "org.apache.zeppelin.spark.SparkRInterpreter,"
+ + "org.apache.zeppelin.spark.SparkSqlInterpreter,"
+ + "org.apache.zeppelin.spark.DepInterpreter,"
+ + "org.apache.zeppelin.markdown.Markdown,"
+ + "org.apache.zeppelin.angular.AngularInterpreter,"
+ + "org.apache.zeppelin.shell.ShellInterpreter,"
+ + "org.apache.zeppelin.livy.LivySparkInterpreter,"
+ + "org.apache.zeppelin.livy.LivySparkSQLInterpreter,"
+ + "org.apache.zeppelin.livy.LivyPySparkInterpreter,"
+ + "org.apache.zeppelin.livy.LivyPySpark3Interpreter,"
+ + "org.apache.zeppelin.livy.LivySparkRInterpreter,"
+ + "org.apache.zeppelin.alluxio.AlluxioInterpreter,"
+ + "org.apache.zeppelin.file.HDFSFileInterpreter,"
+ + "org.apache.zeppelin.pig.PigInterpreter,"
+ + "org.apache.zeppelin.pig.PigQueryInterpreter,"
+ + "org.apache.zeppelin.flink.FlinkInterpreter,"
+ + "org.apache.zeppelin.python.PythonInterpreter,"
+ + "org.apache.zeppelin.python.PythonInterpreterPandasSql,"
+ + "org.apache.zeppelin.python.PythonCondaInterpreter,"
+ + "org.apache.zeppelin.python.PythonDockerInterpreter,"
+ + "org.apache.zeppelin.ignite.IgniteInterpreter,"
+ + "org.apache.zeppelin.ignite.IgniteSqlInterpreter,"
+ + "org.apache.zeppelin.lens.LensInterpreter,"
+ + "org.apache.zeppelin.cassandra.CassandraInterpreter,"
+ + "org.apache.zeppelin.geode.GeodeOqlInterpreter,"
+ + "org.apache.zeppelin.kylin.KylinInterpreter,"
+ + "org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,"
+ + "org.apache.zeppelin.scalding.ScaldingInterpreter,"
+ + "org.apache.zeppelin.jdbc.JDBCInterpreter,"
+ + "org.apache.zeppelin.hbase.HbaseInterpreter,"
+ + "org.apache.zeppelin.bigquery.BigQueryInterpreter,"
+ + "org.apache.zeppelin.beam.BeamInterpreter,"
+ + "org.apache.zeppelin.scio.ScioInterpreter,"
+ + "org.apache.zeppelin.groovy.GroovyInterpreter"
+ ),
+ ZEPPELIN_INTERPRETER_JSON("zeppelin.interpreter.setting", "interpreter-setting.json"),
+ ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"),
+ ZEPPELIN_INTERPRETER_LOCALREPO("zeppelin.interpreter.localRepo", "local-repo"),
+ ZEPPELIN_INTERPRETER_DEP_MVNREPO("zeppelin.interpreter.dep.mvnRepo",
+ "http://repo1.maven.org/maven2/"),
+ ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000),
+ ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10),
+ ZEPPELIN_INTERPRETER_GROUP_ORDER("zeppelin.interpreter.group.order", "spark,md,angular,sh,"
+ + "livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,"
+ + "scalding,jdbc,hbase,bigquery,beam,pig,scio,groovy"),
+ ZEPPELIN_INTERPRETER_OUTPUT_LIMIT("zeppelin.interpreter.output.limit", 1024 * 100),
+ ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"),
+ ZEPPELIN_NOTEBOOK_DIR("zeppelin.notebook.dir", "notebook"),
+ // use specified notebook (id) as homescreen
+ ZEPPELIN_NOTEBOOK_HOMESCREEN("zeppelin.notebook.homescreen", null),
+ // whether homescreen notebook will be hidden from notebook list or not
+ ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE("zeppelin.notebook.homescreen.hide", false),
+ ZEPPELIN_NOTEBOOK_S3_BUCKET("zeppelin.notebook.s3.bucket", "zeppelin"),
+ ZEPPELIN_NOTEBOOK_S3_ENDPOINT("zeppelin.notebook.s3.endpoint", "s3.amazonaws.com"),
+ ZEPPELIN_NOTEBOOK_S3_USER("zeppelin.notebook.s3.user", "user"),
+ ZEPPELIN_NOTEBOOK_S3_EMP("zeppelin.notebook.s3.encryptionMaterialsProvider", null),
+ ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID("zeppelin.notebook.s3.kmsKeyID", null),
+ ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION("zeppelin.notebook.s3.kmsKeyRegion", null),
+ ZEPPELIN_NOTEBOOK_S3_SSE("zeppelin.notebook.s3.sse", false),
+ ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING("zeppelin.notebook.azure.connectionString", null),
+ ZEPPELIN_NOTEBOOK_AZURE_SHARE("zeppelin.notebook.azure.share", "zeppelin"),
+ ZEPPELIN_NOTEBOOK_AZURE_USER("zeppelin.notebook.azure.user", "user"),
+ ZEPPELIN_NOTEBOOK_MONGO_DATABASE("zeppelin.notebook.mongo.database", "zeppelin"),
+ ZEPPELIN_NOTEBOOK_MONGO_COLLECTION("zeppelin.notebook.mongo.collection", "notes"),
+ ZEPPELIN_NOTEBOOK_MONGO_URI("zeppelin.notebook.mongo.uri", "mongodb://localhost"),
+ ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT("zeppelin.notebook.mongo.autoimport", false),
+ ZEPPELIN_NOTEBOOK_STORAGE("zeppelin.notebook.storage",
+ "org.apache.zeppelin.notebook.repo.GitNotebookRepo"),
+ ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC("zeppelin.notebook.one.way.sync", false),
+ // whether by default note is public or private
+ ZEPPELIN_NOTEBOOK_PUBLIC("zeppelin.notebook.public", true),
+ ZEPPELIN_INTERPRETER_REMOTE_RUNNER("zeppelin.interpreter.remoterunner",
+ System.getProperty("os.name")
+ .startsWith("Windows") ? "bin/interpreter.cmd" : "bin/interpreter.sh"),
+ // Decide when new note is created, interpreter settings will be binded automatically or not.
+ ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING("zeppelin.notebook.autoInterpreterBinding", true),
+ ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"),
+ ZEPPELIN_DEP_LOCALREPO("zeppelin.dep.localrepo", "local-repo"),
+ ZEPPELIN_HELIUM_REGISTRY("zeppelin.helium.registry", "helium," + HELIUM_PACKAGE_DEFAULT_URL),
+ ZEPPELIN_HELIUM_NODE_INSTALLER_URL("zeppelin.helium.node.installer.url",
+ "https://nodejs.org/dist/"),
+ ZEPPELIN_HELIUM_NPM_INSTALLER_URL("zeppelin.helium.npm.installer.url",
+ "http://registry.npmjs.org/"),
+ ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL("zeppelin.helium.yarnpkg.installer.url",
+ "https://github.com/yarnpkg/yarn/releases/download/"),
+ // Allows a way to specify a ',' separated list of allowed origins for rest and websockets
+ // i.e. http://localhost:8080
+ ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"),
+ ZEPPELIN_ANONYMOUS_ALLOWED("zeppelin.anonymous.allowed", true),
+ ZEPPELIN_CREDENTIALS_PERSIST("zeppelin.credentials.persist", true),
+ ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE("zeppelin.websocket.max.text.message.size", "1024000"),
+ ZEPPELIN_SERVER_DEFAULT_DIR_ALLOWED("zeppelin.server.default.dir.allowed", false),
+ ZEPPELIN_SERVER_XFRAME_OPTIONS("zeppelin.server.xframe.options", "SAMEORIGIN"),
+ ZEPPELIN_SERVER_JETTY_NAME("zeppelin.server.jetty.name", null),
+ ZEPPELIN_SERVER_STRICT_TRANSPORT("zeppelin.server.strict.transport", "max-age=631138519"),
+ ZEPPELIN_SERVER_X_XSS_PROTECTION("zeppelin.server.xxss.protection", "1"),
+
+ ZEPPELIN_HDFS_KEYTAB("zeppelin.hdfs.keytab", ""),
+ ZEPPELIN_HDFS_PRINCIPAL("zeppelin.hdfs.principal", "");
+
+ private String varName;
+ @SuppressWarnings("rawtypes")
+ private Class varClass;
+ private String stringValue;
+ private VarType type;
+ private int intValue;
+ private float floatValue;
+ private boolean booleanValue;
+ private long longValue;
+
+
+ ConfVars(String varName, String varValue) {
+ this.varName = varName;
+ this.varClass = String.class;
+ this.stringValue = varValue;
+ this.intValue = -1;
+ this.floatValue = -1;
+ this.longValue = -1;
+ this.booleanValue = false;
+ this.type = VarType.STRING;
+ }
+
+ ConfVars(String varName, int intValue) {
+ this.varName = varName;
+ this.varClass = Integer.class;
+ this.stringValue = null;
+ this.intValue = intValue;
+ this.floatValue = -1;
+ this.longValue = -1;
+ this.booleanValue = false;
+ this.type = VarType.INT;
+ }
+
+ ConfVars(String varName, long longValue) {
+ this.varName = varName;
+ this.varClass = Integer.class;
+ this.stringValue = null;
+ this.intValue = -1;
+ this.floatValue = -1;
+ this.longValue = longValue;
+ this.booleanValue = false;
+ this.type = VarType.LONG;
+ }
+
+ ConfVars(String varName, float floatValue) {
+ this.varName = varName;
+ this.varClass = Float.class;
+ this.stringValue = null;
+ this.intValue = -1;
+ this.longValue = -1;
+ this.floatValue = floatValue;
+ this.booleanValue = false;
+ this.type = VarType.FLOAT;
+ }
+
+ ConfVars(String varName, boolean booleanValue) {
+ this.varName = varName;
+ this.varClass = Boolean.class;
+ this.stringValue = null;
+ this.intValue = -1;
+ this.longValue = -1;
+ this.floatValue = -1;
+ this.booleanValue = booleanValue;
+ this.type = VarType.BOOLEAN;
+ }
+
+ public String getVarName() {
+ return varName;
+ }
+
+ @SuppressWarnings("rawtypes")
+ public Class getVarClass() {
+ return varClass;
+ }
+
+ public int getIntValue() {
+ return intValue;
+ }
+
+ public long getLongValue() {
+ return longValue;
+ }
+
+ public float getFloatValue() {
+ return floatValue;
+ }
+
+ public String getStringValue() {
+ return stringValue;
+ }
+
+ public boolean getBooleanValue() {
+ return booleanValue;
+ }
+
+ public VarType getType() {
+ return type;
+ }
+
+ enum VarType {
+ STRING {
+ @Override
+ void checkType(String value) throws Exception {}
+ },
+ INT {
+ @Override
+ void checkType(String value) throws Exception {
+ Integer.valueOf(value);
+ }
+ },
+ LONG {
+ @Override
+ void checkType(String value) throws Exception {
+ Long.valueOf(value);
+ }
+ },
+ FLOAT {
+ @Override
+ void checkType(String value) throws Exception {
+ Float.valueOf(value);
+ }
+ },
+ BOOLEAN {
+ @Override
+ void checkType(String value) throws Exception {
+ Boolean.valueOf(value);
+ }
+ };
+
+ boolean isType(String value) {
+ try {
+ checkType(value);
+ } catch (Exception e) {
+ LOG.error("Exception in ZeppelinConfiguration while isType", e);
+ return false;
+ }
+ return true;
+ }
+
+ String typeString() {
+ return name().toUpperCase();
+ }
+
+ abstract void checkType(String value) throws Exception;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
index 74506dd..05599a0 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
@@ -149,7 +149,6 @@ public abstract class Interpreter {
@ZeppelinApi
public Interpreter(Properties property) {
- logger.debug("Properties: {}", property);
this.property = property;
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
new file mode 100644
index 0000000..a2fdb04
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonatype.aether.RepositoryException;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * //TODO(zjffdu) considering to move to InterpreterSettingManager
+ *
+ * Manage interpreters.
+ */
+public class InterpreterFactory {
+ private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterFactory.class);
+
+ private final InterpreterSettingManager interpreterSettingManager;
+
+ public InterpreterFactory(InterpreterSettingManager interpreterSettingManager) {
+ this.interpreterSettingManager = interpreterSettingManager;
+ }
+
+ private InterpreterSetting getInterpreterSettingByGroup(List<InterpreterSetting> settings,
+ String group) {
+
+ Preconditions.checkNotNull(group, "group should be not null");
+ for (InterpreterSetting setting : settings) {
+ if (group.equals(setting.getName())) {
+ return setting;
+ }
+ }
+ return null;
+ }
+
+ public Interpreter getInterpreter(String user, String noteId, String replName) {
+ List<InterpreterSetting> settings = interpreterSettingManager.getInterpreterSettings(noteId);
+ InterpreterSetting setting;
+ Interpreter interpreter;
+
+ if (settings == null || settings.size() == 0) {
+ LOGGER.error("No interpreter is binded to this note: " + noteId);
+ return null;
+ }
+
+ if (StringUtils.isBlank(replName)) {
+ // Get the default interpreter of the first interpreter binding
+ InterpreterSetting defaultSetting = settings.get(0);
+ return defaultSetting.getDefaultInterpreter(user, noteId);
+ }
+
+ String[] replNameSplit = replName.split("\\.");
+ if (replNameSplit.length == 2) {
+ String group = replNameSplit[0];
+ String name = replNameSplit[1];
+ setting = getInterpreterSettingByGroup(settings, group);
+ if (null != setting) {
+ interpreter = setting.getInterpreter(user, noteId, name);
+ if (null != interpreter) {
+ return interpreter;
+ }
+ }
+ throw new InterpreterException(replName + " interpreter not found");
+
+ } else {
+ // first assume replName is 'name' of interpreter. ('groupName' is ommitted)
+ // search 'name' from first (default) interpreter group
+ // TODO(jl): Handle with noteId to support defaultInterpreter per note.
+ setting = settings.get(0);
+ interpreter = setting.getInterpreter(user, noteId, replName);
+
+ if (null != interpreter) {
+ return interpreter;
+ }
+
+ // next, assume replName is 'group' of interpreter ('name' is ommitted)
+ // search interpreter group and return first interpreter.
+ setting = getInterpreterSettingByGroup(settings, replName);
+
+ if (null != setting) {
+ return setting.getDefaultInterpreter(user, noteId);
+ }
+
+ // Support the legacy way to use it
+ for (InterpreterSetting s : settings) {
+ if (s.getGroup().equals(replName)) {
+ return setting.getDefaultInterpreter(user, noteId);
+ }
+ }
+ }
+ //TODO(zjffdu) throw InterpreterException instead of return null
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
index 5cbab6b..ae7ad3a 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
@@ -17,14 +17,7 @@
package org.apache.zeppelin.interpreter;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Random;
-import java.util.concurrent.ConcurrentHashMap;
-
+import com.google.common.annotations.VisibleForTesting;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.apache.zeppelin.resource.ResourcePool;
@@ -33,91 +26,93 @@ import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+
/**
- * InterpreterGroup is list of interpreters in the same interpreter group.
- * For example spark, pyspark, sql interpreters are in the same 'spark' group
- * and InterpreterGroup will have reference to these all interpreters.
+ * InterpreterGroup is collections of interpreter sessions.
+ * One session could include multiple interpreters.
+ * For example spark, pyspark, sql interpreters are in the same 'spark' interpreter session.
*
* Remember, list of interpreters are dedicated to a session. Session could be shared across user
* or notes, so the sessionId could be user or noteId or their combination.
- * So InterpreterGroup internally manages map of [interpreterSessionKey(noteId, user, or
+ * So InterpreterGroup internally manages map of [sessionId(noteId, user, or
* their combination), list of interpreters]
*
- * A InterpreterGroup runs on interpreter process.
- * And unit of interpreter instantiate, restart, bind, unbind.
+ * A InterpreterGroup runs both in zeppelin server process and interpreter process.
*/
-public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter>> {
- String id;
+public class InterpreterGroup {
private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterGroup.class);
- AngularObjectRegistry angularObjectRegistry;
- InterpreterHookRegistry hookRegistry;
- RemoteInterpreterProcess remoteInterpreterProcess; // attached remote interpreter process
- ResourcePool resourcePool;
- boolean angularRegistryPushed = false;
-
- // map [notebook session, Interpreters in the group], to support per note session interpreters
- //Map<String, List<Interpreter>> interpreters = new ConcurrentHashMap<String,
- // List<Interpreter>>();
-
- private static final Map<String, InterpreterGroup> allInterpreterGroups =
- new ConcurrentHashMap<>();
-
- public static InterpreterGroup getByInterpreterGroupId(String id) {
- return allInterpreterGroups.get(id);
- }
+ private String id;
+ // sessionId --> interpreters
+ private Map<String, List<Interpreter>> sessions = new ConcurrentHashMap();
+ private InterpreterSetting interpreterSetting;
+ private AngularObjectRegistry angularObjectRegistry;
+ private InterpreterHookRegistry hookRegistry;
+ private RemoteInterpreterProcess remoteInterpreterProcess; // attached remote interpreter process
+ private ResourcePool resourcePool;
+ private boolean angularRegistryPushed = false;
- public static Collection<InterpreterGroup> getAll() {
- return new LinkedList(allInterpreterGroups.values());
+ /**
+ * Create InterpreterGroup with given id, used in InterpreterProcess
+ * @param id
+ */
+ public InterpreterGroup(String id) {
+ this.id = id;
}
/**
- * Create InterpreterGroup with given id
+ * Create InterpreterGroup with given id and interpreterSetting, used in ZeppelinServer
* @param id
+ * @param interpreterSetting
*/
- public InterpreterGroup(String id) {
+ InterpreterGroup(String id, InterpreterSetting interpreterSetting) {
this.id = id;
- allInterpreterGroups.put(id, this);
+ this.interpreterSetting = interpreterSetting;
}
/**
* Create InterpreterGroup with autogenerated id
*/
+ @VisibleForTesting
public InterpreterGroup() {
- getId();
- allInterpreterGroups.put(id, this);
+ this.id = generateId();
}
private static String generateId() {
- return "InterpreterGroup_" + System.currentTimeMillis() + "_"
- + new Random().nextInt();
+ return "InterpreterGroup_" + System.currentTimeMillis() + "_" + new Random().nextInt();
}
public String getId() {
- synchronized (this) {
- if (id == null) {
- id = generateId();
- }
- return id;
- }
+ return this.id;
}
- /**
- * Get combined property of all interpreters in this group
- * @return
- */
- public Properties getProperty() {
- Properties p = new Properties();
+ //TODO(zjffdu) change it to getSession. For now just keep this method to reduce code change
+ public synchronized List<Interpreter> get(String sessionId) {
+ return sessions.get(sessionId);
+ }
- for (List<Interpreter> intpGroupForASession : this.values()) {
- for (Interpreter intp : intpGroupForASession) {
- p.putAll(intp.getProperty());
- }
- // it's okay to break here while every List<Interpreters> will have the same property set
- break;
+ //TODO(zjffdu) change it to addSession. For now just keep this method to reduce code change
+ public synchronized void put(String sessionId, List<Interpreter> interpreters) {
+ this.sessions.put(sessionId, interpreters);
+ }
+
+ public synchronized void addInterpreterToSession(Interpreter interpreter, String sessionId) {
+ LOGGER.debug("Add Interpreter {} to session {}", interpreter.getClassName(), sessionId);
+ List<Interpreter> interpreters = get(sessionId);
+ if (interpreters == null) {
+ interpreters = new ArrayList<>();
}
- return p;
+ interpreters.add(interpreter);
+ put(sessionId, interpreters);
+ }
+
+ //TODO(zjffdu) rename it to a more proper name.
+ //For now just keep this method to reduce code change
+ public Collection<List<Interpreter>> values() {
+ return sessions.values();
}
public AngularObjectRegistry getAngularObjectRegistry() {
@@ -136,128 +131,69 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
this.hookRegistry = hookRegistry;
}
- public RemoteInterpreterProcess getRemoteInterpreterProcess() {
+ public InterpreterSetting getInterpreterSetting() {
+ return interpreterSetting;
+ }
+
+ public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess() {
+ if (remoteInterpreterProcess == null) {
+ LOGGER.info("Create InterperterProcess for InterpreterGroup: " + getId());
+ remoteInterpreterProcess = interpreterSetting.createInterpreterProcess();
+ }
return remoteInterpreterProcess;
}
- public void setRemoteInterpreterProcess(RemoteInterpreterProcess remoteInterpreterProcess) {
- this.remoteInterpreterProcess = remoteInterpreterProcess;
+ public RemoteInterpreterProcess getRemoteInterpreterProcess() {
+ return remoteInterpreterProcess;
}
+
/**
* Close all interpreter instances in this group
*/
- public void close() {
- LOGGER.info("Close interpreter group " + getId());
- List<Interpreter> intpToClose = new LinkedList<>();
- for (List<Interpreter> intpGroupForSession : this.values()) {
- intpToClose.addAll(intpGroupForSession);
+ public synchronized void close() {
+ LOGGER.info("Close InterpreterGroup: " + id);
+ for (String sessionId : sessions.keySet()) {
+ close(sessionId);
}
- close(intpToClose);
-
- // make sure remote interpreter process terminates
- if (remoteInterpreterProcess != null) {
- while (remoteInterpreterProcess.referenceCount() > 0) {
- remoteInterpreterProcess.dereference();
- }
- remoteInterpreterProcess = null;
- }
- allInterpreterGroups.remove(id);
}
/**
- * Close all interpreter instances in this group for the session
+ * Close all interpreter instances in this session
* @param sessionId
*/
- public void close(String sessionId) {
- LOGGER.info("Close interpreter group " + getId() + " for session: " + sessionId);
- final List<Interpreter> intpForSession = this.get(sessionId);
-
- close(intpForSession);
- }
-
- private void close(final Collection<Interpreter> intpToClose) {
- close(null, null, null, intpToClose);
+ public synchronized void close(String sessionId) {
+ LOGGER.info("Close Session: " + sessionId);
+ close(sessions.remove(sessionId));
+ //TODO(zjffdu) whether close InterpreterGroup if there's no session left in Zeppelin Server
+ if (sessions.isEmpty() && interpreterSetting != null) {
+ LOGGER.info("Remove this InterpreterGroup {} as all the sessions are closed", id);
+ interpreterSetting.removeInterpreterGroup(id);
+ if (remoteInterpreterProcess != null) {
+ LOGGER.info("Kill RemoteInterpreterProcess");
+ remoteInterpreterProcess.stop();
+ remoteInterpreterProcess = null;
+ }
+ }
}
- public void close(final Map<String, InterpreterGroup> interpreterGroupRef,
- final String processKey, final String sessionKey) {
- LOGGER.info("Close interpreter group " + getId() + " for session: " + sessionKey);
- close(interpreterGroupRef, processKey, sessionKey, this.get(sessionKey));
+ public int getSessionNum() {
+ return sessions.size();
}
- private void close(final Map<String, InterpreterGroup> interpreterGroupRef,
- final String processKey, final String sessionKey, final Collection<Interpreter> intpToClose) {
- if (intpToClose == null) {
+ private void close(Collection<Interpreter> interpreters) {
+ if (interpreters == null) {
return;
}
- Thread t = new Thread() {
- public void run() {
- for (Interpreter interpreter : intpToClose) {
- Scheduler scheduler = interpreter.getScheduler();
- interpreter.close();
-
- if (null != scheduler) {
- SchedulerFactory.singleton().removeScheduler(scheduler.getName());
- }
- }
-
- if (remoteInterpreterProcess != null) {
- //TODO(jl): Because interpreter.close() runs as a seprate thread, we cannot guarantee
- // refernceCount is a proper value. And as the same reason, we must not call
- // remoteInterpreterProcess.dereference twice - this method also be called by
- // interpreter.close().
-
- // remoteInterpreterProcess.dereference();
- if (remoteInterpreterProcess.referenceCount() <= 0) {
- remoteInterpreterProcess = null;
- allInterpreterGroups.remove(id);
- }
- }
-
- // TODO(jl): While closing interpreters in a same session, we should remove after all
- // interpreters are removed. OMG. It's too dirty!!
- if (null != interpreterGroupRef && null != processKey && null != sessionKey) {
- InterpreterGroup interpreterGroup = interpreterGroupRef.get(processKey);
- if (1 == interpreterGroup.size() && interpreterGroup.containsKey(sessionKey)) {
- interpreterGroupRef.remove(processKey);
- } else {
- interpreterGroup.remove(sessionKey);
- }
- }
- }
- };
-
- t.start();
- try {
- t.join();
- } catch (InterruptedException e) {
- LOGGER.error("Can't close interpreter: {}", getId(), e);
- }
-
- }
-
- /**
- * Close all interpreter instances in this group
- */
- public void shutdown() {
- LOGGER.info("Close interpreter group " + getId());
-
- // make sure remote interpreter process terminates
- if (remoteInterpreterProcess != null) {
- while (remoteInterpreterProcess.referenceCount() > 0) {
- remoteInterpreterProcess.dereference();
+ for (Interpreter interpreter : interpreters) {
+ Scheduler scheduler = interpreter.getScheduler();
+ interpreter.close();
+ //TODO(zjffdu) move the close of schedule to Interpreter
+ if (null != scheduler) {
+ SchedulerFactory.singleton().removeScheduler(scheduler.getName());
}
- remoteInterpreterProcess = null;
}
- allInterpreterGroups.remove(id);
-
- List<Interpreter> intpToClose = new LinkedList<>();
- for (List<Interpreter> intpGroupForSession : this.values()) {
- intpToClose.addAll(intpGroupForSession);
- }
- close(intpToClose);
}
public void setResourcePool(ResourcePool resourcePool) {
@@ -275,4 +211,22 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
public void setAngularRegistryPushed(boolean angularRegistryPushed) {
this.angularRegistryPushed = angularRegistryPushed;
}
+
+ public synchronized List<Interpreter> getOrCreateSession(String user, String sessionId) {
+ if (sessions.containsKey(sessionId)) {
+ return sessions.get(sessionId);
+ } else {
+ List<Interpreter> interpreters = interpreterSetting.createInterpreters(user, sessionId);
+ for (Interpreter interpreter : interpreters) {
+ interpreter.setInterpreterGroup(this);
+ }
+ LOGGER.info("Create Session {} in InterpreterGroup {} for user {}", sessionId, id, user);
+ sessions.put(sessionId, interpreters);
+ return interpreters;
+ }
+ }
+
+ public boolean isEmpty() {
+ return sessions.isEmpty();
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java
new file mode 100644
index 0000000..fd632ce
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import com.google.gson.annotations.SerializedName;
+
+import java.util.Map;
+
+/**
+ * Information of interpreters in this interpreter setting.
+ * this will be serialized for conf/interpreter.json and REST api response.
+ */
+public class InterpreterInfo {
+ private String name;
+ @SerializedName("class") private String className;
+ private boolean defaultInterpreter = false;
+ private Map<String, Object> editor;
+
+ public InterpreterInfo(String className, String name, boolean defaultInterpreter,
+ Map<String, Object> editor) {
+ this.className = className;
+ this.name = name;
+ this.defaultInterpreter = defaultInterpreter;
+ this.editor = editor;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getClassName() {
+ return className;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ boolean isDefaultInterpreter() {
+ return defaultInterpreter;
+ }
+
+ public Map<String, Object> getEditor() {
+ return editor;
+ }
+
+ public void setEditor(Map<String, Object> editor) {
+ this.editor = editor;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof InterpreterInfo)) {
+ return false;
+ }
+ InterpreterInfo other = (InterpreterInfo) obj;
+
+ boolean sameName =
+ null == getName() ? null == other.getName() : getName().equals(other.getName());
+ boolean sameClassName = null == getClassName() ?
+ null == other.getClassName() :
+ getClassName().equals(other.getClassName());
+ boolean sameIsDefaultInterpreter = defaultInterpreter == other.isDefaultInterpreter();
+
+ return sameName && sameClassName && sameIsDefaultInterpreter;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java
new file mode 100644
index 0000000..4ccc262
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.internal.StringMap;
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonatype.aether.repository.RemoteRepository;
+
+import java.io.BufferedReader;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.PosixFilePermission;
+import java.util.*;
+
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
+
+/**
+ *
+ */
+public class InterpreterInfoSaving {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterInfoSaving.class);
+ private static final Gson gson = new GsonBuilder().setPrettyPrinting().create();
+
+ public Map<String, InterpreterSetting> interpreterSettings = new HashMap<>();
+ public Map<String, List<String>> interpreterBindings = new HashMap<>();
+ public List<RemoteRepository> interpreterRepositories = new ArrayList<>();
+
+ public static InterpreterInfoSaving loadFromFile(Path file) throws IOException {
+ LOGGER.info("Load interpreter setting from file: " + file);
+ InterpreterInfoSaving infoSaving = null;
+ try (BufferedReader json = Files.newBufferedReader(file, StandardCharsets.UTF_8)) {
+ JsonParser jsonParser = new JsonParser();
+ JsonObject jsonObject = jsonParser.parse(json).getAsJsonObject();
+ infoSaving = InterpreterInfoSaving.fromJson(jsonObject.toString());
+
+ if (infoSaving != null && infoSaving.interpreterSettings != null) {
+ for (InterpreterSetting interpreterSetting : infoSaving.interpreterSettings.values()) {
+ // Always use separate interpreter process
+ // While we decided to turn this feature on always (without providing
+ // enable/disable option on GUI).
+ // previously created setting should turn this feature on here.
+ interpreterSetting.getOption().setRemote(true);
+ interpreterSetting.convertPermissionsFromUsersToOwners(
+ jsonObject.getAsJsonObject("interpreterSettings")
+ .getAsJsonObject(interpreterSetting.getId()));
+ }
+ }
+ }
+ return infoSaving == null ? new InterpreterInfoSaving() : infoSaving;
+ }
+
+ public void saveToFile(Path file) throws IOException {
+ if (!Files.exists(file)) {
+ Files.createFile(file);
+ try {
+ Set<PosixFilePermission> permissions = EnumSet.of(OWNER_READ, OWNER_WRITE);
+ Files.setPosixFilePermissions(file, permissions);
+ } catch (UnsupportedOperationException e) {
+ // File system does not support Posix file permissions (likely windows) - continue anyway.
+ LOGGER.warn("unable to setPosixFilePermissions on '{}'.", file);
+ };
+ }
+ LOGGER.info("Save Interpreter Settings to " + file);
+ IOUtils.write(this.toJson(), new FileOutputStream(file.toFile()));
+ }
+
+ public String toJson() {
+ return gson.toJson(this);
+ }
+
+ public static InterpreterInfoSaving fromJson(String json) {
+ return gson.fromJson(json, InterpreterInfoSaving.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java
index 0bb3d42..ac40de3 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java
@@ -34,6 +34,7 @@ public class InterpreterProperty {
public InterpreterProperty(String name, Object value) {
this.name = name;
this.value = value;
+ this.type = "textarea";
}
public String getName() {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java
index 020564b..ae7bb88 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java
@@ -1,5 +1,6 @@
package org.apache.zeppelin.interpreter;
+import com.google.common.annotations.VisibleForTesting;
import com.google.gson.annotations.SerializedName;
/**
@@ -12,6 +13,16 @@ public class InterpreterRunner {
@SerializedName("win")
private String winPath;
+ public InterpreterRunner() {
+
+ }
+
+ @VisibleForTesting
+ public InterpreterRunner(String linuxPath, String winPath) {
+ this.linuxPath = linuxPath;
+ this.winPath = winPath;
+ }
+
public String getPath() {
return System.getProperty("os.name").startsWith("Windows") ? winPath : linuxPath;
}
[04/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
deleted file mode 100644
index 12e0caa..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
+++ /dev/null
@@ -1,597 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote;
-
-import java.util.*;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.thrift.TException;
-import org.apache.zeppelin.display.AngularObject;
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.display.GUI;
-import org.apache.zeppelin.helium.ApplicationEventListener;
-import org.apache.zeppelin.display.Input;
-import org.apache.zeppelin.interpreter.*;
-import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
-import org.apache.zeppelin.scheduler.Scheduler;
-import org.apache.zeppelin.scheduler.SchedulerFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-
-/**
- * Proxy for Interpreter instance that runs on separate process
- */
-public class RemoteInterpreter extends Interpreter {
- private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreter.class);
-
- private final RemoteInterpreterProcessListener remoteInterpreterProcessListener;
- private final ApplicationEventListener applicationEventListener;
- private Gson gson = new Gson();
- private String interpreterRunner;
- private String interpreterPath;
- private String localRepoPath;
- private String className;
- private String sessionKey;
- private FormType formType;
- private boolean initialized;
- private Map<String, String> env;
- private int connectTimeout;
- private int maxPoolSize;
- private String host;
- private int port;
- private String userName;
- private Boolean isUserImpersonate;
- private int outputLimit = Constants.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT;
- private String interpreterGroupName;
-
- /**
- * Remote interpreter and manage interpreter process
- */
- public RemoteInterpreter(Properties property, String sessionKey, String className,
- String interpreterRunner, String interpreterPath, String localRepoPath, int connectTimeout,
- int maxPoolSize, RemoteInterpreterProcessListener remoteInterpreterProcessListener,
- ApplicationEventListener appListener, String userName, Boolean isUserImpersonate,
- int outputLimit, String interpreterGroupName) {
- super(property);
- this.sessionKey = sessionKey;
- this.className = className;
- initialized = false;
- this.interpreterRunner = interpreterRunner;
- this.interpreterPath = interpreterPath;
- this.localRepoPath = localRepoPath;
- env = getEnvFromInterpreterProperty(property);
- this.connectTimeout = connectTimeout;
- this.maxPoolSize = maxPoolSize;
- this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
- this.applicationEventListener = appListener;
- this.userName = userName;
- this.isUserImpersonate = isUserImpersonate;
- this.outputLimit = outputLimit;
- this.interpreterGroupName = interpreterGroupName;
- }
-
-
- /**
- * Connect to existing process
- */
- public RemoteInterpreter(Properties property, String sessionKey, String className, String host,
- int port, String localRepoPath, int connectTimeout, int maxPoolSize,
- RemoteInterpreterProcessListener remoteInterpreterProcessListener,
- ApplicationEventListener appListener, String userName, Boolean isUserImpersonate,
- int outputLimit) {
- super(property);
- this.sessionKey = sessionKey;
- this.className = className;
- initialized = false;
- this.host = host;
- this.port = port;
- this.localRepoPath = localRepoPath;
- this.connectTimeout = connectTimeout;
- this.maxPoolSize = maxPoolSize;
- this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
- this.applicationEventListener = appListener;
- this.userName = userName;
- this.isUserImpersonate = isUserImpersonate;
- this.outputLimit = outputLimit;
- }
-
-
- // VisibleForTesting
- public RemoteInterpreter(Properties property, String sessionKey, String className,
- String interpreterRunner, String interpreterPath, String localRepoPath,
- Map<String, String> env, int connectTimeout,
- RemoteInterpreterProcessListener remoteInterpreterProcessListener,
- ApplicationEventListener appListener, String userName, Boolean isUserImpersonate) {
- super(property);
- this.className = className;
- this.sessionKey = sessionKey;
- this.interpreterRunner = interpreterRunner;
- this.interpreterPath = interpreterPath;
- this.localRepoPath = localRepoPath;
- env.putAll(getEnvFromInterpreterProperty(property));
- this.env = env;
- this.connectTimeout = connectTimeout;
- this.maxPoolSize = 10;
- this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
- this.applicationEventListener = appListener;
- this.userName = userName;
- this.isUserImpersonate = isUserImpersonate;
- }
-
- private Map<String, String> getEnvFromInterpreterProperty(Properties property) {
- Map<String, String> env = new HashMap<String, String>();
- StringBuilder sparkConfBuilder = new StringBuilder();
- for (String key : property.stringPropertyNames()) {
- if (RemoteInterpreterUtils.isEnvString(key)) {
- env.put(key, property.getProperty(key));
- }
- if (key.equals("master")) {
- sparkConfBuilder.append(" --master " + property.getProperty("master"));
- }
- if (isSparkConf(key, property.getProperty(key))) {
- sparkConfBuilder.append(" --conf " + key + "=" +
- toShellFormat(property.getProperty(key)));
- }
- }
- env.put("ZEPPELIN_SPARK_CONF", sparkConfBuilder.toString());
- return env;
- }
-
- private String toShellFormat(String value) {
- if (value.contains("\'") && value.contains("\"")) {
- throw new RuntimeException("Spark property value could not contain both \" and '");
- } else if (value.contains("\'")) {
- return "\"" + value + "\"";
- } else {
- return "\'" + value + "\'";
- }
- }
-
- static boolean isSparkConf(String key, String value) {
- return !StringUtils.isEmpty(key) && key.startsWith("spark.") && !StringUtils.isEmpty(value);
- }
-
- @Override
- public String getClassName() {
- return className;
- }
-
- private boolean connectToExistingProcess() {
- return host != null && port > 0;
- }
-
- public RemoteInterpreterProcess getInterpreterProcess() {
- InterpreterGroup intpGroup = getInterpreterGroup();
- if (intpGroup == null) {
- return null;
- }
-
- synchronized (intpGroup) {
- if (intpGroup.getRemoteInterpreterProcess() == null) {
- RemoteInterpreterProcess remoteProcess;
- if (connectToExistingProcess()) {
- remoteProcess = new RemoteInterpreterRunningProcess(
- connectTimeout,
- remoteInterpreterProcessListener,
- applicationEventListener,
- host,
- port);
- } else {
- // create new remote process
- remoteProcess = new RemoteInterpreterManagedProcess(
- interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout,
- remoteInterpreterProcessListener, applicationEventListener, interpreterGroupName);
- }
-
- intpGroup.setRemoteInterpreterProcess(remoteProcess);
- }
-
- return intpGroup.getRemoteInterpreterProcess();
- }
- }
-
- public synchronized void init() {
- if (initialized == true) {
- return;
- }
-
- RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
-
- final InterpreterGroup interpreterGroup = getInterpreterGroup();
-
- interpreterProcess.setMaxPoolSize(
- Math.max(this.maxPoolSize, interpreterProcess.getMaxPoolSize()));
- String groupId = interpreterGroup.getId();
-
- synchronized (interpreterProcess) {
- Client client = null;
- try {
- client = interpreterProcess.getClient();
- } catch (Exception e1) {
- throw new InterpreterException(e1);
- }
-
- boolean broken = false;
- try {
- logger.info("Create remote interpreter {}", getClassName());
- if (localRepoPath != null) {
- property.put("zeppelin.interpreter.localRepo", localRepoPath);
- }
-
- property.put("zeppelin.interpreter.output.limit", Integer.toString(outputLimit));
- client.createInterpreter(groupId, sessionKey,
- getClassName(), (Map) property, userName);
- // Push angular object loaded from JSON file to remote interpreter
- if (!interpreterGroup.isAngularRegistryPushed()) {
- pushAngularObjectRegistryToRemote(client);
- interpreterGroup.setAngularRegistryPushed(true);
- }
-
- } catch (TException e) {
- logger.error("Failed to create interpreter: {}", getClassName());
- throw new InterpreterException(e);
- } finally {
- // TODO(jongyoul): Fixed it when not all of interpreter in same interpreter group are broken
- interpreterProcess.releaseClient(client, broken);
- }
- }
- initialized = true;
- }
-
-
- @Override
- public void open() {
- InterpreterGroup interpreterGroup = getInterpreterGroup();
-
- synchronized (interpreterGroup) {
- // initialize all interpreters in this interpreter group
- List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
- // TODO(jl): this open method is called by LazyOpenInterpreter.open(). It, however,
- // initializes all of interpreters with same sessionKey. But LazyOpenInterpreter assumes if it
- // doesn't call open method, it's not open. It causes problem while running intp.close()
- // In case of Spark, this method initializes all of interpreters and init() method increases
- // reference count of RemoteInterpreterProcess. But while closing this interpreter group, all
- // other interpreters doesn't do anything because those LazyInterpreters aren't open.
- // But for now, we have to initialise all of interpreters for some reasons.
- // See Interpreter.getInterpreterInTheSameSessionByClassName(String)
- RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
- if (!initialized) {
- // reference per session
- interpreterProcess.reference(interpreterGroup, userName, isUserImpersonate);
- }
- for (Interpreter intp : new ArrayList<>(interpreters)) {
- Interpreter p = intp;
- while (p instanceof WrappedInterpreter) {
- p = ((WrappedInterpreter) p).getInnerInterpreter();
- }
- try {
- ((RemoteInterpreter) p).init();
- } catch (InterpreterException e) {
- logger.error("Failed to initialize interpreter: {}. Remove it from interpreterGroup",
- p.getClassName());
- interpreters.remove(p);
- }
- }
- }
- }
-
- @Override
- public void close() {
- InterpreterGroup interpreterGroup = getInterpreterGroup();
- synchronized (interpreterGroup) {
- // close all interpreters in this session
- List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
- // TODO(jl): this open method is called by LazyOpenInterpreter.open(). It, however,
- // initializes all of interpreters with same sessionKey. But LazyOpenInterpreter assumes if it
- // doesn't call open method, it's not open. It causes problem while running intp.close()
- // In case of Spark, this method initializes all of interpreters and init() method increases
- // reference count of RemoteInterpreterProcess. But while closing this interpreter group, all
- // other interpreters doesn't do anything because those LazyInterpreters aren't open.
- // But for now, we have to initialise all of interpreters for some reasons.
- // See Interpreter.getInterpreterInTheSameSessionByClassName(String)
- if (initialized) {
- // dereference per session
- getInterpreterProcess().dereference();
- }
- for (Interpreter intp : new ArrayList<>(interpreters)) {
- Interpreter p = intp;
- while (p instanceof WrappedInterpreter) {
- p = ((WrappedInterpreter) p).getInnerInterpreter();
- }
- try {
- ((RemoteInterpreter) p).closeInterpreter();
- } catch (InterpreterException e) {
- logger.error("Failed to initialize interpreter: {}. Remove it from interpreterGroup",
- p.getClassName());
- interpreters.remove(p);
- }
- }
- }
- }
-
- public void closeInterpreter() {
- if (this.initialized == false) {
- return;
- }
- RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
- Client client = null;
- boolean broken = false;
- try {
- client = interpreterProcess.getClient();
- if (client != null) {
- client.close(sessionKey, className);
- }
- } catch (TException e) {
- broken = true;
- throw new InterpreterException(e);
- } catch (Exception e1) {
- throw new InterpreterException(e1);
- } finally {
- if (client != null) {
- interpreterProcess.releaseClient(client, broken);
- }
- this.initialized = false;
- }
- }
-
- @Override
- public InterpreterResult interpret(String st, InterpreterContext context) {
- if (logger.isDebugEnabled()) {
- logger.debug("st:\n{}", st);
- }
-
- FormType form = getFormType();
- RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
- Client client = null;
- try {
- client = interpreterProcess.getClient();
- } catch (Exception e1) {
- throw new InterpreterException(e1);
- }
-
- InterpreterContextRunnerPool interpreterContextRunnerPool = interpreterProcess
- .getInterpreterContextRunnerPool();
-
- List<InterpreterContextRunner> runners = context.getRunners();
- if (runners != null && runners.size() != 0) {
- // assume all runners in this InterpreterContext have the same note id
- String noteId = runners.get(0).getNoteId();
-
- interpreterContextRunnerPool.clear(noteId);
- interpreterContextRunnerPool.addAll(noteId, runners);
- }
-
- boolean broken = false;
- try {
-
- final GUI currentGUI = context.getGui();
- RemoteInterpreterResult remoteResult = client.interpret(
- sessionKey, className, st, convert(context));
-
- Map<String, Object> remoteConfig = (Map<String, Object>) gson.fromJson(
- remoteResult.getConfig(), new TypeToken<Map<String, Object>>() {
- }.getType());
- context.getConfig().clear();
- context.getConfig().putAll(remoteConfig);
-
- if (form == FormType.NATIVE) {
- GUI remoteGui = GUI.fromJson(remoteResult.getGui());
- currentGUI.clear();
- currentGUI.setParams(remoteGui.getParams());
- currentGUI.setForms(remoteGui.getForms());
- } else if (form == FormType.SIMPLE) {
- final Map<String, Input> currentForms = currentGUI.getForms();
- final Map<String, Object> currentParams = currentGUI.getParams();
- final GUI remoteGUI = GUI.fromJson(remoteResult.getGui());
- final Map<String, Input> remoteForms = remoteGUI.getForms();
- final Map<String, Object> remoteParams = remoteGUI.getParams();
- currentForms.putAll(remoteForms);
- currentParams.putAll(remoteParams);
- }
-
- InterpreterResult result = convert(remoteResult);
- return result;
- } catch (TException e) {
- broken = true;
- throw new InterpreterException(e);
- } finally {
- interpreterProcess.releaseClient(client, broken);
- }
- }
-
- @Override
- public void cancel(InterpreterContext context) {
- RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
- Client client = null;
- try {
- client = interpreterProcess.getClient();
- } catch (Exception e1) {
- throw new InterpreterException(e1);
- }
-
- boolean broken = false;
- try {
- client.cancel(sessionKey, className, convert(context));
- } catch (TException e) {
- broken = true;
- throw new InterpreterException(e);
- } finally {
- interpreterProcess.releaseClient(client, broken);
- }
- }
-
- @Override
- public FormType getFormType() {
- open();
-
- if (formType != null) {
- return formType;
- }
-
- RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
- Client client = null;
- try {
- client = interpreterProcess.getClient();
- } catch (Exception e1) {
- throw new InterpreterException(e1);
- }
-
- boolean broken = false;
- try {
- formType = FormType.valueOf(client.getFormType(sessionKey, className));
- return formType;
- } catch (TException e) {
- broken = true;
- throw new InterpreterException(e);
- } finally {
- interpreterProcess.releaseClient(client, broken);
- }
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
- if (interpreterProcess == null || !interpreterProcess.isRunning()) {
- return 0;
- }
-
- Client client = null;
- try {
- client = interpreterProcess.getClient();
- } catch (Exception e1) {
- throw new InterpreterException(e1);
- }
-
- boolean broken = false;
- try {
- return client.getProgress(sessionKey, className, convert(context));
- } catch (TException e) {
- broken = true;
- throw new InterpreterException(e);
- } finally {
- interpreterProcess.releaseClient(client, broken);
- }
- }
-
-
- @Override
- public List<InterpreterCompletion> completion(String buf, int cursor,
- InterpreterContext interpreterContext) {
- RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
- Client client = null;
- try {
- client = interpreterProcess.getClient();
- } catch (Exception e1) {
- throw new InterpreterException(e1);
- }
-
- boolean broken = false;
- try {
- List completion = client.completion(sessionKey, className, buf, cursor,
- convert(interpreterContext));
- return completion;
- } catch (TException e) {
- broken = true;
- throw new InterpreterException(e);
- } finally {
- interpreterProcess.releaseClient(client, broken);
- }
- }
-
- @Override
- public Scheduler getScheduler() {
- int maxConcurrency = maxPoolSize;
- RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
- if (interpreterProcess == null) {
- return null;
- } else {
- return SchedulerFactory.singleton().createOrGetRemoteScheduler(
- RemoteInterpreter.class.getName() + sessionKey + interpreterProcess.hashCode(),
- sessionKey, interpreterProcess, maxConcurrency);
- }
- }
-
- private String getInterpreterGroupKey(InterpreterGroup interpreterGroup) {
- return interpreterGroup.getId();
- }
-
- private RemoteInterpreterContext convert(InterpreterContext ic) {
- return new RemoteInterpreterContext(ic.getNoteId(), ic.getParagraphId(), ic.getReplName(),
- ic.getParagraphTitle(), ic.getParagraphText(), ic.getAuthenticationInfo().toJson(),
- gson.toJson(ic.getConfig()), ic.getGui().toJson(), gson.toJson(ic.getRunners()));
- }
-
- private InterpreterResult convert(RemoteInterpreterResult result) {
- InterpreterResult r = new InterpreterResult(
- InterpreterResult.Code.valueOf(result.getCode()));
-
- for (RemoteInterpreterResultMessage m : result.getMsg()) {
- r.add(InterpreterResult.Type.valueOf(m.getType()), m.getData());
- }
-
- return r;
- }
-
- /**
- * Push local angular object registry to
- * remote interpreter. This method should be
- * call ONLY inside the init() method
- */
- void pushAngularObjectRegistryToRemote(Client client) throws TException {
- final AngularObjectRegistry angularObjectRegistry = this.getInterpreterGroup()
- .getAngularObjectRegistry();
-
- if (angularObjectRegistry != null && angularObjectRegistry.getRegistry() != null) {
- final Map<String, Map<String, AngularObject>> registry = angularObjectRegistry
- .getRegistry();
-
- logger.info("Push local angular object registry from ZeppelinServer to" +
- " remote interpreter group {}", this.getInterpreterGroup().getId());
-
- final java.lang.reflect.Type registryType = new TypeToken<Map<String,
- Map<String, AngularObject>>>() {
- }.getType();
-
- Gson gson = new Gson();
- client.angularRegistryPush(gson.toJson(registry, registryType));
- }
- }
-
- public Map<String, String> getEnv() {
- return env;
- }
-
- public void addEnv(Map<String, String> env) {
- if (this.env == null) {
- this.env = new HashMap<>();
- }
- this.env.putAll(env);
- }
-
- //Only for test
- public String getInterpreterRunner() {
- return interpreterRunner;
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
deleted file mode 100644
index 1fb9b90..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote;
-
-import org.apache.commons.exec.*;
-import org.apache.commons.exec.environment.EnvironmentUtils;
-import org.apache.zeppelin.helium.ApplicationEventListener;
-import org.apache.zeppelin.interpreter.InterpreterException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Map;
-
-/**
- * This class manages start / stop of remote interpreter process
- */
-public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
- implements ExecuteResultHandler {
- private static final Logger logger = LoggerFactory.getLogger(
- RemoteInterpreterManagedProcess.class);
- private final String interpreterRunner;
-
- private DefaultExecutor executor;
- private ExecuteWatchdog watchdog;
- boolean running = false;
- private int port = -1;
- private final String interpreterDir;
- private final String localRepoDir;
- private final String interpreterGroupName;
-
- private Map<String, String> env;
-
- public RemoteInterpreterManagedProcess(
- String intpRunner,
- String intpDir,
- String localRepoDir,
- Map<String, String> env,
- int connectTimeout,
- RemoteInterpreterProcessListener listener,
- ApplicationEventListener appListener,
- String interpreterGroupName) {
- super(new RemoteInterpreterEventPoller(listener, appListener),
- connectTimeout);
- this.interpreterRunner = intpRunner;
- this.env = env;
- this.interpreterDir = intpDir;
- this.localRepoDir = localRepoDir;
- this.interpreterGroupName = interpreterGroupName;
- }
-
- RemoteInterpreterManagedProcess(String intpRunner,
- String intpDir,
- String localRepoDir,
- Map<String, String> env,
- RemoteInterpreterEventPoller remoteInterpreterEventPoller,
- int connectTimeout,
- String interpreterGroupName) {
- super(remoteInterpreterEventPoller,
- connectTimeout);
- this.interpreterRunner = intpRunner;
- this.env = env;
- this.interpreterDir = intpDir;
- this.localRepoDir = localRepoDir;
- this.interpreterGroupName = interpreterGroupName;
- }
-
- @Override
- public String getHost() {
- return "localhost";
- }
-
- @Override
- public int getPort() {
- return port;
- }
-
- @Override
- public void start(String userName, Boolean isUserImpersonate) {
- // start server process
- try {
- port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces();
- } catch (IOException e1) {
- throw new InterpreterException(e1);
- }
-
- CommandLine cmdLine = CommandLine.parse(interpreterRunner);
- cmdLine.addArgument("-d", false);
- cmdLine.addArgument(interpreterDir, false);
- cmdLine.addArgument("-p", false);
- cmdLine.addArgument(Integer.toString(port), false);
- if (isUserImpersonate && !userName.equals("anonymous")) {
- cmdLine.addArgument("-u", false);
- cmdLine.addArgument(userName, false);
- }
- cmdLine.addArgument("-l", false);
- cmdLine.addArgument(localRepoDir, false);
- cmdLine.addArgument("-g", false);
- cmdLine.addArgument(interpreterGroupName, false);
-
- executor = new DefaultExecutor();
-
- ByteArrayOutputStream cmdOut = new ByteArrayOutputStream();
- ProcessLogOutputStream processOutput = new ProcessLogOutputStream(logger);
- processOutput.setOutputStream(cmdOut);
-
- executor.setStreamHandler(new PumpStreamHandler(processOutput));
- watchdog = new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT);
- executor.setWatchdog(watchdog);
-
- try {
- Map procEnv = EnvironmentUtils.getProcEnvironment();
- procEnv.putAll(env);
-
- logger.info("Run interpreter process {}", cmdLine);
- executor.execute(cmdLine, procEnv, this);
- running = true;
- } catch (IOException e) {
- running = false;
- throw new InterpreterException(e);
- }
-
-
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() - startTime < getConnectTimeout()) {
- if (!running) {
- try {
- cmdOut.flush();
- } catch (IOException e) {
- // nothing to do
- }
- throw new InterpreterException(new String(cmdOut.toByteArray()));
- }
-
- try {
- if (RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", port)) {
- break;
- } else {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- logger.error("Exception in RemoteInterpreterProcess while synchronized reference " +
- "Thread.sleep", e);
- }
- }
- } catch (Exception e) {
- if (logger.isDebugEnabled()) {
- logger.debug("Remote interpreter not yet accessible at localhost:" + port);
- }
- }
- }
- processOutput.setOutputStream(null);
- }
-
- public void stop() {
- if (isRunning()) {
- logger.info("kill interpreter process");
- watchdog.destroyProcess();
- }
-
- executor = null;
- watchdog = null;
- running = false;
- logger.info("Remote process terminated");
- }
-
- @Override
- public void onProcessComplete(int exitValue) {
- logger.info("Interpreter process exited {}", exitValue);
- running = false;
-
- }
-
- @Override
- public void onProcessFailed(ExecuteException e) {
- logger.info("Interpreter process failed {}", e);
- running = false;
- }
-
- public boolean isRunning() {
- return running;
- }
-
- private static class ProcessLogOutputStream extends LogOutputStream {
-
- private Logger logger;
- OutputStream out;
-
- public ProcessLogOutputStream(Logger logger) {
- this.logger = logger;
- }
-
- @Override
- protected void processLine(String s, int i) {
- this.logger.debug(s);
- }
-
- @Override
- public void write(byte [] b) throws IOException {
- super.write(b);
-
- if (out != null) {
- synchronized (this) {
- if (out != null) {
- out.write(b);
- }
- }
- }
- }
-
- @Override
- public void write(byte [] b, int offset, int len) throws IOException {
- super.write(b, offset, len);
-
- if (out != null) {
- synchronized (this) {
- if (out != null) {
- out.write(b, offset, len);
- }
- }
- }
- }
-
- public void setOutputStream(OutputStream out) {
- synchronized (this) {
- this.out = out;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java
deleted file mode 100644
index bb176be..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.zeppelin.interpreter.remote;
-
-import org.apache.zeppelin.helium.ApplicationEventListener;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class connects to existing process
- */
-public class RemoteInterpreterRunningProcess extends RemoteInterpreterProcess {
- private final Logger logger = LoggerFactory.getLogger(RemoteInterpreterRunningProcess.class);
- private final String host;
- private final int port;
-
- public RemoteInterpreterRunningProcess(
- int connectTimeout,
- RemoteInterpreterProcessListener listener,
- ApplicationEventListener appListener,
- String host,
- int port
- ) {
- super(connectTimeout, listener, appListener);
- this.host = host;
- this.port = port;
- }
-
- @Override
- public String getHost() {
- return host;
- }
-
- @Override
- public int getPort() {
- return port;
- }
-
- @Override
- public void start(String userName, Boolean isUserImpersonate) {
- // assume process is externally managed. nothing to do
- }
-
- @Override
- public void stop() {
- // assume process is externally managed. nothing to do
- }
-
- @Override
- public boolean isRunning() {
- return RemoteInterpreterUtils.checkIfRemoteEndpointAccessible(getHost(), getPort());
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java
index 1505db9..bc71d89 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java
@@ -17,7 +17,6 @@
package org.apache.zeppelin.notebook;
import org.apache.zeppelin.helium.HeliumPackage;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
/**
* Current state of application
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
index 198e278..4a93d08 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
@@ -41,7 +41,6 @@ import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.utility.IdHashes;
-import org.apache.zeppelin.resource.ResourcePoolUtils;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.search.SearchService;
@@ -126,11 +125,6 @@ public class Note implements ParagraphJobListener, JsonSerializable {
id = IdHashes.generateId();
}
- private String getDefaultInterpreterName() {
- InterpreterSetting setting = interpreterSettingManager.getDefaultInterpreterSetting(getId());
- return null != setting ? setting.getName() : StringUtils.EMPTY;
- }
-
public boolean isPersonalizedMode() {
Object v = getConfig().get("personalizedMode");
return null != v && "true".equals(v);
@@ -385,7 +379,7 @@ public class Note implements ParagraphJobListener, JsonSerializable {
*/
public Paragraph removeParagraph(String user, String paragraphId) {
removeAllAngularObjectInParagraph(user, paragraphId);
- ResourcePoolUtils.removeResourcesBelongsToParagraph(getId(), paragraphId);
+ interpreterSettingManager.removeResourcesBelongsToParagraph(getId(), paragraphId);
synchronized (paragraphs) {
Iterator<Paragraph> i = paragraphs.iterator();
while (i.hasNext()) {
@@ -690,7 +684,7 @@ public class Note implements ParagraphJobListener, JsonSerializable {
}
for (InterpreterSetting setting : settings) {
- InterpreterGroup intpGroup = setting.getInterpreterGroup(user, id);
+ InterpreterGroup intpGroup = setting.getOrCreateInterpreterGroup(user, id);
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
angularObjects.put(intpGroup.getId(), registry.getAllWithGlobal(id));
}
@@ -705,7 +699,7 @@ public class Note implements ParagraphJobListener, JsonSerializable {
}
for (InterpreterSetting setting : settings) {
- InterpreterGroup intpGroup = setting.getInterpreterGroup(user, id);
+ InterpreterGroup intpGroup = setting.getOrCreateInterpreterGroup(user, id);
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
if (registry instanceof RemoteAngularObjectRegistry) {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
index a0c1dff..fd3111b 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
@@ -60,7 +60,6 @@ import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.repo.NotebookRepo.Revision;
import org.apache.zeppelin.notebook.repo.NotebookRepoSync;
-import org.apache.zeppelin.resource.ResourcePoolUtils;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.search.SearchService;
@@ -140,7 +139,7 @@ public class Notebook implements NoteEventListener {
Preconditions.checkNotNull(subject, "AuthenticationInfo should not be null");
Note note;
if (conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING)) {
- note = createNote(interpreterSettingManager.getDefaultInterpreterSettingList(), subject);
+ note = createNote(interpreterSettingManager.getInterpreterSettingIds(), subject);
} else {
note = createNote(null, subject);
}
@@ -270,8 +269,8 @@ public class Notebook implements NoteEventListener {
}
}
- interpreterSettingManager.setInterpreters(user, note.getId(), interpreterSettingIds);
- // comment out while note.getNoteReplLoader().setInterpreters(...) do the same
+ interpreterSettingManager.setInterpreterBinding(user, note.getId(), interpreterSettingIds);
+ // comment out while note.getNoteReplLoader().setInterpreterBinding(...) do the same
// replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds);
}
}
@@ -279,7 +278,7 @@ public class Notebook implements NoteEventListener {
List<String> getBindedInterpreterSettingsIds(String id) {
Note note = getNote(id);
if (note != null) {
- return interpreterSettingManager.getInterpreters(note.getId());
+ return interpreterSettingManager.getInterpreterBinding(note.getId());
} else {
return new LinkedList<>();
}
@@ -313,9 +312,10 @@ public class Notebook implements NoteEventListener {
}
public void moveNoteToTrash(String noteId) {
- for (InterpreterSetting interpreterSetting : interpreterSettingManager
- .getInterpreterSettings(noteId)) {
- interpreterSettingManager.removeInterpretersForNote(interpreterSetting, "", noteId);
+ try {
+ interpreterSettingManager.setInterpreterBinding("", noteId, new ArrayList<String>());
+ } catch (IOException e) {
+ e.printStackTrace();
}
}
@@ -339,7 +339,7 @@ public class Notebook implements NoteEventListener {
// remove from all interpreter instance's angular object registry
for (InterpreterSetting settings : interpreterSettingManager.get()) {
AngularObjectRegistry registry =
- settings.getInterpreterGroup(subject.getUser(), id).getAngularObjectRegistry();
+ settings.getOrCreateInterpreterGroup(subject.getUser(), id).getAngularObjectRegistry();
if (registry instanceof RemoteAngularObjectRegistry) {
// remove paragraph scope object
for (Paragraph p : note.getParagraphs()) {
@@ -374,7 +374,7 @@ public class Notebook implements NoteEventListener {
}
}
- ResourcePoolUtils.removeResourcesBelongsToNote(id);
+ interpreterSettingManager.removeResourcesBelongsToNote(id);
fireNoteRemoveEvent(note);
@@ -521,7 +521,8 @@ public class Notebook implements NoteEventListener {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
List<InterpreterSetting> settings = interpreterSettingManager.get();
for (InterpreterSetting setting : settings) {
- InterpreterGroup intpGroup = setting.getInterpreterGroup(subject.getUser(), note.getId());
+ InterpreterGroup intpGroup = setting.getOrCreateInterpreterGroup(subject.getUser(),
+ note.getId());
if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
String noteId = snapshot.getAngularObject().getNoteId();
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
index 37138e6..bfe4566 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
@@ -93,10 +93,10 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
// since zeppelin-0.7.0, zeppelin stores multiple results of the paragraph
// see ZEPPELIN-212
- Object results;
+ volatile Object results;
// For backward compatibility of note.json format after ZEPPELIN-212
- Object result;
+ volatile Object result;
private Map<String, ParagraphRuntimeInfo> runtimeInfos;
/**
@@ -157,7 +157,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
}
@Override
- public void setResult(Object results) {
+ public synchronized void setResult(Object results) {
this.results = results;
}
@@ -354,7 +354,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
}
@Override
- public Object getReturn() {
+ public synchronized Object getReturn() {
return results;
}
@@ -401,6 +401,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
logger.error("Can not find interpreter name " + repl);
throw new RuntimeException("Can not find interpreter for " + getRequiredReplName());
}
+ //TODO(zjffdu) check interpreter setting status in interpreter setting itself
InterpreterSetting intp = getInterpreterSettingById(repl.getInterpreterGroup().getId());
while (intp.getStatus().equals(
org.apache.zeppelin.interpreter.InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES)) {
@@ -560,8 +561,10 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) {
InterpreterSetting intpGroup =
interpreterSettingManager.getInterpreterSettings(note.getId()).get(0);
- registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
- resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
+ registry = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
+ .getAngularObjectRegistry();
+ resourcePool = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
+ .getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<>();
@@ -591,8 +594,10 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) {
InterpreterSetting intpGroup =
interpreterSettingManager.getInterpreterSettings(note.getId()).get(0);
- registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
- resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
+ registry = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
+ .getAngularObjectRegistry();
+ resourcePool = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
+ .getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<>();
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java
deleted file mode 100644
index be45b9e..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.util;
-
-import org.apache.commons.lang3.StringUtils;
-
-import java.io.IOException;
-import java.util.Properties;
-
-/**
- * TODO(moon) : add description.
- */
-public class Util {
- private static final String PROJECT_PROPERTIES_VERSION_KEY = "version";
- private static final String GIT_PROPERTIES_COMMIT_ID_KEY = "git.commit.id.abbrev";
- private static final String GIT_PROPERTIES_COMMIT_TS_KEY = "git.commit.time";
-
- private static Properties projectProperties;
- private static Properties gitProperties;
-
- static {
- projectProperties = new Properties();
- gitProperties = new Properties();
- try {
- projectProperties.load(Util.class.getResourceAsStream("/project.properties"));
- gitProperties.load(Util.class.getResourceAsStream("/git.properties"));
- } catch (IOException e) {
- //Fail to read project.properties
- }
- }
-
- /**
- * Get Zeppelin version
- *
- * @return Current Zeppelin version
- */
- public static String getVersion() {
- return StringUtils.defaultIfEmpty(projectProperties.getProperty(PROJECT_PROPERTIES_VERSION_KEY),
- StringUtils.EMPTY);
- }
-
- /**
- * Get Zeppelin Git latest commit id
- *
- * @return Latest Zeppelin commit id
- */
- public static String getGitCommitId() {
- return StringUtils.defaultIfEmpty(gitProperties.getProperty(GIT_PROPERTIES_COMMIT_ID_KEY),
- StringUtils.EMPTY);
- }
-
- /**
- * Get Zeppelin Git latest commit timestamp
- *
- * @return Latest Zeppelin commit timestamp
- */
- public static String getGitTimestamp() {
- return StringUtils.defaultIfEmpty(gitProperties.getProperty(GIT_PROPERTIES_COMMIT_TS_KEY),
- StringUtils.EMPTY);
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
index 305258a..c204711 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
@@ -16,14 +16,14 @@
*/
package org.apache.zeppelin.helium;
-import com.google.common.collect.Maps;
import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
+import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.notebook.*;
import org.apache.zeppelin.notebook.repo.VFSNotebookRepo;
import org.apache.zeppelin.scheduler.Job;
@@ -45,14 +45,9 @@ import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
-public class HeliumApplicationFactoryTest implements JobListenerFactory {
- private File tmpDir;
- private File notebookDir;
- private ZeppelinConfiguration conf;
+public class HeliumApplicationFactoryTest extends AbstractInterpreterTest implements JobListenerFactory {
+
private SchedulerFactory schedulerFactory;
- private DependencyResolver depResolver;
- private InterpreterFactory factory;
- private InterpreterSettingManager interpreterSettingManager;
private VFSNotebookRepo notebookRepo;
private Notebook notebook;
private HeliumApplicationFactory heliumAppFactory;
@@ -60,46 +55,15 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
@Before
public void setUp() throws Exception {
- tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZepelinLTest_"+System.currentTimeMillis());
- tmpDir.mkdirs();
- File confDir = new File(tmpDir, "conf");
- confDir.mkdirs();
- notebookDir = new File(tmpDir + "/notebook");
- notebookDir.mkdirs();
-
- File home = new File(getClass().getClassLoader().getResource("note").getFile()) // zeppelin/zeppelin-zengine/target/test-classes/note
- .getParentFile() // zeppelin/zeppelin-zengine/target/test-classes
- .getParentFile() // zeppelin/zeppelin-zengine/target
- .getParentFile() // zeppelin/zeppelin-zengine
- .getParentFile(); // zeppelin
-
- System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), home.getAbsolutePath());
- System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), tmpDir.getAbsolutePath() + "/conf");
- System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
-
- conf = new ZeppelinConfiguration();
-
- this.schedulerFactory = new SchedulerFactory();
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "mock1,mock2");
+ super.setUp();
+ this.schedulerFactory = SchedulerFactory.singleton();
heliumAppFactory = new HeliumApplicationFactory();
- depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- factory = new InterpreterFactory(conf, null, null, heliumAppFactory, depResolver, false, interpreterSettingManager);
- HashMap<String, String> env = new HashMap<>();
- env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
- factory.setEnv(env);
-
- ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
- interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
- interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
-
- ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
- interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock2", interpreterInfos2, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock2", null);
- interpreterSettingManager.createNewSetting("mock2", "mock2", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
+ // set AppEventListener properly
+ for (InterpreterSetting interpreterSetting : interpreterSettingManager.get()) {
+ interpreterSetting.setAppEventListener(heliumAppFactory);
+ }
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
@@ -108,7 +72,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
conf,
notebookRepo,
schedulerFactory,
- factory,
+ interpreterFactory,
interpreterSettingManager,
this,
search,
@@ -124,16 +88,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
@After
public void tearDown() throws Exception {
- List<InterpreterSetting> settings = interpreterSettingManager.get();
- for (InterpreterSetting setting : settings) {
- for (InterpreterGroup intpGroup : setting.getAllInterpreterGroups()) {
- intpGroup.close();
- }
- }
-
- FileUtils.deleteDirectory(tmpDir);
- System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(),
- ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getStringValue());
+ super.tearDown();
}
@@ -150,7 +105,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters("user", note1.getId(),interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding("user", note1.getId(),interpreterSettingManager.getInterpreterSettingIds());
Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@@ -196,7 +151,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding("user", note1.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@@ -236,7 +191,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
- notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@@ -297,7 +252,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
- notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getInterpreterSettingIds());
String mock1IntpSettingId = null;
for (InterpreterSetting setting : notebook.getBindedInterpreterSettings(note1.getId())) {
if (setting.getName().equals("mock1")) {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java
index 6b4932d..bdd639e 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java
@@ -52,7 +52,7 @@ public class HeliumTest {
// given
File heliumConf = new File(tmpDir, "helium.conf");
Helium helium = new Helium(heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(),
- null, null, null);
+ null, null, null, null);
assertFalse(heliumConf.exists());
// when
@@ -63,14 +63,14 @@ public class HeliumTest {
// then load without exception
Helium heliumRestored = new Helium(
- heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null);
+ heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null, null);
}
@Test
public void testRestoreRegistryInstances() throws IOException, URISyntaxException, TaskRunnerException {
File heliumConf = new File(tmpDir, "helium.conf");
Helium helium = new Helium(
- heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null);
+ heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null, null);
HeliumTestRegistry registry1 = new HeliumTestRegistry("r1", "r1");
HeliumTestRegistry registry2 = new HeliumTestRegistry("r2", "r2");
helium.addRegistry(registry1);
@@ -105,7 +105,7 @@ public class HeliumTest {
public void testRefresh() throws IOException, URISyntaxException, TaskRunnerException {
File heliumConf = new File(tmpDir, "helium.conf");
Helium helium = new Helium(
- heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null);
+ heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null, null);
HeliumTestRegistry registry1 = new HeliumTestRegistry("r1", "r1");
helium.addRegistry(registry1);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
deleted file mode 100644
index aaa8864..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
+++ /dev/null
@@ -1,497 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.NullArgumentException;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
-import org.apache.zeppelin.dep.Dependency;
-import org.apache.zeppelin.dep.DependencyResolver;
-import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
-import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
-import org.apache.zeppelin.notebook.JobListenerFactory;
-import org.apache.zeppelin.notebook.Note;
-import org.apache.zeppelin.notebook.Notebook;
-import org.apache.zeppelin.notebook.NotebookAuthorization;
-import org.apache.zeppelin.notebook.repo.NotebookRepo;
-import org.apache.zeppelin.notebook.repo.VFSNotebookRepo;
-import org.apache.zeppelin.scheduler.SchedulerFactory;
-import org.apache.zeppelin.search.SearchService;
-import org.apache.zeppelin.user.AuthenticationInfo;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mock;
-import org.quartz.SchedulerException;
-import org.sonatype.aether.RepositoryException;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-public class InterpreterFactoryTest {
-
- private InterpreterFactory factory;
- private InterpreterSettingManager interpreterSettingManager;
- private File tmpDir;
- private ZeppelinConfiguration conf;
- private InterpreterContext context;
- private Notebook notebook;
- private NotebookRepo notebookRepo;
- private DependencyResolver depResolver;
- private SchedulerFactory schedulerFactory;
- private NotebookAuthorization notebookAuthorization;
- @Mock
- private JobListenerFactory jobListenerFactory;
-
- @Before
- public void setUp() throws Exception {
- tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
- tmpDir.mkdirs();
- new File(tmpDir, "conf").mkdirs();
- FileUtils.copyDirectory(new File("src/test/resources/interpreter"), new File(tmpDir, "interpreter"));
-
- System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
- System.setProperty(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(),
- "mock1,mock2,mock11,dev");
- conf = new ZeppelinConfiguration();
- schedulerFactory = new SchedulerFactory();
- depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
- context = new InterpreterContext("note", "id", null, "title", "text", null, null, null, null, null, null, null);
-
- ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
- interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
- Map<String, InterpreterProperty> intp1Properties = new HashMap<String, InterpreterProperty>();
- intp1Properties.put("PROPERTY_1",
- new InterpreterProperty("PROPERTY_1", "VALUE_1"));
- intp1Properties.put("property_2",
- new InterpreterProperty("property_2", "value_2"));
- interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), intp1Properties);
-
- ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
- interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock2", interpreterInfos2, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock2", null);
- interpreterSettingManager.createNewSetting("mock2", "mock2", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
-
- SearchService search = mock(SearchService.class);
- notebookRepo = new VFSNotebookRepo(conf);
- notebookAuthorization = NotebookAuthorization.init(conf);
- notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, jobListenerFactory, search,
- notebookAuthorization, null);
- }
-
- @After
- public void tearDown() throws Exception {
- FileUtils.deleteDirectory(tmpDir);
- }
-
- @Test
- public void testBasic() {
- List<InterpreterSetting> all = interpreterSettingManager.get();
- InterpreterSetting mock1Setting = null;
- for (InterpreterSetting setting : all) {
- if (setting.getName().equals("mock1")) {
- mock1Setting = setting;
- break;
- }
- }
-
-// mock1Setting = factory.createNewSetting("mock11", "mock1", new ArrayList<Dependency>(), new InterpreterOption(false), new Properties());
-
- InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("user", "sharedProcess");
- factory.createInterpretersForNote(mock1Setting, "user", "sharedProcess", "session");
-
- // get interpreter
- assertNotNull("get Interpreter", interpreterGroup.get("session").get(0));
-
- // try to get unavailable interpreter
- assertNull(interpreterSettingManager.get("unknown"));
-
- // restart interpreter
- interpreterSettingManager.restart(mock1Setting.getId());
- assertNull(mock1Setting.getInterpreterGroup("user", "sharedProcess").get("session"));
- }
-
- @Test
- public void testRemoteRepl() throws Exception {
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
- interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
- Map<String, InterpreterProperty> intp1Properties = new HashMap<String, InterpreterProperty>();
- intp1Properties.put("PROPERTY_1",
- new InterpreterProperty("PROPERTY_1", "VALUE_1"));
- intp1Properties.put("property_2", new InterpreterProperty("property_2", "value_2"));
- interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), intp1Properties);
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
- List<InterpreterSetting> all = interpreterSettingManager.get();
- InterpreterSetting mock1Setting = null;
- for (InterpreterSetting setting : all) {
- if (setting.getName().equals("mock1")) {
- mock1Setting = setting;
- break;
- }
- }
- InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("user", "sharedProcess");
- factory.createInterpretersForNote(mock1Setting, "user", "sharedProcess", "session");
- // get interpreter
- assertNotNull("get Interpreter", interpreterGroup.get("session").get(0));
- assertTrue(interpreterGroup.get("session").get(0) instanceof LazyOpenInterpreter);
- LazyOpenInterpreter lazyInterpreter = (LazyOpenInterpreter)(interpreterGroup.get("session").get(0));
- assertTrue(lazyInterpreter.getInnerInterpreter() instanceof RemoteInterpreter);
- RemoteInterpreter remoteInterpreter = (RemoteInterpreter) lazyInterpreter.getInnerInterpreter();
- assertEquals("VALUE_1", remoteInterpreter.getEnv().get("PROPERTY_1"));
- assertEquals("value_2", remoteInterpreter.getProperty("property_2"));
- }
-
- /**
- * 2 users' interpreters in scoped mode. Each user has one session. Restarting user1's interpreter
- * won't affect user2's interpreter
- * @throws Exception
- */
- @Test
- public void testRestartInterpreterInScopedMode() throws Exception {
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
- interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
- Map<String, InterpreterProperty> intp1Properties = new HashMap<String, InterpreterProperty>();
- intp1Properties.put("PROPERTY_1",
- new InterpreterProperty("PROPERTY_1", "VALUE_1"));
- intp1Properties.put("property_2",
- new InterpreterProperty("property_2", "value_2"));
- interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), intp1Properties);
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
- List<InterpreterSetting> all = interpreterSettingManager.get();
- InterpreterSetting mock1Setting = null;
- for (InterpreterSetting setting : all) {
- if (setting.getName().equals("mock1")) {
- mock1Setting = setting;
- break;
- }
- }
- mock1Setting.getOption().setPerUser("scoped");
- mock1Setting.getOption().setPerNote("shared");
- // set remote as false so that we won't create new remote interpreter process
- mock1Setting.getOption().setRemote(false);
- mock1Setting.getOption().setHost("localhost");
- mock1Setting.getOption().setPort(2222);
- InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("user1", "sharedProcess");
- factory.createInterpretersForNote(mock1Setting, "user1", "sharedProcess", "user1");
- factory.createInterpretersForNote(mock1Setting, "user2", "sharedProcess", "user2");
-
- LazyOpenInterpreter interpreter1 = (LazyOpenInterpreter)interpreterGroup.get("user1").get(0);
- interpreter1.open();
- LazyOpenInterpreter interpreter2 = (LazyOpenInterpreter)interpreterGroup.get("user2").get(0);
- interpreter2.open();
-
- mock1Setting.closeAndRemoveInterpreterGroup("sharedProcess", "user1");
- assertFalse(interpreter1.isOpen());
- assertTrue(interpreter2.isOpen());
- }
-
- /**
- * 2 users' interpreters in isolated mode. Each user has one interpreterGroup. Restarting user1's interpreter
- * won't affect user2's interpreter
- * @throws Exception
- */
- @Test
- public void testRestartInterpreterInIsolatedMode() throws Exception {
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
- interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
- Map<String, InterpreterProperty> intp1Properties = new HashMap<String, InterpreterProperty>();
- intp1Properties.put("PROPERTY_1",
- new InterpreterProperty("PROPERTY_1", "VALUE_1"));
- intp1Properties.put("property_2",
- new InterpreterProperty("property_2", "value_2"));
- interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), intp1Properties);
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
- List<InterpreterSetting> all = interpreterSettingManager.get();
- InterpreterSetting mock1Setting = null;
- for (InterpreterSetting setting : all) {
- if (setting.getName().equals("mock1")) {
- mock1Setting = setting;
- break;
- }
- }
- mock1Setting.getOption().setPerUser("isolated");
- mock1Setting.getOption().setPerNote("shared");
- // set remote as false so that we won't create new remote interpreter process
- mock1Setting.getOption().setRemote(false);
- mock1Setting.getOption().setHost("localhost");
- mock1Setting.getOption().setPort(2222);
- InterpreterGroup interpreterGroup1 = mock1Setting.getInterpreterGroup("user1", "note1");
- InterpreterGroup interpreterGroup2 = mock1Setting.getInterpreterGroup("user2", "note2");
- factory.createInterpretersForNote(mock1Setting, "user1", "note1", "shared_session");
- factory.createInterpretersForNote(mock1Setting, "user2", "note2", "shared_session");
-
- LazyOpenInterpreter interpreter1 = (LazyOpenInterpreter)interpreterGroup1.get("shared_session").get(0);
- interpreter1.open();
- LazyOpenInterpreter interpreter2 = (LazyOpenInterpreter)interpreterGroup2.get("shared_session").get(0);
- interpreter2.open();
-
- mock1Setting.closeAndRemoveInterpreterGroup("note1", "user1");
- assertFalse(interpreter1.isOpen());
- assertTrue(interpreter2.isOpen());
- }
-
- @Test
- public void testFactoryDefaultList() throws IOException, RepositoryException {
- // get default settings
- List<String> all = interpreterSettingManager.getDefaultInterpreterSettingList();
- assertTrue(interpreterSettingManager.get().size() >= all.size());
- }
-
- @Test
- public void testExceptions() throws InterpreterException, IOException, RepositoryException {
- List<String> all = interpreterSettingManager.getDefaultInterpreterSettingList();
- // add setting with null option & properties expected nullArgumentException.class
- try {
- interpreterSettingManager.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null);
- } catch(NullArgumentException e) {
- assertEquals("Test null option" , e.getMessage(),new NullArgumentException("option").getMessage());
- }
- try {
- interpreterSettingManager.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null);
- } catch (NullArgumentException e){
- assertEquals("Test null properties" , e.getMessage(),new NullArgumentException("properties").getMessage());
- }
- }
-
-
- @Test
- public void testSaveLoad() throws IOException, RepositoryException {
- // interpreter settings
- int numInterpreters = interpreterSettingManager.get().size();
-
- // check if file saved
- assertTrue(new File(conf.getInterpreterSettingPath()).exists());
-
- interpreterSettingManager.createNewSetting("new-mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new HashMap<String, InterpreterProperty>());
- assertEquals(numInterpreters + 1, interpreterSettingManager.get().size());
-
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
-
- /*
- Current situation, if InterpreterSettinfRef doesn't have the key of InterpreterSetting, it would be ignored.
- Thus even though interpreter.json have several interpreterSetting in that file, it would be ignored and would not be initialized from loadFromFile.
- In this case, only "mock11" would be referenced from file under interpreter/mock, and "mock11" group would be initialized.
- */
- // TODO(jl): Decide how to handle the know referenced interpreterSetting.
- assertEquals(1, interpreterSettingManager.get().size());
- }
-
- @Test
- public void testInterpreterSettingPropertyClass() throws IOException, RepositoryException {
- // check if default interpreter reference's property type is map
- Map<String, InterpreterSetting> interpreterSettingRefs = interpreterSettingManager.getAvailableInterpreterSettings();
- InterpreterSetting intpSetting = interpreterSettingRefs.get("mock1");
- Map<String, DefaultInterpreterProperty> intpProperties =
- (Map<String, DefaultInterpreterProperty>) intpSetting.getProperties();
- assertTrue(intpProperties instanceof Map);
-
- // check if interpreter instance is saved as Properties in conf/interpreter.json file
- Map<String, InterpreterProperty> properties = new HashMap<String, InterpreterProperty>();
- properties.put("key1", new InterpreterProperty("key1", "value1", "type1"));
- properties.put("key2", new InterpreterProperty("key2", "value2", "type2"));
-
- interpreterSettingManager.createNewSetting("newMock", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), properties);
-
- String confFilePath = conf.getInterpreterSettingPath();
- byte[] encoded = Files.readAllBytes(Paths.get(confFilePath));
- String json = new String(encoded, "UTF-8");
-
- InterpreterInfoSaving infoSaving = InterpreterInfoSaving.fromJson(json);
- Map<String, InterpreterSetting> interpreterSettings = infoSaving.interpreterSettings;
- for (String key : interpreterSettings.keySet()) {
- InterpreterSetting setting = interpreterSettings.get(key);
- if (setting.getName().equals("newMock")) {
- assertEquals(setting.getProperties().toString(), properties.toString());
- }
- }
- }
-
- @Test
- public void testInterpreterAliases() throws IOException, RepositoryException {
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
- final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null);
- final InterpreterInfo info2 = new InterpreterInfo("className2", "name1", true, null);
- interpreterSettingManager.add("group1", new ArrayList<InterpreterInfo>() {{
- add(info1);
- }}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path1", null);
- interpreterSettingManager.add("group2", new ArrayList<InterpreterInfo>(){{
- add(info2);
- }}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path2", null);
-
- final InterpreterSetting setting1 = interpreterSettingManager.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
- final InterpreterSetting setting2 = interpreterSettingManager.createNewSetting("test-group2", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
-
- interpreterSettingManager.setInterpreters("user", "note", new ArrayList<String>() {{
- add(setting1.getId());
- add(setting2.getId());
- }});
-
- assertEquals("className1", factory.getInterpreter("user1", "note", "test-group1").getClassName());
- assertEquals("className1", factory.getInterpreter("user1", "note", "group1").getClassName());
- }
-
- @Test
- public void testMultiUser() throws IOException, RepositoryException {
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- factory = new InterpreterFactory(conf, null, null, null, depResolver, true, interpreterSettingManager);
- final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null);
- interpreterSettingManager.add("group1", new ArrayList<InterpreterInfo>(){{
- add(info1);
- }}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path1", null);
-
- InterpreterOption perUserInterpreterOption = new InterpreterOption(true, InterpreterOption.ISOLATED, InterpreterOption.SHARED);
- final InterpreterSetting setting1 = interpreterSettingManager.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), perUserInterpreterOption, new HashMap<String, InterpreterProperty>());
-
- interpreterSettingManager.setInterpreters("user1", "note", new ArrayList<String>() {{
- add(setting1.getId());
- }});
-
- interpreterSettingManager.setInterpreters("user2", "note", new ArrayList<String>() {{
- add(setting1.getId());
- }});
-
- assertNotEquals(factory.getInterpreter("user1", "note", "test-group1"), factory.getInterpreter("user2", "note", "test-group1"));
- }
-
-
- @Test
- public void testInvalidInterpreterSettingName() {
- try {
- interpreterSettingManager.createNewSetting("new.mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new HashMap<String, InterpreterProperty>());
- fail("expect fail because of invalid InterpreterSetting Name");
- } catch (IOException e) {
- assertEquals("'.' is invalid for InterpreterSetting name.", e.getMessage());
- }
- }
-
-
- @Test
- public void getEditorSetting() throws IOException, RepositoryException, SchedulerException {
- List<String> intpIds = new ArrayList<>();
- for(InterpreterSetting intpSetting: interpreterSettingManager.get()) {
- if (intpSetting.getName().startsWith("mock1")) {
- intpIds.add(intpSetting.getId());
- }
- }
- Note note = notebook.createNote(intpIds, new AuthenticationInfo("anonymous"));
-
- Interpreter interpreter = factory.getInterpreter("user1", note.getId(), "mock11");
- // get editor setting from interpreter-setting.json
- Map<String, Object> editor = interpreterSettingManager.getEditorSetting(interpreter, "user1", note.getId(), "mock11");
- assertEquals("java", editor.get("language"));
-
- // when interpreter is not loaded via interpreter-setting.json
- // or editor setting doesn't exit
- editor = interpreterSettingManager.getEditorSetting(factory.getInterpreter("user1", note.getId(), "mock1"),"user1", note.getId(), "mock1");
- assertEquals(null, editor.get("language"));
-
- // when interpreter is not bound to note
- editor = interpreterSettingManager.getEditorSetting(factory.getInterpreter("user1", note.getId(), "mock11"),"user1", note.getId(), "mock2");
- assertEquals("text", editor.get("language"));
- }
-
- @Test
- public void registerCustomInterpreterRunner() throws IOException {
- InterpreterSettingManager spyInterpreterSettingManager = spy(interpreterSettingManager);
-
- doNothing().when(spyInterpreterSettingManager).saveToFile();
-
- ArrayList<InterpreterInfo> interpreterInfos1 = new ArrayList<>();
- interpreterInfos1.add(new InterpreterInfo("name1.class", "name1", true, Maps.<String, Object>newHashMap()));
-
- spyInterpreterSettingManager.add("normalGroup1", interpreterInfos1, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, DefaultInterpreterProperty>newHashMap(), "/normalGroup1", null);
-
- spyInterpreterSettingManager.createNewSetting("normalGroup1", "normalGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
-
- ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
- interpreterInfos2.add(new InterpreterInfo("name1.class", "name1", true, Maps.<String, Object>newHashMap()));
-
- InterpreterRunner mockInterpreterRunner = mock(InterpreterRunner.class);
-
- when(mockInterpreterRunner.getPath()).thenReturn("custom-linux-path.sh");
-
- spyInterpreterSettingManager.add("customGroup1", interpreterInfos2, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, DefaultInterpreterProperty>newHashMap(), "/customGroup1", mockInterpreterRunner);
-
- spyInterpreterSettingManager.createNewSetting("customGroup1", "customGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
-
- spyInterpreterSettingManager.setInterpreters("anonymous", "noteCustome", spyInterpreterSettingManager.getDefaultInterpreterSettingList());
-
- factory.getInterpreter("anonymous", "noteCustome", "customGroup1");
-
- verify(mockInterpreterRunner, times(1)).getPath();
- }
-
- @Test
- public void interpreterRunnerTest() {
- InterpreterRunner mockInterpreterRunner = mock(InterpreterRunner.class);
- String testInterpreterRunner = "relativePath.sh";
- when(mockInterpreterRunner.getPath()).thenReturn(testInterpreterRunner); // This test only for Linux
- Interpreter i = factory.createRemoteRepl("path1", "sessionKey", "className", new Properties(), interpreterSettingManager.get().get(0).getId(), "userName", false, mockInterpreterRunner);
- String interpreterRunner = ((RemoteInterpreter) ((LazyOpenInterpreter) i).getInnerInterpreter()).getInterpreterRunner();
- assertNotEquals(interpreterRunner, testInterpreterRunner);
-
- testInterpreterRunner = "/AbsolutePath.sh";
- when(mockInterpreterRunner.getPath()).thenReturn(testInterpreterRunner);
- i = factory.createRemoteRepl("path1", "sessionKey", "className", new Properties(), interpreterSettingManager.get().get(0).getId(), "userName", false, mockInterpreterRunner);
- interpreterRunner = ((RemoteInterpreter) ((LazyOpenInterpreter) i).getInnerInterpreter()).getInterpreterRunner();
- assertEquals(interpreterRunner, testInterpreterRunner);
- }
-}