You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2017/09/03 02:41:20 UTC
[1/9] zeppelin git commit: [ZEPPELIN-2627] Interpreter Component
Refactoring
Repository: zeppelin
Updated Branches:
refs/heads/master 69d58f373 -> d6203c51e
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
index 634ac30..f4b8c32 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
@@ -17,31 +17,28 @@
package org.apache.zeppelin.notebook;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.mock;
-
-import com.google.common.collect.Maps;
-import java.io.File;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
import com.google.common.collect.Sets;
import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
-import org.apache.zeppelin.dep.Dependency;
-import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.AbstractInterpreterTest;
+import org.apache.zeppelin.interpreter.ClassloaderInterpreter;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterFactory;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterOption;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResultMessage;
+import org.apache.zeppelin.interpreter.InterpreterSetting;
+import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.repo.VFSNotebookRepo;
import org.apache.zeppelin.resource.LocalResourcePool;
-import org.apache.zeppelin.resource.ResourcePoolUtils;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.scheduler.SchedulerFactory;
@@ -56,18 +53,35 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.RepositoryException;
-public class NotebookTest implements JobListenerFactory{
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+
+public class NotebookTest extends AbstractInterpreterTest implements JobListenerFactory {
private static final Logger logger = LoggerFactory.getLogger(NotebookTest.class);
- private File tmpDir;
- private ZeppelinConfiguration conf;
private SchedulerFactory schedulerFactory;
- private File notebookDir;
private Notebook notebook;
private NotebookRepo notebookRepo;
- private InterpreterFactory factory;
- private InterpreterSettingManager interpreterSettingManager;
- private DependencyResolver depResolver;
private NotebookAuthorization notebookAuthorization;
private Credentials credentials;
private AuthenticationInfo anonymous = AuthenticationInfo.ANONYMOUS;
@@ -75,57 +89,30 @@ public class NotebookTest implements JobListenerFactory{
@Before
public void setUp() throws Exception {
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC.getVarName(), "true");
+ System.setProperty(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "mock1,mock2");
+ super.setUp();
- tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
- tmpDir.mkdirs();
- new File(tmpDir, "conf").mkdirs();
- notebookDir = new File(tmpDir + "/notebook");
- notebookDir.mkdirs();
-
- System.setProperty(ConfVars.ZEPPELIN_CONF_DIR.getVarName(), tmpDir.toString() + "/conf");
- System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
- System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
-
- conf = ZeppelinConfiguration.create();
-
- this.schedulerFactory = new SchedulerFactory();
-
- depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(false));
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
-
- ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
- interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
- interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
-
- ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
- interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock2", interpreterInfos2, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock2", null);
- interpreterSettingManager.createNewSetting("mock2", "mock2", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
-
+ schedulerFactory = SchedulerFactory.singleton();
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath());
- notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, this, search,
+ notebook = new Notebook(conf, notebookRepo, schedulerFactory, interpreterFactory, interpreterSettingManager, this, search,
notebookAuthorization, credentials);
- System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC.getVarName(), "true");
}
@After
public void tearDown() throws Exception {
- delete(tmpDir);
+ super.tearDown();
}
@Test
public void testSelectingReplImplementation() throws IOException {
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
// run with default repl
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@@ -259,7 +246,7 @@ public class NotebookTest implements JobListenerFactory{
Notebook notebook2 = new Notebook(
conf, notebookRepo, schedulerFactory,
- new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager),
+ new InterpreterFactory(interpreterSettingManager),
interpreterSettingManager, null, null, null, null);
assertEquals(1, notebook2.getAllNotes().size());
@@ -316,7 +303,7 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testRunAll() throws IOException {
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
// p1
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@@ -355,7 +342,7 @@ public class NotebookTest implements JobListenerFactory{
public void testSchedule() throws InterruptedException, IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map config = new HashMap<>();
@@ -428,8 +415,8 @@ public class NotebookTest implements JobListenerFactory{
public void testAutoRestartInterpreterAfterSchedule() throws InterruptedException, IOException{
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
-
+ interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
+
Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map config = new HashMap<>();
p.setConfig(config);
@@ -448,21 +435,17 @@ public class NotebookTest implements JobListenerFactory{
notebook.refreshCron(note.getId());
- MockInterpreter1 mock1 = ((MockInterpreter1) (((ClassloaderInterpreter)
- ((LazyOpenInterpreter) factory.getInterpreter(anonymous.getUser(), note.getId(), "mock1")).getInnerInterpreter())
- .getInnerInterpreter()));
+ RemoteInterpreter mock1 = (RemoteInterpreter) interpreterFactory.getInterpreter(anonymous.getUser(), note.getId(), "mock1");
- MockInterpreter2 mock2 = ((MockInterpreter2) (((ClassloaderInterpreter)
- ((LazyOpenInterpreter) factory.getInterpreter(anonymous.getUser(), note.getId(), "mock2")).getInnerInterpreter())
- .getInnerInterpreter()));
+ RemoteInterpreter mock2 = (RemoteInterpreter) interpreterFactory.getInterpreter(anonymous.getUser(), note.getId(), "mock2");
// wait until interpreters are started
- while (!mock1.isOpen() || !mock2.isOpen()) {
+ while (!mock1.isOpened() || !mock2.isOpened()) {
Thread.yield();
}
// wait until interpreters are closed
- while (mock1.isOpen() || mock2.isOpen()) {
+ while (mock1.isOpened() || mock2.isOpened()) {
Thread.yield();
}
@@ -481,7 +464,7 @@ public class NotebookTest implements JobListenerFactory{
public void testExportAndImportNote() throws IOException, CloneNotSupportedException,
InterruptedException, InterpreterException, SchedulerException, RepositoryException {
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
final Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
String simpleText = "hello world";
@@ -520,7 +503,7 @@ public class NotebookTest implements JobListenerFactory{
public void testCloneNote() throws IOException, CloneNotSupportedException,
InterruptedException, InterpreterException, SchedulerException, RepositoryException {
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
final Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p.setText("hello world");
@@ -554,7 +537,7 @@ public class NotebookTest implements JobListenerFactory{
public void testCloneNoteWithNoName() throws IOException, CloneNotSupportedException,
InterruptedException {
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
Note cloneNote = notebook.cloneNote(note.getId(), null, anonymous);
assertEquals(cloneNote.getName(), "Note " + cloneNote.getId());
@@ -566,7 +549,7 @@ public class NotebookTest implements JobListenerFactory{
public void testCloneNoteWithExceptionResult() throws IOException, CloneNotSupportedException,
InterruptedException {
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
final Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p.setText("hello world");
@@ -591,28 +574,28 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testResourceRemovealOnParagraphNoteRemove() throws IOException {
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
- for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
- intpGroup.setResourcePool(new LocalResourcePool(intpGroup.getId()));
- }
+ interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
+
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p1.setText("hello");
Paragraph p2 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p2.setText("%mock2 world");
-
+ for (InterpreterGroup intpGroup : interpreterSettingManager.getAllInterpreterGroup()) {
+ intpGroup.setResourcePool(new LocalResourcePool(intpGroup.getId()));
+ }
note.runAll();
while (p1.isTerminated() == false || p1.getResult() == null) Thread.yield();
while (p2.isTerminated() == false || p2.getResult() == null) Thread.yield();
- assertEquals(2, ResourcePoolUtils.getAllResources().size());
+ assertEquals(2, interpreterSettingManager.getAllResources().size());
// remove a paragraph
note.removeParagraph(anonymous.getUser(), p1.getId());
- assertEquals(1, ResourcePoolUtils.getAllResources().size());
+ assertEquals(1, interpreterSettingManager.getAllResources().size());
// remove note
notebook.removeNote(note.getId(), anonymous);
- assertEquals(0, ResourcePoolUtils.getAllResources().size());
+ assertEquals(0, interpreterSettingManager.getAllResources().size());
}
@Test
@@ -620,10 +603,10 @@ public class NotebookTest implements JobListenerFactory{
IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
AngularObjectRegistry registry = interpreterSettingManager
- .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
+ .getInterpreterSettings(note.getId()).get(0).getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@@ -653,10 +636,10 @@ public class NotebookTest implements JobListenerFactory{
IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
AngularObjectRegistry registry = interpreterSettingManager
- .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
+ .getInterpreterSettings(note.getId()).get(0).getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@@ -687,10 +670,10 @@ public class NotebookTest implements JobListenerFactory{
IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
AngularObjectRegistry registry = interpreterSettingManager
- .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
+ .getInterpreterSettings(note.getId()).get(0).getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
// add local scope object
@@ -700,14 +683,13 @@ public class NotebookTest implements JobListenerFactory{
// restart interpreter
interpreterSettingManager.restart(interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getId());
- registry = interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
- .getAngularObjectRegistry();
-
- // local and global scope object should be removed
- // But InterpreterGroup does not implement angularObjectRegistry per session (scoped, isolated)
- // So for now, does not have good way to remove all objects in particular session on restart.
- assertNotNull(registry.get("o1", note.getId(), null));
- assertNotNull(registry.get("o2", null, null));
+ registry = interpreterSettingManager.getInterpreterSettings(note.getId()).get(0)
+ .getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess")
+ .getAngularObjectRegistry();
+
+ // New InterpreterGroup will be created and its AngularObjectRegistry will be created
+ assertNull(registry.get("o1", note.getId(), null));
+ assertNull(registry.get("o2", null, null));
notebook.removeNote(note.getId(), anonymous);
}
@@ -815,7 +797,7 @@ public class NotebookTest implements JobListenerFactory{
public void testAbortParagraphStatusOnInterpreterRestart() throws InterruptedException,
IOException {
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
// create three paragraphs
Paragraph p1 = note.addNewParagraph(anonymous);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
index 7bd6819..8a4bea0 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
@@ -21,6 +21,7 @@ package org.apache.zeppelin.notebook;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
@@ -39,7 +40,7 @@ import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectBuilder;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.Input;
-import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.Interpreter.FormType;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterGroup;
@@ -47,10 +48,7 @@ import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
-import org.apache.zeppelin.interpreter.InterpreterResultMessage;
-import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.InterpreterSetting.Status;
-import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.resource.ResourcePool;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.user.Credentials;
@@ -173,7 +171,7 @@ public class ParagraphTest {
Interpreter mockInterpreter = mock(Interpreter.class);
doReturn(mockInterpreter).when(spyParagraph).getRepl(anyString());
- InterpreterGroup mockInterpreterGroup = mock(InterpreterGroup.class);
+ ManagedInterpreterGroup mockInterpreterGroup = mock(ManagedInterpreterGroup.class);
when(mockInterpreter.getInterpreterGroup()).thenReturn(mockInterpreterGroup);
when(mockInterpreterGroup.getId()).thenReturn("mock_id_1");
when(mockInterpreterGroup.getAngularObjectRegistry()).thenReturn(mock(AngularObjectRegistry.class));
@@ -186,7 +184,7 @@ public class ParagraphTest {
when(mockInterpreterOption.permissionIsSet()).thenReturn(false);
when(mockInterpreterSetting.getStatus()).thenReturn(Status.READY);
when(mockInterpreterSetting.getId()).thenReturn("mock_id_1");
- when(mockInterpreterSetting.getInterpreterGroup(anyString(), anyString())).thenReturn(mockInterpreterGroup);
+ when(mockInterpreterSetting.getOrCreateInterpreterGroup(anyString(), anyString())).thenReturn(mockInterpreterGroup);
spyInterpreterSettingList.add(mockInterpreterSetting);
when(mockNote.getId()).thenReturn("any_id");
when(mockInterpreterSettingManager.getInterpreterSettings(anyString())).thenReturn(spyInterpreterSettingList);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
index a6c9393..8a0b5a1 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
@@ -33,10 +33,12 @@ import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.DependencyResolver;
+import org.apache.zeppelin.display.AngularObjectRegistryListener;
+import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.InterpreterFactory;
-import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterResultMessage;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.notebook.*;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Job.Status;
@@ -69,7 +71,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
private Credentials credentials;
private AuthenticationInfo anonymous;
private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncTest.class);
-
+
@Before
public void setUp() throws Exception {
String zpath = System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis();
@@ -91,12 +93,13 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
LOG.info("secondary note dir : " + secNotePath);
conf = ZeppelinConfiguration.create();
- this.schedulerFactory = new SchedulerFactory();
+ this.schedulerFactory = SchedulerFactory.singleton();
depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo");
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
-
+ interpreterSettingManager = new InterpreterSettingManager(conf,
+ mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
+ factory = new InterpreterFactory(interpreterSettingManager);
+
search = mock(SearchService.class);
notebookRepoSync = new NotebookRepoSync(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
@@ -110,19 +113,19 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
public void tearDown() throws Exception {
delete(mainZepDir);
}
-
+
@Test
public void testRepoCount() throws IOException {
assertTrue(notebookRepoSync.getMaxRepoNum() >= notebookRepoSync.getRepoCount());
}
-
+
@Test
public void testSyncOnCreate() throws IOException {
/* check that both storage systems are empty */
assertTrue(notebookRepoSync.getRepoCount() > 1);
assertEquals(0, notebookRepoSync.list(0, anonymous).size());
assertEquals(0, notebookRepoSync.list(1, anonymous).size());
-
+
/* create note */
Note note = notebookSync.createNote(anonymous);
@@ -130,7 +133,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertEquals(1, notebookRepoSync.list(0, anonymous).size());
assertEquals(1, notebookRepoSync.list(1, anonymous).size());
assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId());
-
+
notebookSync.removeNote(notebookRepoSync.list(0, null).get(0).getId(), anonymous);
}
@@ -140,26 +143,26 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertTrue(notebookRepoSync.getRepoCount() > 1);
assertEquals(0, notebookRepoSync.list(0, anonymous).size());
assertEquals(0, notebookRepoSync.list(1, anonymous).size());
-
+
Note note = notebookSync.createNote(anonymous);
/* check that created in both storage systems */
assertEquals(1, notebookRepoSync.list(0, anonymous).size());
assertEquals(1, notebookRepoSync.list(1, anonymous).size());
assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId());
-
+
/* remove Note */
notebookSync.removeNote(notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous);
-
+
/* check that deleted in both storages */
assertEquals(0, notebookRepoSync.list(0, anonymous).size());
assertEquals(0, notebookRepoSync.list(1, anonymous).size());
-
+
}
-
+
@Test
public void testSyncUpdateMain() throws IOException {
-
+
/* create note */
Note note = notebookSync.createNote(anonymous);
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@@ -167,19 +170,19 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
config.put("enabled", true);
p1.setConfig(config);
p1.setText("hello world");
-
+
/* new paragraph exists in note instance */
assertEquals(1, note.getParagraphs().size());
-
+
/* new paragraph not yet saved into storages */
assertEquals(0, notebookRepoSync.get(0,
notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size());
assertEquals(0, notebookRepoSync.get(1,
notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size());
-
- /* save to storage under index 0 (first storage) */
+
+ /* save to storage under index 0 (first storage) */
notebookRepoSync.save(0, note, anonymous);
-
+
/* check paragraph saved to first storage */
assertEquals(1, notebookRepoSync.get(0,
notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size());
@@ -284,45 +287,45 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
// one git versioned storage initialized
assertThat(vRepoSync.getRepoCount()).isEqualTo(1);
assertThat(vRepoSync.getRepo(0)).isInstanceOf(GitNotebookRepo.class);
-
+
GitNotebookRepo gitRepo = (GitNotebookRepo) vRepoSync.getRepo(0);
-
+
// no notes
assertThat(vRepoSync.list(anonymous).size()).isEqualTo(0);
// create note
Note note = vNotebookSync.createNote(anonymous);
assertThat(vRepoSync.list(anonymous).size()).isEqualTo(1);
-
+
String noteId = vRepoSync.list(anonymous).get(0).getId();
// first checkpoint
vRepoSync.checkpoint(noteId, "checkpoint message", anonymous);
int vCount = gitRepo.revisionHistory(noteId, anonymous).size();
assertThat(vCount).isEqualTo(1);
-
+
Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map<String, Object> config = p.getConfig();
config.put("enabled", true);
p.setConfig(config);
p.setText("%md checkpoint test");
-
+
// save and checkpoint again
vRepoSync.save(note, anonymous);
vRepoSync.checkpoint(noteId, "checkpoint message 2", anonymous);
assertThat(gitRepo.revisionHistory(noteId, anonymous).size()).isEqualTo(vCount + 1);
notebookRepoSync.remove(note.getId(), anonymous);
}
-
+
@Test
public void testSyncWithAcl() throws IOException {
/* scenario 1 - note exists with acl on main storage */
AuthenticationInfo user1 = new AuthenticationInfo("user1");
Note note = notebookSync.createNote(user1);
assertEquals(0, note.getParagraphs().size());
-
+
// saved on both storages
assertEquals(1, notebookRepoSync.list(0, null).size());
assertEquals(1, notebookRepoSync.list(1, null).size());
-
+
/* check that user1 is the only owner */
NotebookAuthorization authInfo = NotebookAuthorization.getInstance();
Set<String> entity = new HashSet<String>();
@@ -332,23 +335,23 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertEquals(0, authInfo.getReaders(note.getId()).size());
assertEquals(0, authInfo.getRunners(note.getId()).size());
assertEquals(0, authInfo.getWriters(note.getId()).size());
-
+
/* update note and save on secondary storage */
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p1.setText("hello world");
assertEquals(1, note.getParagraphs().size());
notebookRepoSync.save(1, note, null);
-
+
/* check paragraph isn't saved into first storage */
assertEquals(0, notebookRepoSync.get(0,
notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size());
/* check paragraph is saved into second storage */
assertEquals(1, notebookRepoSync.get(1,
notebookRepoSync.list(1, null).get(0).getId(), null).getParagraphs().size());
-
+
/* now sync by user1 */
notebookRepoSync.sync(user1);
-
+
/* check that note updated and acl are same on main storage*/
assertEquals(1, notebookRepoSync.get(0,
notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size());
@@ -357,7 +360,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertEquals(0, authInfo.getReaders(note.getId()).size());
assertEquals(0, authInfo.getRunners(note.getId()).size());
assertEquals(0, authInfo.getWriters(note.getId()).size());
-
+
/* scenario 2 - note doesn't exist on main storage */
/* remove from main storage */
notebookRepoSync.remove(0, note.getId(), user1);
@@ -368,7 +371,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertEquals(0, authInfo.getReaders(note.getId()).size());
assertEquals(0, authInfo.getRunners(note.getId()).size());
assertEquals(0, authInfo.getWriters(note.getId()).size());
-
+
/* now sync - should bring note from secondary storage with added acl */
notebookRepoSync.sync(user1);
assertEquals(1, notebookRepoSync.list(0, null).size());
@@ -428,5 +431,5 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
}
};
}
-
+
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
index 6f85bf6..98a08d9 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
@@ -22,25 +22,14 @@ import static org.mockito.Mockito.mock;
import java.io.File;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Properties;
-import com.google.common.collect.Maps;
import org.apache.commons.io.FileUtils;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
-import org.apache.zeppelin.dep.Dependency;
-import org.apache.zeppelin.dep.DependencyResolver;
-import org.apache.zeppelin.interpreter.InterpreterFactory;
-import org.apache.zeppelin.interpreter.InterpreterInfo;
-import org.apache.zeppelin.interpreter.InterpreterOption;
-import org.apache.zeppelin.interpreter.DefaultInterpreterProperty;
-import org.apache.zeppelin.interpreter.InterpreterProperty;
-import org.apache.zeppelin.interpreter.InterpreterSettingManager;
-import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
+
+import org.apache.zeppelin.interpreter.AbstractInterpreterTest;
+
import org.apache.zeppelin.notebook.JobListenerFactory;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Notebook;
@@ -58,59 +47,34 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableMap;
-public class VFSNotebookRepoTest implements JobListenerFactory {
+public class VFSNotebookRepoTest extends AbstractInterpreterTest implements JobListenerFactory {
+
private static final Logger LOG = LoggerFactory.getLogger(VFSNotebookRepoTest.class);
- private ZeppelinConfiguration conf;
+
private SchedulerFactory schedulerFactory;
private Notebook notebook;
private NotebookRepo notebookRepo;
- private InterpreterSettingManager interpreterSettingManager;
- private InterpreterFactory factory;
- private DependencyResolver depResolver;
private NotebookAuthorization notebookAuthorization;
- private File mainZepDir;
- private File mainNotebookDir;
-
@Before
public void setUp() throws Exception {
- String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis();
- mainZepDir = new File(zpath);
- mainZepDir.mkdirs();
- new File(mainZepDir, "conf").mkdirs();
- String mainNotePath = zpath + "/notebook";
- mainNotebookDir = new File(mainNotePath);
- mainNotebookDir.mkdirs();
-
- System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath());
- System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath());
+ System.setProperty(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "mock1,mock2");
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.VFSNotebookRepo");
- conf = ZeppelinConfiguration.create();
-
- this.schedulerFactory = new SchedulerFactory();
-
- this.schedulerFactory = new SchedulerFactory();
- depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo");
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
- ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
- interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
- interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
+ super.setUp();
+ this.schedulerFactory = SchedulerFactory.singleton();
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
- notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, this, search,
+ notebook = new Notebook(conf, notebookRepo, schedulerFactory, interpreterFactory, interpreterSettingManager, this, search,
notebookAuthorization, null);
}
@After
public void tearDown() throws Exception {
- if (!FileUtils.deleteQuietly(mainZepDir)) {
- LOG.error("Failed to delete {} ", mainZepDir.getName());
+ if (!FileUtils.deleteQuietly(testRootDir)) {
+ LOG.error("Failed to delete {} ", testRootDir.getName());
}
}
@@ -120,7 +84,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
int numNotes = notebookRepo.list(null).size();
// when create invalid json file
- File testNoteDir = new File(mainNotebookDir, "test");
+ File testNoteDir = new File(notebookDir, "interpreter/test");
testNoteDir.mkdir();
FileUtils.writeStringToFile(new File(testNoteDir, "note.json"), "");
@@ -132,7 +96,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
public void testSaveNotebook() throws IOException, InterruptedException {
AuthenticationInfo anonymous = new AuthenticationInfo("anonymous");
Note note = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map<String, Object> config = p1.getConfig();
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java
index 46134e5..469d5b5 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java
@@ -17,35 +17,27 @@
package org.apache.zeppelin.resource;
import com.google.gson.Gson;
-import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventPoller;
-import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterResourcePool;
+import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import java.io.File;
import java.util.HashMap;
import java.util.LinkedList;
-import java.util.Properties;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
/**
* Unittest for DistributedResourcePool
*/
-public class DistributedResourcePoolTest {
- private static final String INTERPRETER_SCRIPT =
- System.getProperty("os.name").startsWith("Windows") ?
- "../bin/interpreter.cmd" :
- "../bin/interpreter.sh";
- private InterpreterGroup intpGroup1;
- private InterpreterGroup intpGroup2;
- private HashMap<String, String> env;
+public class DistributedResourcePoolTest extends AbstractInterpreterTest {
+
private RemoteInterpreter intp1;
private RemoteInterpreter intp2;
private InterpreterContext context;
@@ -55,50 +47,10 @@ public class DistributedResourcePoolTest {
@Before
public void setUp() throws Exception {
- env = new HashMap<>();
- env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
-
- Properties p = new Properties();
-
- intp1 = new RemoteInterpreter(
- p,
- "note",
- MockInterpreterResourcePool.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- null,
- null,
- "anonymous",
- false
- );
-
- intpGroup1 = new InterpreterGroup("intpGroup1");
- intpGroup1.put("note", new LinkedList<Interpreter>());
- intpGroup1.get("note").add(intp1);
- intp1.setInterpreterGroup(intpGroup1);
-
- intp2 = new RemoteInterpreter(
- p,
- "note",
- MockInterpreterResourcePool.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- null,
- null,
- "anonymous",
- false
- );
-
- intpGroup2 = new InterpreterGroup("intpGroup2");
- intpGroup2.put("note", new LinkedList<Interpreter>());
- intpGroup2.get("note").add(intp2);
- intp2.setInterpreterGroup(intpGroup2);
+ super.setUp();
+ InterpreterSetting interpreterSetting = interpreterSettingManager.getByName("mock_resource_pool");
+ intp1 = (RemoteInterpreter) interpreterSetting.getInterpreter("user1", "note1", "mock_resource_pool");
+ intp2 = (RemoteInterpreter) interpreterSetting.getInterpreter("user2", "note1", "mock_resource_pool");
context = new InterpreterContext(
"note",
@@ -117,26 +69,13 @@ public class DistributedResourcePoolTest {
intp1.open();
intp2.open();
- eventPoller1 = new RemoteInterpreterEventPoller(null, null);
- eventPoller1.setInterpreterGroup(intpGroup1);
- eventPoller1.setInterpreterProcess(intpGroup1.getRemoteInterpreterProcess());
-
- eventPoller2 = new RemoteInterpreterEventPoller(null, null);
- eventPoller2.setInterpreterGroup(intpGroup2);
- eventPoller2.setInterpreterProcess(intpGroup2.getRemoteInterpreterProcess());
-
- eventPoller1.start();
- eventPoller2.start();
+ eventPoller1 = intp1.getInterpreterGroup().getRemoteInterpreterProcess().getRemoteInterpreterEventPoller();
+ eventPoller2 = intp1.getInterpreterGroup().getRemoteInterpreterProcess().getRemoteInterpreterEventPoller();
}
@After
public void tearDown() throws Exception {
- eventPoller1.shutdown();
- intp1.close();
- intpGroup1.close();
- eventPoller2.shutdown();
- intp2.close();
- intpGroup2.close();
+ interpreterSettingManager.close();
}
@Test
@@ -235,13 +174,13 @@ public class DistributedResourcePoolTest {
// then get all resources.
- assertEquals(4, ResourcePoolUtils.getAllResources().size());
+ assertEquals(4, interpreterSettingManager.getAllResources().size());
// when remove all resources from note1
- ResourcePoolUtils.removeResourcesBelongsToNote("note1");
+ interpreterSettingManager.removeResourcesBelongsToNote("note1");
// then resources should be removed.
- assertEquals(2, ResourcePoolUtils.getAllResources().size());
+ assertEquals(2, interpreterSettingManager.getAllResources().size());
assertEquals("", gson.fromJson(
intp1.interpret("get note1:paragraph1:key1", context).message().get(0).getData(),
String.class));
@@ -251,10 +190,10 @@ public class DistributedResourcePoolTest {
// when remove all resources from note2:paragraph1
- ResourcePoolUtils.removeResourcesBelongsToParagraph("note2", "paragraph1");
+ interpreterSettingManager.removeResourcesBelongsToParagraph("note2", "paragraph1");
// then 1
- assertEquals(1, ResourcePoolUtils.getAllResources().size());
+ assertEquals(1, interpreterSettingManager.getAllResources().size());
assertEquals("value2", gson.fromJson(
intp1.interpret("get note2:paragraph2:key2", context).message().get(0).getData(),
String.class));
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
index ebb5100..a758591 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
@@ -17,83 +17,84 @@
package org.apache.zeppelin.scheduler;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.interpreter.*;
-import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterContextRunner;
+import org.apache.zeppelin.interpreter.InterpreterInfo;
+import org.apache.zeppelin.interpreter.InterpreterOption;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterRunner;
+import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterA;
import org.apache.zeppelin.resource.LocalResourcePool;
import org.apache.zeppelin.scheduler.Job.Status;
+import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
private static final String INTERPRETER_SCRIPT =
- System.getProperty("os.name").startsWith("Windows") ?
- "../bin/interpreter.cmd" :
- "../bin/interpreter.sh";
+ System.getProperty("os.name").startsWith("Windows") ?
+ "../bin/interpreter.cmd" :
+ "../bin/interpreter.sh";
+
+ private InterpreterSetting interpreterSetting;
private SchedulerFactory schedulerSvc;
private static final int TICK_WAIT = 100;
private static final int MAX_WAIT_CYCLES = 100;
@Before
- public void setUp() throws Exception{
+ public void setUp() throws Exception {
schedulerSvc = new SchedulerFactory();
+
+ InterpreterOption interpreterOption = new InterpreterOption();
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(MockInterpreterA.class.getName(), "mock", true, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT);
+ interpreterSetting = new InterpreterSetting.Builder()
+ .setId("test")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .setRunner(runner)
+ .setInterpreterDir("../interpeters/test")
+ .create();
}
@After
- public void tearDown(){
-
+ public void tearDown() {
+ interpreterSetting.close();
}
@Test
public void test() throws Exception {
- Properties p = new Properties();
- final InterpreterGroup intpGroup = new InterpreterGroup();
- Map<String, String> env = new HashMap<>();
- env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
-
- final RemoteInterpreter intpA = new RemoteInterpreter(
- p,
- "note",
- MockInterpreterA.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- this,
- null,
- "anonymous",
- false);
-
- intpGroup.put("note", new LinkedList<Interpreter>());
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
+ final RemoteInterpreter intpA = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
intpA.open();
- Scheduler scheduler = schedulerSvc.createOrGetRemoteScheduler("test", "note",
- intpA.getInterpreterProcess(),
- 10);
+ Scheduler scheduler = intpA.getScheduler();
Job job = new Job("jobId", "jobName", null, 200) {
Object results;
+
@Override
public Object getReturn() {
return results;
@@ -120,7 +121,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
+ null,
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null));
return "1000";
@@ -145,7 +146,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
}
assertTrue(job.isRunning());
- Thread.sleep(5*TICK_WAIT);
+ Thread.sleep(5 * TICK_WAIT);
assertEquals(0, scheduler.getJobsWaiting().size());
assertEquals(1, scheduler.getJobsRunning().size());
@@ -165,34 +166,10 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
@Test
public void testAbortOnPending() throws Exception {
- Properties p = new Properties();
- final InterpreterGroup intpGroup = new InterpreterGroup();
- Map<String, String> env = new HashMap<>();
- env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
-
- final RemoteInterpreter intpA = new RemoteInterpreter(
- p,
- "note",
- MockInterpreterA.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- this,
- null,
- "anonymous",
- false);
-
- intpGroup.put("note", new LinkedList<Interpreter>());
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
+ final RemoteInterpreter intpA = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
intpA.open();
- Scheduler scheduler = schedulerSvc.createOrGetRemoteScheduler("test", "note",
- intpA.getInterpreterProcess(),
- 10);
+ Scheduler scheduler = intpA.getScheduler();
Job job1 = new Job("jobId1", "jobName1", null, 200) {
Object results;
@@ -205,7 +182,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
+ null,
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null);
@@ -255,7 +232,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
+ null,
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null);
@@ -358,7 +335,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
}
@Override
- public void onParaInfosReceived(String noteId, String paragraphId,
- String interpreterSettingId, Map<String, String> metaInfos) {
+ public void onParaInfosReceived(String noteId, String paragraphId,
+ String interpreterSettingId, Map<String, String> metaInfos) {
}
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/resources/conf/interpreter.json
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/resources/conf/interpreter.json b/zeppelin-zengine/src/test/resources/conf/interpreter.json
new file mode 100644
index 0000000..45e1d60
--- /dev/null
+++ b/zeppelin-zengine/src/test/resources/conf/interpreter.json
@@ -0,0 +1,115 @@
+{
+ "interpreterSettings": {
+ "2C3RWCVAG": {
+ "id": "2C3RWCVAG",
+ "name": "test",
+ "group": "test",
+ "properties": {
+ "property_1": "value_1",
+ "property_2": "new_value_2",
+ "property_3": "value_3"
+ },
+ "status": "READY",
+ "interpreterGroup": [
+ {
+ "name": "echo",
+ "class": "org.apache.zeppelin.interpreter.EchoInterpreter",
+ "defaultInterpreter": true,
+ "editor": {
+ "language": "java",
+ "editOnDblClick": false
+ }
+ }
+ ],
+ "dependencies": [],
+ "option": {
+ "remote": true,
+ "port": -1,
+ "perNote": "shared",
+ "perUser": "shared",
+ "isExistingProcess": false,
+ "setPermission": false,
+ "users": [],
+ "isUserImpersonate": false
+ }
+ },
+
+ "2CKWE7B19": {
+ "id": "2CKWE7B19",
+ "name": "test2",
+ "group": "test",
+ "properties": {
+ "property_1": "value_1",
+ "property_2": "new_value_2",
+ "property_3": "value_3"
+ },
+ "status": "READY",
+ "interpreterGroup": [
+ {
+ "name": "echo",
+ "class": "org.apache.zeppelin.interpreter.EchoInterpreter",
+ "defaultInterpreter": true,
+ "editor": {
+ "language": "java",
+ "editOnDblClick": false
+ }
+ }
+ ],
+ "dependencies": [],
+ "option": {
+ "remote": true,
+ "port": -1,
+ "perNote": "shared",
+ "perUser": "shared",
+ "isExistingProcess": false,
+ "setPermission": false,
+ "users": [],
+ "isUserImpersonate": false
+ }
+ }
+ },
+ "interpreterBindings": {
+ "2C6793KRV": [
+ "2C48Y7FSJ",
+ "2C63XW4XE",
+ "2C66GE1VB",
+ "2C5VH924X",
+ "2C4BJDRRZ",
+ "2C3SQSB7V",
+ "2C4HKDCQW",
+ "2C3DR183X",
+ "2C66Z9XPQ",
+ "2C3PTPMUH",
+ "2C69WE69N",
+ "2C5SRRXHM",
+ "2C4ZD49PF",
+ "2C6V3D44K",
+ "2C4UB1UZA",
+ "2C5S1R21W",
+ "2C5DCRVGM",
+ "2C686X8ZH",
+ "2C3RWCVAG",
+ "2C3JKFMJU",
+ "2C3VECEG2"
+ ]
+ },
+ "interpreterRepositories": [
+ {
+ "id": "central",
+ "type": "default",
+ "url": "http://repo1.maven.org/maven2/",
+ "releasePolicy": {
+ "enabled": true,
+ "updatePolicy": "daily",
+ "checksumPolicy": "warn"
+ },
+ "snapshotPolicy": {
+ "enabled": true,
+ "updatePolicy": "daily",
+ "checksumPolicy": "warn"
+ },
+ "mirroredRepositories": [],
+ "repositoryManager": false
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/resources/interpreter/mock/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/resources/interpreter/mock/interpreter-setting.json b/zeppelin-zengine/src/test/resources/interpreter/mock/interpreter-setting.json
deleted file mode 100644
index 65568ef..0000000
--- a/zeppelin-zengine/src/test/resources/interpreter/mock/interpreter-setting.json
+++ /dev/null
@@ -1,12 +0,0 @@
-[
- {
- "group": "mock11",
- "name": "mock11",
- "className": "org.apache.zeppelin.interpreter.mock.MockInterpreter11",
- "properties": {
- },
- "editor": {
- "language": "java"
- }
- }
-]
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/resources/interpreter/mock1/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/resources/interpreter/mock1/interpreter-setting.json b/zeppelin-zengine/src/test/resources/interpreter/mock1/interpreter-setting.json
new file mode 100644
index 0000000..2f628da
--- /dev/null
+++ b/zeppelin-zengine/src/test/resources/interpreter/mock1/interpreter-setting.json
@@ -0,0 +1,19 @@
+[
+ {
+ "group": "mock1",
+ "name": "mock1",
+ "className": "org.apache.zeppelin.interpreter.mock.MockInterpreter1",
+ "properties": {
+ },
+ "option": {
+ "remote": true,
+ "port": -1,
+ "perNote": "shared",
+ "perUser": "shared",
+ "isExistingProcess": false,
+ "setPermission": false,
+ "users": [],
+ "isUserImpersonate": false
+ }
+ }
+]
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/resources/interpreter/mock2/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/resources/interpreter/mock2/interpreter-setting.json b/zeppelin-zengine/src/test/resources/interpreter/mock2/interpreter-setting.json
new file mode 100644
index 0000000..fae03f6
--- /dev/null
+++ b/zeppelin-zengine/src/test/resources/interpreter/mock2/interpreter-setting.json
@@ -0,0 +1,19 @@
+[
+ {
+ "group": "mock2",
+ "name": "mock2",
+ "className": "org.apache.zeppelin.interpreter.mock.MockInterpreter2",
+ "properties": {
+ },
+ "option": {
+ "remote": true,
+ "port": -1,
+ "perNote": "shared",
+ "perUser": "isolated",
+ "isExistingProcess": false,
+ "setPermission": false,
+ "users": [],
+ "isUserImpersonate": false
+ }
+ }
+]
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/resources/interpreter/mock_resource_pool/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/resources/interpreter/mock_resource_pool/interpreter-setting.json b/zeppelin-zengine/src/test/resources/interpreter/mock_resource_pool/interpreter-setting.json
new file mode 100644
index 0000000..4dfe0a7
--- /dev/null
+++ b/zeppelin-zengine/src/test/resources/interpreter/mock_resource_pool/interpreter-setting.json
@@ -0,0 +1,19 @@
+[
+ {
+ "group": "mock_resource_pool",
+ "name": "mock_resource_pool",
+ "className": "org.apache.zeppelin.interpreter.remote.mock.MockInterpreterResourcePool",
+ "properties": {
+ },
+ "option": {
+ "remote": true,
+ "port": -1,
+ "perNote": "shared",
+ "perUser": "shared",
+ "isExistingProcess": false,
+ "setPermission": false,
+ "users": [],
+ "isUserImpersonate": false
+ }
+ }
+]
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/resources/interpreter/test/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/resources/interpreter/test/interpreter-setting.json b/zeppelin-zengine/src/test/resources/interpreter/test/interpreter-setting.json
new file mode 100644
index 0000000..596fc91
--- /dev/null
+++ b/zeppelin-zengine/src/test/resources/interpreter/test/interpreter-setting.json
@@ -0,0 +1,53 @@
+[
+ {
+ "group": "test",
+ "name": "double_echo",
+ "className": "org.apache.zeppelin.interpreter.DoubleEchoInterpreter",
+ "properties": {
+ "property_1": {
+ "envName": "PROPERTY_1",
+ "propertyName": "property_1",
+ "defaultValue": "value_1",
+ "description": "desc_1"
+ },
+ "property_2": {
+ "envName": "PROPERTY_2",
+ "propertyName": "property_2",
+ "defaultValue": "value_2",
+ "description": "desc_2"
+ }
+ },
+ "editor": {
+ "language": "java",
+ "editOnDblClick": false
+ }
+ },
+
+ {
+ "group": "test",
+ "name": "echo",
+ "defaultInterpreter": true,
+ "className": "org.apache.zeppelin.interpreter.EchoInterpreter",
+ "properties": {
+ "property_1": {
+ "envName": "PROPERTY_1",
+ "propertyName": "property_1",
+ "defaultValue": "value_1",
+ "description": "desc_1"
+ },
+ "property_2": {
+ "envName": "PROPERTY_2",
+ "propertyName": "property_2",
+ "defaultValue": "value_2",
+ "description": "desc_2"
+ }
+ },
+ "runner": {
+ "linux": "linux_runner"
+ },
+ "editor": {
+ "language": "java",
+ "editOnDblClick": false
+ }
+ }
+]
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/resources/log4j.properties b/zeppelin-zengine/src/test/resources/log4j.properties
index 001a222..74f619b 100644
--- a/zeppelin-zengine/src/test/resources/log4j.properties
+++ b/zeppelin-zengine/src/test/resources/log4j.properties
@@ -35,7 +35,6 @@ log4j.logger.org.apache.hadoop.mapred=WARN
log4j.logger.org.apache.hadoop.hive.ql=WARN
log4j.logger.org.apache.hadoop.hive.metastore=WARN
log4j.logger.org.apache.haadoop.hive.service.HiveServer=WARN
-log4j.logger.org.apache.zeppelin.scheduler=WARN
log4j.logger.org.quartz=WARN
log4j.logger.DataNucleus=WARN
@@ -45,4 +44,6 @@ log4j.logger.DataNucleus.Datastore=ERROR
# Log all JDBC parameters
log4j.logger.org.hibernate.type=ALL
+log4j.logger.org.apache.zeppelin.interpreter=DEBUG
+log4j.logger.org.apache.zeppelin.scheduler=DEBUG
[6/9] zeppelin git commit: [ZEPPELIN-2627] Interpreter Component
Refactoring
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
index 12545d6..585a58a 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
@@ -17,212 +17,205 @@
package org.apache.zeppelin.interpreter;
-import java.io.BufferedReader;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.reflect.TypeToken;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.dep.Dependency;
+import org.apache.zeppelin.dep.DependencyResolver;
+import org.apache.zeppelin.display.AngularObjectRegistryListener;
+import org.apache.zeppelin.helium.ApplicationEventListener;
+import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
+import org.apache.zeppelin.resource.Resource;
+import org.apache.zeppelin.resource.ResourcePool;
+import org.apache.zeppelin.resource.ResourceSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonatype.aether.repository.Authentication;
+import org.sonatype.aether.repository.Proxy;
+import org.sonatype.aether.repository.RemoteRepository;
+
import java.io.File;
import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
import java.lang.reflect.Type;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
-import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream.Filter;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.nio.file.attribute.PosixFilePermission;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
-import java.util.EnumSet;
-import java.util.Enumeration;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
-import org.apache.zeppelin.dep.Dependency;
-import org.apache.zeppelin.dep.DependencyResolver;
-import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter;
-import org.apache.zeppelin.scheduler.Job;
-import org.apache.zeppelin.scheduler.Job.Status;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.sonatype.aether.RepositoryException;
-import org.sonatype.aether.repository.Authentication;
-import org.sonatype.aether.repository.Proxy;
-import org.sonatype.aether.repository.RemoteRepository;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import com.google.gson.internal.StringMap;
-import com.google.gson.reflect.TypeToken;
-
-import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
-import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
/**
- * TBD
+ * InterpreterSettingManager is the component which manage all the interpreter settings.
+ * (load/create/update/remove/get)
+ * Besides that InterpreterSettingManager also manage the interpreter setting binding.
+ * TODO(zjffdu) We could move it into another separated component.
*/
public class InterpreterSettingManager {
- private static final Logger logger = LoggerFactory.getLogger(InterpreterSettingManager.class);
- private static final String SHARED_SESSION = "shared_session";
+ private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterSettingManager.class);
private static final Map<String, Object> DEFAULT_EDITOR = ImmutableMap.of(
"language", (Object) "text",
"editOnDblClick", false);
- private final ZeppelinConfiguration zeppelinConfiguration;
+ private final ZeppelinConfiguration conf;
private final Path interpreterDirPath;
- private final Path interpreterBindingPath;
+ private final Path interpreterSettingPath;
/**
- * This is only references with default settings, name and properties
- * key: InterpreterSetting.name
+ * This is only InterpreterSetting templates with default name and properties
+ * name --> InterpreterSetting
*/
- private final Map<String, InterpreterSetting> interpreterSettingsRef;
+ private final Map<String, InterpreterSetting> interpreterSettingTemplates =
+ Maps.newConcurrentMap();
/**
* This is used by creating and running Interpreters
- * key: InterpreterSetting.id <- This is becuase backward compatibility
+ * id --> InterpreterSetting
+ * TODO(zjffdu) change it to name --> InterpreterSetting
*/
- private final Map<String, InterpreterSetting> interpreterSettings;
- private final Map<String, List<String>> interpreterBindings;
-
- private final DependencyResolver dependencyResolver;
- private final List<RemoteRepository> interpreterRepositories;
+ private final Map<String, InterpreterSetting> interpreterSettings =
+ Maps.newConcurrentMap();
- private final InterpreterOption defaultOption;
+ /**
+ * noteId --> list of InterpreterSettingId
+ */
+ private final Map<String, List<String>> interpreterBindings =
+ Maps.newConcurrentMap();
- private final Map<String, URLClassLoader> cleanCl;
+ private final List<RemoteRepository> interpreterRepositories;
+ private InterpreterOption defaultOption;
+ private List<String> interpreterGroupOrderList;
+ private final Gson gson;
- @Deprecated
- private String[] interpreterClassList;
- private String[] interpreterGroupOrderList;
- private InterpreterGroupFactory interpreterGroupFactory;
+ private AngularObjectRegistryListener angularObjectRegistryListener;
+ private RemoteInterpreterProcessListener remoteInterpreterProcessListener;
+ private ApplicationEventListener appEventListener;
+ private DependencyResolver dependencyResolver;
- private final Gson gson;
public InterpreterSettingManager(ZeppelinConfiguration zeppelinConfiguration,
- DependencyResolver dependencyResolver, InterpreterOption interpreterOption)
- throws IOException, RepositoryException {
- this.zeppelinConfiguration = zeppelinConfiguration;
- this.interpreterDirPath = Paths.get(zeppelinConfiguration.getInterpreterDir());
- logger.debug("InterpreterRootPath: {}", interpreterDirPath);
- this.interpreterBindingPath = Paths.get(zeppelinConfiguration.getInterpreterSettingPath());
- logger.debug("InterpreterBindingPath: {}", interpreterBindingPath);
-
- this.interpreterSettingsRef = Maps.newConcurrentMap();
- this.interpreterSettings = Maps.newConcurrentMap();
- this.interpreterBindings = Maps.newConcurrentMap();
-
- this.dependencyResolver = dependencyResolver;
+ AngularObjectRegistryListener angularObjectRegistryListener,
+ RemoteInterpreterProcessListener
+ remoteInterpreterProcessListener,
+ ApplicationEventListener appEventListener)
+ throws IOException {
+ this(zeppelinConfiguration, new InterpreterOption(true),
+ angularObjectRegistryListener,
+ remoteInterpreterProcessListener,
+ appEventListener);
+ }
+
+ @VisibleForTesting
+ public InterpreterSettingManager(ZeppelinConfiguration conf,
+ InterpreterOption defaultOption,
+ AngularObjectRegistryListener angularObjectRegistryListener,
+ RemoteInterpreterProcessListener
+ remoteInterpreterProcessListener,
+ ApplicationEventListener appEventListener) throws IOException {
+ this.conf = conf;
+ this.defaultOption = defaultOption;
+ this.interpreterDirPath = Paths.get(conf.getInterpreterDir());
+ LOGGER.debug("InterpreterRootPath: {}", interpreterDirPath);
+ this.interpreterSettingPath = Paths.get(conf.getInterpreterSettingPath());
+ LOGGER.debug("InterpreterBindingPath: {}", interpreterSettingPath);
+ this.dependencyResolver = new DependencyResolver(
+ conf.getString(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO));
this.interpreterRepositories = dependencyResolver.getRepos();
+ this.interpreterGroupOrderList = Arrays.asList(conf.getString(
+ ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER).split(","));
+ this.gson = new GsonBuilder().setPrettyPrinting().create();
- this.defaultOption = interpreterOption;
-
- this.cleanCl = Collections.synchronizedMap(new HashMap<String, URLClassLoader>());
-
- String replsConf = zeppelinConfiguration.getString(ConfVars.ZEPPELIN_INTERPRETERS);
- this.interpreterClassList = replsConf.split(",");
- String groupOrder = zeppelinConfiguration.getString(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER);
- this.interpreterGroupOrderList = groupOrder.split(",");
-
- GsonBuilder gsonBuilder = new GsonBuilder();
- gsonBuilder.setPrettyPrinting();
- this.gson = gsonBuilder.create();
-
+ this.angularObjectRegistryListener = angularObjectRegistryListener;
+ this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
+ this.appEventListener = appEventListener;
init();
}
/**
- * Remember this method doesn't keep current connections after being called
+ * Load interpreter setting from interpreter-setting.json
*/
private void loadFromFile() {
- if (!Files.exists(interpreterBindingPath)) {
+ if (!Files.exists(interpreterSettingPath)) {
// nothing to read
+ LOGGER.warn("Interpreter Setting file {} doesn't exist", interpreterSettingPath);
return;
}
- InterpreterInfoSaving infoSaving;
- try (BufferedReader jsonReader =
- Files.newBufferedReader(interpreterBindingPath, StandardCharsets.UTF_8)) {
- JsonParser jsonParser = new JsonParser();
- JsonObject jsonObject = jsonParser.parse(jsonReader).getAsJsonObject();
- infoSaving = gson.fromJson(jsonObject.toString(), InterpreterInfoSaving.class);
-
- for (String k : infoSaving.interpreterSettings.keySet()) {
- InterpreterSetting setting = infoSaving.interpreterSettings.get(k);
-
- setting.convertFlatPropertiesToPropertiesWithWidgets();
-
- List<InterpreterInfo> infos = setting.getInterpreterInfos();
-
- // Convert json StringMap to Properties
- StringMap<StringMap> p = (StringMap<StringMap>) setting.getProperties();
- Map<String, InterpreterProperty> properties = new HashMap();
- for (String key : p.keySet()) {
- StringMap<String> fields = (StringMap<String>) p.get(key);
- String type = InterpreterPropertyType.TEXTAREA.getValue();
- try {
- type = InterpreterPropertyType.byValue(fields.get("type")).getValue();
- } catch (Exception e) {
- logger.warn("Incorrect type of property {} in settings {}", key,
- setting.getId());
- }
- properties.put(key, new InterpreterProperty(key, fields.get("value"), type));
- }
- setting.setProperties(properties);
-
- // Always use separate interpreter process
- // While we decided to turn this feature on always (without providing
- // enable/disable option on GUI).
- // previously created setting should turn this feature on here.
- setting.getOption().setRemote(true);
-
- setting.convertPermissionsFromUsersToOwners(
- jsonObject.getAsJsonObject("interpreterSettings").getAsJsonObject(setting.getId()));
-
- // Update transient information from InterpreterSettingRef
- InterpreterSetting interpreterSettingObject =
- interpreterSettingsRef.get(setting.getGroup());
- if (interpreterSettingObject == null) {
- logger.warn("can't get InterpreterSetting " +
- "Information From loaded Interpreter Setting Ref - {} ", setting.getGroup());
- continue;
+
+ try {
+ InterpreterInfoSaving infoSaving = InterpreterInfoSaving.loadFromFile(interpreterSettingPath);
+ //TODO(zjffdu) still ugly (should move all to InterpreterInfoSaving)
+ for (InterpreterSetting savedInterpreterSetting : infoSaving.interpreterSettings.values()) {
+ savedInterpreterSetting.setConf(conf);
+ savedInterpreterSetting.setInterpreterSettingManager(this);
+ savedInterpreterSetting.setAngularObjectRegistryListener(angularObjectRegistryListener);
+ savedInterpreterSetting.setRemoteInterpreterProcessListener(
+ remoteInterpreterProcessListener);
+ savedInterpreterSetting.setAppEventListener(appEventListener);
+ savedInterpreterSetting.setDependencyResolver(dependencyResolver);
+ savedInterpreterSetting.setProperties(InterpreterSetting.convertInterpreterProperties(
+ savedInterpreterSetting.getProperties()
+ ));
+
+ InterpreterSetting interpreterSettingTemplate =
+ interpreterSettingTemplates.get(savedInterpreterSetting.getGroup());
+ // InterpreterSettingTemplate is from interpreter-setting.json which represent the latest
+ // InterpreterSetting, while InterpreterSetting is from interpreter.json which represent
+ // the user saved interpreter setting
+ if (interpreterSettingTemplate != null) {
+ savedInterpreterSetting.setInterpreterDir(interpreterSettingTemplate.getInterpreterDir());
+ // merge properties from interpreter-setting.json and interpreter.json
+ Map<String, InterpreterProperty> mergedProperties =
+ new HashMap<>(InterpreterSetting.convertInterpreterProperties(
+ interpreterSettingTemplate.getProperties()));
+ mergedProperties.putAll(InterpreterSetting.convertInterpreterProperties(
+ savedInterpreterSetting.getProperties()));
+ savedInterpreterSetting.setProperties(mergedProperties);
+ // merge InterpreterInfo
+ savedInterpreterSetting.setInterpreterInfos(
+ interpreterSettingTemplate.getInterpreterInfos());
+ savedInterpreterSetting.setInterpreterRunner(
+ interpreterSettingTemplate.getInterpreterRunner());
+ } else {
+ LOGGER.warn("No InterpreterSetting Template found for InterpreterSetting: "
+ + savedInterpreterSetting.getGroup());
}
- String depClassPath = interpreterSettingObject.getPath();
- setting.setPath(depClassPath);
-
- for (InterpreterInfo info : infos) {
- if (info.getEditor() == null) {
- Map<String, Object> editor = getEditorFromSettingByClassName(interpreterSettingObject,
- info.getClassName());
- info.setEditor(editor);
+
+ // Overwrite the default InterpreterSetting we registered from InterpreterSetting Templates
+ // remove it first
+ for (InterpreterSetting setting : interpreterSettings.values()) {
+ if (setting.getName().equals(savedInterpreterSetting.getName())) {
+ interpreterSettings.remove(setting.getId());
}
}
-
- setting.setInterpreterGroupFactory(interpreterGroupFactory);
-
- loadInterpreterDependencies(setting);
- interpreterSettings.put(k, setting);
+ savedInterpreterSetting.postProcessing();
+ LOGGER.info("Create Interpreter Setting {} from interpreter.json",
+ savedInterpreterSetting.getName());
+ interpreterSettings.put(savedInterpreterSetting.getId(), savedInterpreterSetting);
}
interpreterBindings.putAll(infoSaving.interpreterBindings);
@@ -235,53 +228,27 @@ public class InterpreterSettingManager {
}
}
} catch (IOException e) {
- e.printStackTrace();
+ LOGGER.error("Fail to load interpreter setting configuration file: "
+ + interpreterSettingPath, e);
}
}
public void saveToFile() throws IOException {
- String jsonString;
-
synchronized (interpreterSettings) {
InterpreterInfoSaving info = new InterpreterInfoSaving();
info.interpreterBindings = interpreterBindings;
info.interpreterSettings = interpreterSettings;
info.interpreterRepositories = interpreterRepositories;
-
- jsonString = info.toJson();
+ info.saveToFile(interpreterSettingPath);
}
-
- if (!Files.exists(interpreterBindingPath)) {
- Files.createFile(interpreterBindingPath);
-
- try {
- Set<PosixFilePermission> permissions = EnumSet.of(OWNER_READ, OWNER_WRITE);
- Files.setPosixFilePermissions(interpreterBindingPath, permissions);
- } catch (UnsupportedOperationException e) {
- // File system does not support Posix file permissions (likely windows) - continue anyway.
- logger.warn("unable to setPosixFilePermissions on '{}'.", interpreterBindingPath);
- }
- }
-
- FileOutputStream fos = new FileOutputStream(interpreterBindingPath.toFile(), false);
- OutputStreamWriter out = new OutputStreamWriter(fos);
- out.append(jsonString);
- out.close();
- fos.close();
}
- //TODO(jl): Fix it to remove InterpreterGroupFactory
- public void setInterpreterGroupFactory(InterpreterGroupFactory interpreterGroupFactory) {
- for (InterpreterSetting setting : interpreterSettings.values()) {
- setting.setInterpreterGroupFactory(interpreterGroupFactory);
- }
- this.interpreterGroupFactory = interpreterGroupFactory;
- }
+ private void init() throws IOException {
- private void init() throws InterpreterException, IOException, RepositoryException {
- String interpreterJson = zeppelinConfiguration.getInterpreterJson();
+ // 1. detect interpreter setting via interpreter-setting.json in each interpreter folder
+ // 2. detect interpreter setting in interpreter.json that is saved before
+ String interpreterJson = conf.getInterpreterJson();
ClassLoader cl = Thread.currentThread().getContextClassLoader();
-
if (Files.exists(interpreterDirPath)) {
for (Path interpreterDir : Files
.newDirectoryStream(interpreterDirPath, new Filter<Path>() {
@@ -298,227 +265,144 @@ public class InterpreterSettingManager {
* interpreter-setting.json
* 2. Register it from interpreter-setting.json in classpath
* {ZEPPELIN_HOME}/interpreter/{interpreter_name}
- * 3. Register it by Interpreter.register
*/
if (!registerInterpreterFromPath(interpreterDirString, interpreterJson)) {
if (!registerInterpreterFromResource(cl, interpreterDirString, interpreterJson)) {
- /*
- * TODO(jongyoul)
- * - Remove these codes below because of legacy code
- * - Support ThreadInterpreter
- */
- URLClassLoader ccl = new URLClassLoader(
- recursiveBuildLibList(interpreterDir.toFile()), cl);
- for (String className : interpreterClassList) {
- try {
- // Load classes
- Class.forName(className, true, ccl);
- Set<String> interpreterKeys = Interpreter.registeredInterpreters.keySet();
- for (String interpreterKey : interpreterKeys) {
- if (className
- .equals(Interpreter.registeredInterpreters.get(interpreterKey)
- .getClassName())) {
- Interpreter.registeredInterpreters.get(interpreterKey)
- .setPath(interpreterDirString);
- logger.info("Interpreter " + interpreterKey + " found. class=" + className);
- cleanCl.put(interpreterDirString, ccl);
- }
- }
- } catch (Throwable t) {
- // nothing to do
- }
- }
+ LOGGER.warn("No interpreter-setting.json found in " + interpreterDirPath);
}
}
}
- }
-
- for (RegisteredInterpreter registeredInterpreter : Interpreter.registeredInterpreters
- .values()) {
- logger
- .debug("Registered: {} -> {}. Properties: {}", registeredInterpreter.getInterpreterKey(),
- registeredInterpreter.getClassName(), registeredInterpreter.getProperties());
- }
-
- // RegisteredInterpreters -> interpreterSettingRef
- InterpreterInfo interpreterInfo;
- for (RegisteredInterpreter r : Interpreter.registeredInterpreters.values()) {
- interpreterInfo =
- new InterpreterInfo(r.getClassName(), r.getName(), r.isDefaultInterpreter(),
- r.getEditor());
- add(r.getGroup(), interpreterInfo, r.getProperties(), defaultOption, r.getPath(),
- r.getRunner());
- }
-
- for (String settingId : interpreterSettingsRef.keySet()) {
- InterpreterSetting setting = interpreterSettingsRef.get(settingId);
- logger.info("InterpreterSettingRef name {}", setting.getName());
+ } else {
+ LOGGER.warn("InterpreterDir {} doesn't exist", interpreterDirPath);
}
loadFromFile();
-
- // if no interpreter settings are loaded, create default set
- if (0 == interpreterSettings.size()) {
- Map<String, InterpreterSetting> temp = new HashMap<>();
- InterpreterSetting interpreterSetting;
- for (InterpreterSetting setting : interpreterSettingsRef.values()) {
- interpreterSetting = createFromInterpreterSettingRef(setting);
- temp.put(setting.getName(), interpreterSetting);
- }
-
- for (String group : interpreterGroupOrderList) {
- if (null != (interpreterSetting = temp.remove(group))) {
- interpreterSettings.put(interpreterSetting.getId(), interpreterSetting);
- }
- }
-
- for (InterpreterSetting setting : temp.values()) {
- interpreterSettings.put(setting.getId(), setting);
- }
-
- saveToFile();
- }
-
- for (String settingId : interpreterSettings.keySet()) {
- InterpreterSetting setting = interpreterSettings.get(settingId);
- logger.info("InterpreterSetting group {} : id={}, name={}", setting.getGroup(), settingId,
- setting.getName());
- }
+ saveToFile();
}
private boolean registerInterpreterFromResource(ClassLoader cl, String interpreterDir,
- String interpreterJson) throws IOException, RepositoryException {
+ String interpreterJson) throws IOException {
URL[] urls = recursiveBuildLibList(new File(interpreterDir));
ClassLoader tempClassLoader = new URLClassLoader(urls, cl);
- Enumeration<URL> interpreterSettings = tempClassLoader.getResources(interpreterJson);
- if (!interpreterSettings.hasMoreElements()) {
+ URL url = tempClassLoader.getResource(interpreterJson);
+ if (url == null) {
return false;
}
- for (URL url : Collections.list(interpreterSettings)) {
- try (InputStream inputStream = url.openStream()) {
- logger.debug("Reading {} from {}", interpreterJson, url);
- List<RegisteredInterpreter> registeredInterpreterList =
- getInterpreterListFromJson(inputStream);
- registerInterpreters(registeredInterpreterList, interpreterDir);
- }
- }
+
+ LOGGER.debug("Reading interpreter-setting.json from {} as Resource", url);
+ List<RegisteredInterpreter> registeredInterpreterList =
+ getInterpreterListFromJson(url.openStream());
+ registerInterpreterSetting(registeredInterpreterList, interpreterDir);
return true;
}
private boolean registerInterpreterFromPath(String interpreterDir, String interpreterJson)
- throws IOException, RepositoryException {
+ throws IOException {
Path interpreterJsonPath = Paths.get(interpreterDir, interpreterJson);
if (Files.exists(interpreterJsonPath)) {
- logger.debug("Reading {}", interpreterJsonPath);
+ LOGGER.debug("Reading interpreter-setting.json from file {}", interpreterJsonPath);
List<RegisteredInterpreter> registeredInterpreterList =
- getInterpreterListFromJson(interpreterJsonPath);
- registerInterpreters(registeredInterpreterList, interpreterDir);
+ getInterpreterListFromJson(new FileInputStream(interpreterJsonPath.toFile()));
+ registerInterpreterSetting(registeredInterpreterList, interpreterDir);
return true;
}
return false;
}
- private List<RegisteredInterpreter> getInterpreterListFromJson(Path filename)
- throws FileNotFoundException {
- return getInterpreterListFromJson(new FileInputStream(filename.toFile()));
- }
-
private List<RegisteredInterpreter> getInterpreterListFromJson(InputStream stream) {
Type registeredInterpreterListType = new TypeToken<List<RegisteredInterpreter>>() {
}.getType();
return gson.fromJson(new InputStreamReader(stream), registeredInterpreterListType);
}
- private void registerInterpreters(List<RegisteredInterpreter> registeredInterpreters,
- String absolutePath) throws IOException, RepositoryException {
+ private void registerInterpreterSetting(List<RegisteredInterpreter> registeredInterpreters,
+ String interpreterDir) throws IOException {
+ Map<String, DefaultInterpreterProperty> properties = new HashMap<>();
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ InterpreterOption option = defaultOption;
+ String group = null;
+ InterpreterRunner runner = null;
for (RegisteredInterpreter registeredInterpreter : registeredInterpreters) {
+ //TODO(zjffdu) merge RegisteredInterpreter & InterpreterInfo
InterpreterInfo interpreterInfo =
new InterpreterInfo(registeredInterpreter.getClassName(), registeredInterpreter.getName(),
registeredInterpreter.isDefaultInterpreter(), registeredInterpreter.getEditor());
+ group = registeredInterpreter.getGroup();
+ runner = registeredInterpreter.getRunner();
// use defaultOption if it is not specified in interpreter-setting.json
- InterpreterOption option = registeredInterpreter.getOption() == null ? defaultOption :
- registeredInterpreter.getOption();
- add(registeredInterpreter.getGroup(), interpreterInfo, registeredInterpreter.getProperties(),
- option, absolutePath, registeredInterpreter.getRunner());
- }
-
- }
-
- public InterpreterSetting getDefaultInterpreterSetting(List<InterpreterSetting> settings) {
- if (settings == null || settings.isEmpty()) {
- return null;
- }
- return settings.get(0);
- }
-
+ if (registeredInterpreter.getOption() != null) {
+ option = registeredInterpreter.getOption();
+ }
+ properties.putAll(registeredInterpreter.getProperties());
+ interpreterInfos.add(interpreterInfo);
+ }
+
+ InterpreterSetting interpreterSettingTemplate = new InterpreterSetting.Builder()
+ .setGroup(group)
+ .setName(group)
+ .setInterpreterInfos(interpreterInfos)
+ .setProperties(properties)
+ .setDependencies(new ArrayList<Dependency>())
+ .setOption(option)
+ .setRunner(runner)
+ .setInterpreterDir(interpreterDir)
+ .setRunner(runner)
+ .setConf(conf)
+ .setIntepreterSettingManager(this)
+ .create();
+
+ LOGGER.info("Register InterpreterSettingTemplate & InterpreterSetting: {}",
+ interpreterSettingTemplate.getName());
+ interpreterSettingTemplates.put(interpreterSettingTemplate.getName(),
+ interpreterSettingTemplate);
+
+ InterpreterSetting interpreterSetting = new InterpreterSetting(interpreterSettingTemplate);
+ interpreterSetting.setAngularObjectRegistryListener(angularObjectRegistryListener);
+ interpreterSetting.setRemoteInterpreterProcessListener(remoteInterpreterProcessListener);
+ interpreterSetting.setAppEventListener(appEventListener);
+ interpreterSetting.setDependencyResolver(dependencyResolver);
+ interpreterSetting.setInterpreterSettingManager(this);
+ interpreterSetting.postProcessing();
+ interpreterSettings.put(interpreterSetting.getId(), interpreterSetting);
+ }
+
+ @VisibleForTesting
public InterpreterSetting getDefaultInterpreterSetting(String noteId) {
- return getDefaultInterpreterSetting(getInterpreterSettings(noteId));
+ return getInterpreterSettings(noteId).get(0);
}
public List<InterpreterSetting> getInterpreterSettings(String noteId) {
- List<String> interpreterSettingIds = getNoteInterpreterSettingBinding(noteId);
- LinkedList<InterpreterSetting> settings = new LinkedList<>();
-
- Iterator<String> iter = interpreterSettingIds.iterator();
- while (iter.hasNext()) {
- String id = iter.next();
- InterpreterSetting setting = get(id);
- if (setting == null) {
- // interpreter setting is removed from factory. remove id from here, too
- iter.remove();
- } else {
- settings.add(setting);
+ List<InterpreterSetting> settings = new ArrayList<>();
+ synchronized (interpreterSettings) {
+ List<String> interpreterSettingIds = interpreterBindings.get(noteId);
+ if (interpreterSettingIds != null) {
+ for (String settingId : interpreterSettingIds) {
+ if (interpreterSettings.containsKey(settingId)) {
+ settings.add(interpreterSettings.get(settingId));
+ } else {
+ LOGGER.warn("InterpreterSetting {} has been removed, but note {} still bind to it.",
+ settingId, noteId);
+ }
+ }
}
}
return settings;
}
- private List<String> getNoteInterpreterSettingBinding(String noteId) {
- LinkedList<String> bindings = new LinkedList<>();
- List<String> settingIds = interpreterBindings.get(noteId);
- if (settingIds != null) {
- bindings.addAll(settingIds);
- }
- return bindings;
- }
-
- private InterpreterSetting createFromInterpreterSettingRef(String name) {
- Preconditions.checkNotNull(name, "reference name should be not null");
- InterpreterSetting settingRef = interpreterSettingsRef.get(name);
- return createFromInterpreterSettingRef(settingRef);
- }
-
- private InterpreterSetting createFromInterpreterSettingRef(InterpreterSetting o) {
- // should return immutable objects
- List<InterpreterInfo> infos = (null == o.getInterpreterInfos()) ?
- new ArrayList<InterpreterInfo>() : new ArrayList<>(o.getInterpreterInfos());
- List<Dependency> deps = (null == o.getDependencies()) ?
- new ArrayList<Dependency>() : new ArrayList<>(o.getDependencies());
- Map<String, InterpreterProperty> props =
- convertInterpreterProperties((Map<String, DefaultInterpreterProperty>) o.getProperties());
- InterpreterOption option = InterpreterOption.fromInterpreterOption(o.getOption());
-
- InterpreterSetting setting = new InterpreterSetting(o.getName(), o.getName(),
- infos, props, deps, option, o.getPath(), o.getInterpreterRunner());
- setting.setInterpreterGroupFactory(interpreterGroupFactory);
- return setting;
- }
-
- private Map<String, InterpreterProperty> convertInterpreterProperties(
- Map<String, DefaultInterpreterProperty> defaultProperties) {
- Map<String, InterpreterProperty> properties = new HashMap<>();
-
- for (String key : defaultProperties.keySet()) {
- DefaultInterpreterProperty defaultInterpreterProperty = defaultProperties.get(key);
- properties.put(key, new InterpreterProperty(key, defaultInterpreterProperty.getValue(),
- defaultInterpreterProperty.getType()));
+ public ManagedInterpreterGroup getInterpreterGroupById(String groupId) {
+ for (InterpreterSetting setting : interpreterSettings.values()) {
+ ManagedInterpreterGroup interpreterGroup = setting.getInterpreterGroup(groupId);
+ if (interpreterGroup != null) {
+ return interpreterGroup;
+ }
}
- return properties;
+ return null;
}
+ //TODO(zjffdu) logic here is a little ugly
public Map<String, Object> getEditorSetting(Interpreter interpreter, String user, String noteId,
String replName) {
Map<String, Object> editor = DEFAULT_EDITOR;
@@ -533,97 +417,133 @@ public class InterpreterSettingManager {
}
// when replName is 'name' of interpreter
if (defaultSettingName.equals(intpSetting.getName())) {
- editor = getEditorFromSettingByClassName(intpSetting, interpreter.getClassName());
+ editor = intpSetting.getEditorFromSettingByClassName(interpreter.getClassName());
}
// when replName is 'alias name' of interpreter or 'group' of interpreter
if (replName.equals(intpSetting.getName()) || group.equals(intpSetting.getName())) {
- editor = getEditorFromSettingByClassName(intpSetting, interpreter.getClassName());
+ editor = intpSetting.getEditorFromSettingByClassName(interpreter.getClassName());
break;
}
}
} catch (NullPointerException e) {
// Use `debug` level because this log occurs frequently
- logger.debug("Couldn't get interpreter editor setting");
+ LOGGER.debug("Couldn't get interpreter editor setting");
}
return editor;
}
- public Map<String, Object> getEditorFromSettingByClassName(InterpreterSetting intpSetting,
- String className) {
- List<InterpreterInfo> intpInfos = intpSetting.getInterpreterInfos();
- for (InterpreterInfo intpInfo : intpInfos) {
+ public List<ManagedInterpreterGroup> getAllInterpreterGroup() {
+ List<ManagedInterpreterGroup> interpreterGroups = new ArrayList<>();
+ for (InterpreterSetting interpreterSetting : interpreterSettings.values()) {
+ interpreterGroups.addAll(interpreterSetting.getAllInterpreterGroups());
+ }
+ return interpreterGroups;
+ }
- if (className.equals(intpInfo.getClassName())) {
- if (intpInfo.getEditor() == null) {
- break;
+ //TODO(zjffdu) move Resource related api to ResourceManager
+ public ResourceSet getAllResources() {
+ return getAllResourcesExcept(null);
+ }
+
+ private ResourceSet getAllResourcesExcept(String interpreterGroupExcludsion) {
+ ResourceSet resourceSet = new ResourceSet();
+ for (ManagedInterpreterGroup intpGroup : getAllInterpreterGroup()) {
+ if (interpreterGroupExcludsion != null &&
+ intpGroup.getId().equals(interpreterGroupExcludsion)) {
+ continue;
+ }
+
+ RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
+ if (remoteInterpreterProcess == null) {
+ ResourcePool localPool = intpGroup.getResourcePool();
+ if (localPool != null) {
+ resourceSet.addAll(localPool.getAll());
+ }
+ } else if (remoteInterpreterProcess.isRunning()) {
+ List<String> resourceList = remoteInterpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<List<String>>() {
+ @Override
+ public List<String> call(RemoteInterpreterService.Client client) throws Exception {
+ return client.resourcePoolGetAll();
+ }
+ });
+ for (String res : resourceList) {
+ resourceSet.add(Resource.fromJson(res));
}
- return intpInfo.getEditor();
}
}
- return DEFAULT_EDITOR;
+ return resourceSet;
}
- private void loadInterpreterDependencies(final InterpreterSetting setting) {
- setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
- setting.setErrorReason(null);
- interpreterSettings.put(setting.getId(), setting);
- synchronized (interpreterSettings) {
- final Thread t = new Thread() {
- public void run() {
- try {
- // dependencies to prevent library conflict
- File localRepoDir = new File(zeppelinConfiguration.getInterpreterLocalRepoPath() + "/" +
- setting.getId());
- if (localRepoDir.exists()) {
- try {
- FileUtils.forceDelete(localRepoDir);
- } catch (FileNotFoundException e) {
- logger.info("A file that does not exist cannot be deleted, nothing to worry", e);
+ public void removeResourcesBelongsToParagraph(String noteId, String paragraphId) {
+ for (ManagedInterpreterGroup intpGroup : getAllInterpreterGroup()) {
+ ResourceSet resourceSet = new ResourceSet();
+ RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
+ if (remoteInterpreterProcess == null) {
+ ResourcePool localPool = intpGroup.getResourcePool();
+ if (localPool != null) {
+ resourceSet.addAll(localPool.getAll());
+ }
+ if (noteId != null) {
+ resourceSet = resourceSet.filterByNoteId(noteId);
+ }
+ if (paragraphId != null) {
+ resourceSet = resourceSet.filterByParagraphId(paragraphId);
+ }
+
+ for (Resource r : resourceSet) {
+ localPool.remove(
+ r.getResourceId().getNoteId(),
+ r.getResourceId().getParagraphId(),
+ r.getResourceId().getName());
+ }
+ } else if (remoteInterpreterProcess.isRunning()) {
+ List<String> resourceList = remoteInterpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<List<String>>() {
+ @Override
+ public List<String> call(RemoteInterpreterService.Client client) throws Exception {
+ return client.resourcePoolGetAll();
}
- }
+ });
+ for (String res : resourceList) {
+ resourceSet.add(Resource.fromJson(res));
+ }
- // load dependencies
- List<Dependency> deps = setting.getDependencies();
- if (deps != null) {
- for (Dependency d : deps) {
- File destDir = new File(
- zeppelinConfiguration.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
+ if (noteId != null) {
+ resourceSet = resourceSet.filterByNoteId(noteId);
+ }
+ if (paragraphId != null) {
+ resourceSet = resourceSet.filterByParagraphId(paragraphId);
+ }
- if (d.getExclusions() != null) {
- dependencyResolver.load(d.getGroupArtifactVersion(), d.getExclusions(),
- new File(destDir, setting.getId()));
- } else {
- dependencyResolver
- .load(d.getGroupArtifactVersion(), new File(destDir, setting.getId()));
+ for (final Resource r : resourceSet) {
+ remoteInterpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<Void>() {
+
+ @Override
+ public Void call(RemoteInterpreterService.Client client) throws Exception {
+ client.resourceRemove(
+ r.getResourceId().getNoteId(),
+ r.getResourceId().getParagraphId(),
+ r.getResourceId().getName());
+ return null;
}
- }
- }
-
- setting.setStatus(InterpreterSetting.Status.READY);
- setting.setErrorReason(null);
- } catch (Exception e) {
- logger.error(String.format("Error while downloading repos for interpreter group : %s," +
- " go to interpreter setting page click on edit and save it again to make " +
- "this interpreter work properly. : %s",
- setting.getGroup(), e.getLocalizedMessage()), e);
- setting.setErrorReason(e.getLocalizedMessage());
- setting.setStatus(InterpreterSetting.Status.ERROR);
- } finally {
- interpreterSettings.put(setting.getId(), setting);
- }
+ });
}
- };
- t.start();
+ }
}
}
+ public void removeResourcesBelongsToNote(String noteId) {
+ removeResourcesBelongsToParagraph(noteId, null);
+ }
+
/**
* Overwrite dependency jar under local-repo/{interpreterId}
* if jar file in original path is changed
*/
private void copyDependenciesFromLocalPath(final InterpreterSetting setting) {
setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
- interpreterSettings.put(setting.getId(), setting);
synchronized (interpreterSettings) {
final Thread t = new Thread() {
public void run() {
@@ -632,7 +552,7 @@ public class InterpreterSettingManager {
if (deps != null) {
for (Dependency d : deps) {
File destDir = new File(
- zeppelinConfiguration.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
+ conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
int numSplits = d.getGroupArtifactVersion().split(":").length;
if (!(numSplits >= 3 && numSplits <= 6)) {
@@ -643,14 +563,14 @@ public class InterpreterSettingManager {
}
setting.setStatus(InterpreterSetting.Status.READY);
} catch (Exception e) {
- logger.error(String.format("Error while copying deps for interpreter group : %s," +
+ LOGGER.error(String.format("Error while copying deps for interpreter group : %s," +
" go to interpreter setting page click on edit and save it again to make " +
"this interpreter work properly.",
setting.getGroup()), e);
setting.setErrorReason(e.getLocalizedMessage());
setting.setStatus(InterpreterSetting.Status.ERROR);
} finally {
- interpreterSettings.put(setting.getId(), setting);
+
}
}
};
@@ -663,220 +583,107 @@ public class InterpreterSettingManager {
* The list does not contain more than one setting from the same interpreter class.
* Order by InterpreterClass (order defined by ZEPPELIN_INTERPRETERS), Interpreter setting name
*/
- public List<String> getDefaultInterpreterSettingList() {
- // this list will contain default interpreter setting list
- List<String> defaultSettings = new LinkedList<>();
-
- // to ignore the same interpreter group
- Map<String, Boolean> interpreterGroupCheck = new HashMap<>();
-
- List<InterpreterSetting> sortedSettings = get();
-
- for (InterpreterSetting setting : sortedSettings) {
- if (defaultSettings.contains(setting.getId())) {
- continue;
- }
-
- if (!interpreterGroupCheck.containsKey(setting.getName())) {
- defaultSettings.add(setting.getId());
- interpreterGroupCheck.put(setting.getName(), true);
- }
- }
- return defaultSettings;
- }
-
- List<RegisteredInterpreter> getRegisteredInterpreterList() {
- return new ArrayList<>(Interpreter.registeredInterpreters.values());
- }
-
-
- private boolean findDefaultInterpreter(List<InterpreterInfo> infos) {
- for (InterpreterInfo interpreterInfo : infos) {
- if (interpreterInfo.isDefaultInterpreter()) {
- return true;
- }
+ public List<String> getInterpreterSettingIds() {
+ List<String> settingIdList = new ArrayList<>();
+ for (InterpreterSetting interpreterSetting : get()) {
+ settingIdList.add(interpreterSetting.getId());
}
- return false;
+ return settingIdList;
}
public InterpreterSetting createNewSetting(String name, String group,
List<Dependency> dependencies, InterpreterOption option, Map<String, InterpreterProperty> p)
throws IOException {
+
if (name.indexOf(".") >= 0) {
throw new IOException("'.' is invalid for InterpreterSetting name.");
}
- InterpreterSetting setting = createFromInterpreterSettingRef(group);
+ // check if name is existed
+ for (InterpreterSetting interpreterSetting : interpreterSettings.values()) {
+ if (interpreterSetting.getName().equals(name)) {
+ throw new IOException("Interpreter " + name + " already existed");
+ }
+ }
+ InterpreterSetting setting = new InterpreterSetting(interpreterSettingTemplates.get(group));
setting.setName(name);
setting.setGroup(group);
+ //TODO(zjffdu) Should use setDependencies
setting.appendDependencies(dependencies);
setting.setInterpreterOption(option);
setting.setProperties(p);
- setting.setInterpreterGroupFactory(interpreterGroupFactory);
+ setting.setAppEventListener(appEventListener);
+ setting.setRemoteInterpreterProcessListener(remoteInterpreterProcessListener);
+ setting.setDependencyResolver(dependencyResolver);
+ setting.setAngularObjectRegistryListener(angularObjectRegistryListener);
+ setting.setInterpreterSettingManager(this);
+ setting.postProcessing();
interpreterSettings.put(setting.getId(), setting);
- loadInterpreterDependencies(setting);
saveToFile();
return setting;
}
- private InterpreterSetting add(String group, InterpreterInfo interpreterInfo,
- Map<String, DefaultInterpreterProperty> interpreterProperties, InterpreterOption option,
- String path, InterpreterRunner runner)
- throws InterpreterException, IOException, RepositoryException {
- ArrayList<InterpreterInfo> infos = new ArrayList<>();
- infos.add(interpreterInfo);
- return add(group, infos, new ArrayList<Dependency>(), option, interpreterProperties, path,
- runner);
- }
-
- /**
- * @param group InterpreterSetting reference name
- */
- public InterpreterSetting add(String group, ArrayList<InterpreterInfo> interpreterInfos,
- List<Dependency> dependencies, InterpreterOption option,
- Map<String, DefaultInterpreterProperty> interpreterProperties, String path,
- InterpreterRunner runner) {
- Preconditions.checkNotNull(group, "name should not be null");
- Preconditions.checkNotNull(interpreterInfos, "interpreterInfos should not be null");
- Preconditions.checkNotNull(dependencies, "dependencies should not be null");
- Preconditions.checkNotNull(option, "option should not be null");
- Preconditions.checkNotNull(interpreterProperties, "properties should not be null");
-
- InterpreterSetting interpreterSetting;
-
- synchronized (interpreterSettingsRef) {
- if (interpreterSettingsRef.containsKey(group)) {
- interpreterSetting = interpreterSettingsRef.get(group);
-
- // Append InterpreterInfo
- List<InterpreterInfo> infos = interpreterSetting.getInterpreterInfos();
- boolean hasDefaultInterpreter = findDefaultInterpreter(infos);
- for (InterpreterInfo interpreterInfo : interpreterInfos) {
- if (!infos.contains(interpreterInfo)) {
- if (!hasDefaultInterpreter && interpreterInfo.isDefaultInterpreter()) {
- hasDefaultInterpreter = true;
- infos.add(0, interpreterInfo);
- } else {
- infos.add(interpreterInfo);
- }
- }
- }
-
- // Append dependencies
- List<Dependency> dependencyList = interpreterSetting.getDependencies();
- for (Dependency dependency : dependencies) {
- if (!dependencyList.contains(dependency)) {
- dependencyList.add(dependency);
- }
- }
-
- // Append properties
- Map<String, DefaultInterpreterProperty> properties =
- (Map<String, DefaultInterpreterProperty>) interpreterSetting.getProperties();
- for (String key : interpreterProperties.keySet()) {
- if (!properties.containsKey(key)) {
- properties.put(key, interpreterProperties.get(key));
- }
- }
-
- } else {
- interpreterSetting =
- new InterpreterSetting(group, null, interpreterInfos, interpreterProperties,
- dependencies, option, path, runner);
- interpreterSettingsRef.put(group, interpreterSetting);
- }
- }
-
- if (dependencies.size() > 0) {
- loadInterpreterDependencies(interpreterSetting);
- }
-
- interpreterSetting.setInterpreterGroupFactory(interpreterGroupFactory);
- return interpreterSetting;
+ @VisibleForTesting
+ public void addInterpreterSetting(InterpreterSetting interpreterSetting) {
+ interpreterSettingTemplates.put(interpreterSetting.getName(), interpreterSetting);
+ interpreterSetting.setAppEventListener(appEventListener);
+ interpreterSetting.setDependencyResolver(dependencyResolver);
+ interpreterSetting.setAngularObjectRegistryListener(angularObjectRegistryListener);
+ interpreterSetting.setRemoteInterpreterProcessListener(remoteInterpreterProcessListener);
+ interpreterSetting.setInterpreterSettingManager(this);
+ interpreterSettings.put(interpreterSetting.getId(), interpreterSetting);
}
/**
* map interpreter ids into noteId
*
+ * @param user user name
* @param noteId note id
- * @param ids InterpreterSetting id list
+ * @param settingIdList InterpreterSetting id list
*/
- public void setInterpreters(String user, String noteId, List<String> ids) throws IOException {
- putNoteInterpreterSettingBinding(user, noteId, ids);
- }
-
- private void putNoteInterpreterSettingBinding(String user, String noteId,
- List<String> settingList) throws IOException {
- List<String> unBindedSettings = new LinkedList<>();
+ public void setInterpreterBinding(String user, String noteId, List<String> settingIdList)
+ throws IOException {
+ List<String> unBindedSettingIdList = new LinkedList<>();
synchronized (interpreterSettings) {
- List<String> oldSettings = interpreterBindings.get(noteId);
- if (oldSettings != null) {
- for (String oldSettingId : oldSettings) {
- if (!settingList.contains(oldSettingId)) {
- unBindedSettings.add(oldSettingId);
+ List<String> oldSettingIdList = interpreterBindings.get(noteId);
+ if (oldSettingIdList != null) {
+ for (String oldSettingId : oldSettingIdList) {
+ if (!settingIdList.contains(oldSettingId)) {
+ unBindedSettingIdList.add(oldSettingId);
}
}
}
- interpreterBindings.put(noteId, settingList);
+ interpreterBindings.put(noteId, settingIdList);
saveToFile();
- for (String settingId : unBindedSettings) {
- InterpreterSetting setting = get(settingId);
- removeInterpretersForNote(setting, user, noteId);
+ for (String settingId : unBindedSettingIdList) {
+ InterpreterSetting interpreterSetting = interpreterSettings.get(settingId);
+ //TODO(zjffdu) Add test for this scenario
+ //only close Interpreters when it is note scoped
+ if (interpreterSetting.getOption().perNoteIsolated() ||
+ interpreterSetting.getOption().perNoteScoped()) {
+ interpreterSetting.closeInterpreters(user, noteId);
+ }
}
}
}
- public void removeInterpretersForNote(InterpreterSetting interpreterSetting, String user,
- String noteId) {
- //TODO(jl): This is only for hotfix. You should fix it as a beautiful way
- InterpreterOption interpreterOption = interpreterSetting.getOption();
- if (!(InterpreterOption.SHARED.equals(interpreterOption.perNote)
- && InterpreterOption.SHARED.equals(interpreterOption.perUser))) {
- interpreterSetting.closeAndRemoveInterpreterGroup(noteId, "");
- }
- }
-
- public String getInterpreterSessionKey(String user, String noteId, InterpreterSetting setting) {
- InterpreterOption option = setting.getOption();
- String key;
- if (option.isExistingProcess()) {
- key = Constants.EXISTING_PROCESS;
- } else if (option.perNoteScoped() && option.perUserScoped()) {
- key = user + ":" + noteId;
- } else if (option.perUserScoped()) {
- key = user;
- } else if (option.perNoteScoped()) {
- key = noteId;
- } else {
- key = SHARED_SESSION;
- }
-
- logger.debug("Interpreter session key: {}, for note: {}, user: {}, InterpreterSetting Name: " +
- "{}", key, noteId, user, setting.getName());
- return key;
- }
-
-
- public List<String> getInterpreters(String noteId) {
- return getNoteInterpreterSettingBinding(noteId);
+ public List<String> getInterpreterBinding(String noteId) {
+ return interpreterBindings.get(noteId);
}
+ @VisibleForTesting
public void closeNote(String user, String noteId) {
// close interpreters in this note session
+ LOGGER.info("Close Note: {}", noteId);
List<InterpreterSetting> settings = getInterpreterSettings(noteId);
- if (settings == null || settings.size() == 0) {
- return;
- }
-
- logger.info("closeNote: {}", noteId);
for (InterpreterSetting setting : settings) {
- removeInterpretersForNote(setting, user, noteId);
+ setting.closeInterpreters(user, noteId);
}
}
- public Map<String, InterpreterSetting> getAvailableInterpreterSettings() {
- return interpreterSettingsRef;
+ public Map<String, InterpreterSetting> getInterpreterSettingTemplates() {
+ return interpreterSettingTemplates;
}
private URL[] recursiveBuildLibList(File path) throws MalformedURLException {
@@ -914,36 +721,25 @@ public class InterpreterSettingManager {
}
public void removeNoteInterpreterSettingBinding(String user, String noteId) throws IOException {
- List<String> settingIds = interpreterBindings.remove(noteId);
- if (settingIds != null) {
- for (String settingId : settingIds) {
- InterpreterSetting setting = get(settingId);
- if (setting != null) {
- this.removeInterpretersForNote(setting, user, noteId);
- }
- }
- }
- saveToFile();
+ setInterpreterBinding(user, noteId, new ArrayList<String>());
+ interpreterBindings.remove(noteId);
}
/**
* Change interpreter property and restart
*/
public void setPropertyAndRestart(String id, InterpreterOption option,
- Map<String, InterpreterProperty> properties,
- List<Dependency> dependencies) throws IOException {
+ Map<String, InterpreterProperty> properties,
+ List<Dependency> dependencies) throws IOException {
synchronized (interpreterSettings) {
InterpreterSetting intpSetting = interpreterSettings.get(id);
if (intpSetting != null) {
try {
- stopJobAllInterpreter(intpSetting);
-
- intpSetting.closeAndRemoveAllInterpreterGroups();
+ intpSetting.close();
intpSetting.setOption(option);
intpSetting.setProperties(properties);
intpSetting.setDependencies(dependencies);
- loadInterpreterDependencies(intpSetting);
-
+ intpSetting.postProcessing();
saveToFile();
} catch (Exception e) {
loadFromFile();
@@ -955,6 +751,7 @@ public class InterpreterSettingManager {
}
}
+ // restart in note page
public void restart(String settingId, String noteId, String user) {
InterpreterSetting intpSetting = interpreterSettings.get(settingId);
Preconditions.checkNotNull(intpSetting);
@@ -967,11 +764,10 @@ public class InterpreterSettingManager {
intpSetting.setInfos(null);
copyDependenciesFromLocalPath(intpSetting);
- stopJobAllInterpreter(intpSetting);
if (user.equals("anonymous")) {
- intpSetting.closeAndRemoveAllInterpreterGroups();
+ intpSetting.close();
} else {
- intpSetting.closeAndRemoveInterpreterGroup(noteId, user);
+ intpSetting.closeInterpreters(user, noteId);
}
} else {
@@ -984,39 +780,33 @@ public class InterpreterSettingManager {
restart(id, "", "anonymous");
}
- private void stopJobAllInterpreter(InterpreterSetting intpSetting) {
- if (intpSetting != null) {
- for (InterpreterGroup intpGroup : intpSetting.getAllInterpreterGroups()) {
- for (List<Interpreter> interpreters : intpGroup.values()) {
- for (Interpreter intp : interpreters) {
- for (Job job : intp.getScheduler().getJobsRunning()) {
- job.abort();
- job.setStatus(Status.ABORT);
- logger.info("Job " + job.getJobName() + " aborted ");
- }
- for (Job job : intp.getScheduler().getJobsWaiting()) {
- job.abort();
- job.setStatus(Status.ABORT);
- logger.info("Job " + job.getJobName() + " aborted ");
- }
- }
- }
- }
+ public InterpreterSetting get(String id) {
+ synchronized (interpreterSettings) {
+ return interpreterSettings.get(id);
}
}
- public InterpreterSetting get(String name) {
- synchronized (interpreterSettings) {
- return interpreterSettings.get(name);
+ @VisibleForTesting
+ public InterpreterSetting getByName(String name) {
+ for (InterpreterSetting interpreterSetting : interpreterSettings.values()) {
+ if (interpreterSetting.getName().equals(name)) {
+ return interpreterSetting;
+ }
}
+ throw new RuntimeException("No InterpreterSetting: " + name);
}
public void remove(String id) throws IOException {
+ // 1. close interpreter groups of this interpreter setting
+ // 2. remove this interpreter setting
+ // 3. remove this interpreter setting from note binding
+ // 4. clean local repo directory
+ LOGGER.info("Remove interpreter setting: " + id);
synchronized (interpreterSettings) {
if (interpreterSettings.containsKey(id)) {
- InterpreterSetting intp = interpreterSettings.get(id);
- intp.closeAndRemoveAllInterpreterGroups();
+ InterpreterSetting intp = interpreterSettings.get(id);
+ intp.close();
interpreterSettings.remove(id);
for (List<String> settings : interpreterBindings.values()) {
Iterator<String> it = settings.iterator();
@@ -1031,7 +821,7 @@ public class InterpreterSettingManager {
}
}
- File localRepoDir = new File(zeppelinConfiguration.getInterpreterLocalRepoPath() + "/" + id);
+ File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + id);
FileUtils.deleteDirectory(localRepoDir);
}
@@ -1040,84 +830,58 @@ public class InterpreterSettingManager {
*/
public List<InterpreterSetting> get() {
synchronized (interpreterSettings) {
- List<InterpreterSetting> orderedSettings = new LinkedList<>();
-
- Map<String, List<InterpreterSetting>> nameInterpreterSettingMap = new HashMap<>();
- for (InterpreterSetting interpreterSetting : interpreterSettings.values()) {
- String group = interpreterSetting.getGroup();
- if (!nameInterpreterSettingMap.containsKey(group)) {
- nameInterpreterSettingMap.put(group, new ArrayList<InterpreterSetting>());
- }
- nameInterpreterSettingMap.get(group).add(interpreterSetting);
- }
-
- for (String groupName : interpreterGroupOrderList) {
- List<InterpreterSetting> interpreterSettingList =
- nameInterpreterSettingMap.remove(groupName);
- if (null != interpreterSettingList) {
- for (InterpreterSetting interpreterSetting : interpreterSettingList) {
- orderedSettings.add(interpreterSetting);
- }
- }
- }
-
- List<InterpreterSetting> settings = new ArrayList<>();
-
- for (List<InterpreterSetting> interpreterSettingList : nameInterpreterSettingMap.values()) {
- for (InterpreterSetting interpreterSetting : interpreterSettingList) {
- settings.add(interpreterSetting);
- }
- }
-
- Collections.sort(settings, new Comparator<InterpreterSetting>() {
+ List<InterpreterSetting> orderedSettings = new ArrayList<>(interpreterSettings.values());
+ Collections.sort(orderedSettings, new Comparator<InterpreterSetting>() {
@Override
public int compare(InterpreterSetting o1, InterpreterSetting o2) {
- return o1.getName().compareTo(o2.getName());
+ int i = interpreterGroupOrderList.indexOf(o1.getGroup());
+ int j = interpreterGroupOrderList.indexOf(o2.getGroup());
+ if (i < 0) {
+ LOGGER.warn("InterpreterGroup " + o1.getGroup()
+ + " is not specified in " + ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName());
+ // move the unknown interpreter to last
+ i = Integer.MAX_VALUE;
+ }
+ if (j < 0) {
+ LOGGER.warn("InterpreterGroup " + o2.getGroup()
+ + " is not specified in " + ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName());
+ // move the unknown interpreter to last
+ j = Integer.MAX_VALUE;
+ }
+ if (i < j) {
+ return -1;
+ } else if (i > j) {
+ return 1;
+ } else {
+ return 0;
+ }
}
});
-
- orderedSettings.addAll(settings);
-
return orderedSettings;
}
}
- public void close(InterpreterSetting interpreterSetting) {
- interpreterSetting.closeAndRemoveAllInterpreterGroups();
- }
-
- public void close() {
- List<Thread> closeThreads = new LinkedList<>();
- synchronized (interpreterSettings) {
- Collection<InterpreterSetting> intpSettings = interpreterSettings.values();
- for (final InterpreterSetting intpSetting : intpSettings) {
- Thread t = new Thread() {
- public void run() {
- intpSetting.closeAndRemoveAllInterpreterGroups();
- }
- };
- t.start();
- closeThreads.add(t);
- }
+ @VisibleForTesting
+ public List<String> getSettingIds() {
+ List<String> settingIds = new ArrayList<>();
+ for (InterpreterSetting interpreterSetting : get()) {
+ settingIds.add(interpreterSetting.getId());
}
+ return settingIds;
+ }
- for (Thread t : closeThreads) {
- try {
- t.join();
- } catch (InterruptedException e) {
- logger.error("Can't close interpreterGroup", e);
- }
- }
+ public void close(String settingId) {
+ get(settingId).close();
}
- public void shutdown() {
+ public void close() {
List<Thread> closeThreads = new LinkedList<>();
synchronized (interpreterSettings) {
Collection<InterpreterSetting> intpSettings = interpreterSettings.values();
for (final InterpreterSetting intpSetting : intpSettings) {
Thread t = new Thread() {
public void run() {
- intpSetting.shutdownAndRemoveAllInterpreterGroups();
+ intpSetting.close();
}
};
t.start();
@@ -1129,8 +893,9 @@ public class InterpreterSettingManager {
try {
t.join();
} catch (InterruptedException e) {
- logger.error("Can't close interpreterGroup", e);
+ LOGGER.error("Can't close interpreterGroup", e);
}
}
}
+
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java
new file mode 100644
index 0000000..1d7d916
--- /dev/null
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.zeppelin.interpreter;
+
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
+import org.apache.zeppelin.scheduler.Job;
+import org.apache.zeppelin.scheduler.Scheduler;
+import org.apache.zeppelin.scheduler.SchedulerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * ManagedInterpreterGroup runs under zeppelin server
+ */
+public class ManagedInterpreterGroup extends InterpreterGroup {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ManagedInterpreterGroup.class);
+
+ private InterpreterSetting interpreterSetting;
+ private RemoteInterpreterProcess remoteInterpreterProcess; // attached remote interpreter process
+
+ /**
+ * Create InterpreterGroup with given id and interpreterSetting, used in ZeppelinServer
+ * @param id
+ * @param interpreterSetting
+ */
+ ManagedInterpreterGroup(String id, InterpreterSetting interpreterSetting) {
+ super(id);
+ this.interpreterSetting = interpreterSetting;
+ }
+
+ public InterpreterSetting getInterpreterSetting() {
+ return interpreterSetting;
+ }
+
+ public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess() {
+ if (remoteInterpreterProcess == null) {
+ LOGGER.info("Create InterperterProcess for InterpreterGroup: " + getId());
+ remoteInterpreterProcess = interpreterSetting.createInterpreterProcess();
+ }
+ return remoteInterpreterProcess;
+ }
+
+ public RemoteInterpreterProcess getRemoteInterpreterProcess() {
+ return remoteInterpreterProcess;
+ }
+
+
+ /**
+ * Close all interpreter instances in this group
+ */
+ public synchronized void close() {
+ LOGGER.info("Close InterpreterGroup: " + id);
+ for (String sessionId : sessions.keySet()) {
+ close(sessionId);
+ }
+ }
+
+ /**
+ * Close all interpreter instances in this session
+ * @param sessionId
+ */
+ public synchronized void close(String sessionId) {
+ LOGGER.info("Close Session: " + sessionId);
+ close(sessions.remove(sessionId));
+ //TODO(zjffdu) whether close InterpreterGroup if there's no session left in Zeppelin Server
+ if (sessions.isEmpty() && interpreterSetting != null) {
+ LOGGER.info("Remove this InterpreterGroup {} as all the sessions are closed", id);
+ interpreterSetting.removeInterpreterGroup(id);
+ if (remoteInterpreterProcess != null) {
+ LOGGER.info("Kill RemoteIntetrpreterProcess");
+ remoteInterpreterProcess.stop();
+ remoteInterpreterProcess = null;
+ }
+ }
+ }
+
+ private void close(Collection<Interpreter> interpreters) {
+ if (interpreters == null) {
+ return;
+ }
+
+ for (Interpreter interpreter : interpreters) {
+ Scheduler scheduler = interpreter.getScheduler();
+ for (Job job : scheduler.getJobsRunning()) {
+ job.abort();
+ job.setStatus(Job.Status.ABORT);
+ LOGGER.info("Job " + job.getJobName() + " aborted ");
+ }
+ for (Job job : scheduler.getJobsWaiting()) {
+ job.abort();
+ job.setStatus(Job.Status.ABORT);
+ LOGGER.info("Job " + job.getJobName() + " aborted ");
+ }
+
+ interpreter.close();
+ //TODO(zjffdu) move the close of schedule to Interpreter
+ if (null != scheduler) {
+ SchedulerFactory.singleton().removeScheduler(scheduler.getName());
+ }
+ }
+ }
+
+ public synchronized List<Interpreter> getOrCreateSession(String user, String sessionId) {
+ if (sessions.containsKey(sessionId)) {
+ return sessions.get(sessionId);
+ } else {
+ List<Interpreter> interpreters = interpreterSetting.createInterpreters(user, sessionId);
+ for (Interpreter interpreter : interpreters) {
+ interpreter.setInterpreterGroup(this);
+ }
+ LOGGER.info("Create Session {} in InterpreterGroup {} for user {}", sessionId, id, user);
+ sessions.put(sessionId, interpreters);
+ return interpreters;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java
index 3838f63..0817595 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java
@@ -17,19 +17,17 @@
package org.apache.zeppelin.interpreter.install;
import org.apache.commons.io.FileUtils;
-import org.apache.log4j.ConsoleAppender;
-import org.apache.log4j.Logger;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.util.Util;
import org.sonatype.aether.RepositoryException;
+
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
-import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputBuffer.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputBuffer.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputBuffer.java
new file mode 100644
index 0000000..b139404
--- /dev/null
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputBuffer.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote;
+
+/**
+ * This element stores the buffered
+ * append-data of paragraph's output.
+ */
+public class AppendOutputBuffer {
+
+ private String noteId;
+ private String paragraphId;
+ private int index;
+ private String data;
+
+ public AppendOutputBuffer(String noteId, String paragraphId, int index, String data) {
+ this.noteId = noteId;
+ this.paragraphId = paragraphId;
+ this.index = index;
+ this.data = data;
+ }
+
+ public String getNoteId() {
+ return noteId;
+ }
+
+ public String getParagraphId() {
+ return paragraphId;
+ }
+
+ public int getIndex() {
+ return index;
+ }
+
+ public String getData() {
+ return data;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunner.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunner.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunner.java
new file mode 100644
index 0000000..2a88dc2
--- /dev/null
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunner.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+/**
+ * This thread sends paragraph's append-data
+ * periodically, rather than continously, with
+ * a period of BUFFER_TIME_MS. It handles append-data
+ * for all paragraphs across all notebooks.
+ */
+public class AppendOutputRunner implements Runnable {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(AppendOutputRunner.class);
+ public static final Long BUFFER_TIME_MS = new Long(100);
+ private static final Long SAFE_PROCESSING_TIME = new Long(10);
+ private static final Long SAFE_PROCESSING_STRING_SIZE = new Long(100000);
+
+ private final BlockingQueue<AppendOutputBuffer> queue = new LinkedBlockingQueue<>();
+ private final RemoteInterpreterProcessListener listener;
+
+ public AppendOutputRunner(RemoteInterpreterProcessListener listener) {
+ this.listener = listener;
+ }
+
+ @Override
+ public void run() {
+
+ Map<String, StringBuilder> stringBufferMap = new HashMap<>();
+ List<AppendOutputBuffer> list = new LinkedList<>();
+
+ /* "drainTo" method does not wait for any element
+ * to be present in the queue, and thus this loop would
+ * continuosly run (with period of BUFFER_TIME_MS). "take()" method
+ * waits for the queue to become non-empty and then removes
+ * one element from it. Rest elements from queue (if present) are
+ * removed using "drainTo" method. Thus we save on some un-necessary
+ * cpu-cycles.
+ */
+ try {
+ list.add(queue.take());
+ } catch (InterruptedException e) {
+ logger.error("Wait for OutputBuffer queue interrupted: " + e.getMessage());
+ }
+ Long processingStartTime = System.currentTimeMillis();
+ queue.drainTo(list);
+
+ for (AppendOutputBuffer buffer: list) {
+ String noteId = buffer.getNoteId();
+ String paragraphId = buffer.getParagraphId();
+ int index = buffer.getIndex();
+ String stringBufferKey = noteId + ":" + paragraphId + ":" + index;
+
+ StringBuilder builder = stringBufferMap.containsKey(stringBufferKey) ?
+ stringBufferMap.get(stringBufferKey) : new StringBuilder();
+
+ builder.append(buffer.getData());
+ stringBufferMap.put(stringBufferKey, builder);
+ }
+ Long processingTime = System.currentTimeMillis() - processingStartTime;
+
+ if (processingTime > SAFE_PROCESSING_TIME) {
+ logger.warn("Processing time for buffered append-output is high: " +
+ processingTime + " milliseconds.");
+ } else {
+ logger.debug("Processing time for append-output took "
+ + processingTime + " milliseconds");
+ }
+
+ Long sizeProcessed = new Long(0);
+ for (String stringBufferKey : stringBufferMap.keySet()) {
+ StringBuilder buffer = stringBufferMap.get(stringBufferKey);
+ sizeProcessed += buffer.length();
+ String[] keys = stringBufferKey.split(":");
+ listener.onOutputAppend(keys[0], keys[1], Integer.parseInt(keys[2]), buffer.toString());
+ }
+
+ if (sizeProcessed > SAFE_PROCESSING_STRING_SIZE) {
+ logger.warn("Processing size for buffered append-output is high: " +
+ sizeProcessed + " characters.");
+ } else {
+ logger.debug("Processing size for append-output is " +
+ sizeProcessed + " characters");
+ }
+ }
+
+ public void appendBuffer(String noteId, String paragraphId, int index, String outputToAppend) {
+ queue.offer(new AppendOutputBuffer(noteId, paragraphId, index, outputToAppend));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java
new file mode 100644
index 0000000..b2cb78f
--- /dev/null
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.pool2.BasePooledObjectFactory;
+import org.apache.commons.pool2.PooledObject;
+import org.apache.commons.pool2.impl.DefaultPooledObject;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.protocol.TProtocol;
+import org.apache.thrift.transport.TSocket;
+import org.apache.thrift.transport.TTransportException;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
+
+/**
+ *
+ */
+public class ClientFactory extends BasePooledObjectFactory<Client>{
+ private String host;
+ private int port;
+ Map<Client, TSocket> clientSocketMap = new HashMap<>();
+
+ public ClientFactory(String host, int port) {
+ this.host = host;
+ this.port = port;
+ }
+
+ @Override
+ public Client create() throws Exception {
+ TSocket transport = new TSocket(host, port);
+ try {
+ transport.open();
+ } catch (TTransportException e) {
+ throw new InterpreterException(e);
+ }
+
+ TProtocol protocol = new TBinaryProtocol(transport);
+ Client client = new RemoteInterpreterService.Client(protocol);
+
+ synchronized (clientSocketMap) {
+ clientSocketMap.put(client, transport);
+ }
+ return client;
+ }
+
+ @Override
+ public PooledObject<Client> wrap(Client client) {
+ return new DefaultPooledObject<>(client);
+ }
+
+ @Override
+ public void destroyObject(PooledObject<Client> p) {
+ synchronized (clientSocketMap) {
+ if (clientSocketMap.containsKey(p.getObject())) {
+ clientSocketMap.get(p.getObject()).close();
+ clientSocketMap.remove(p.getObject());
+ }
+ }
+ }
+
+ @Override
+ public boolean validateObject(PooledObject<Client> p) {
+ return p.getObject().getOutputProtocol().getTransport().isOpen();
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java
new file mode 100644
index 0000000..064abd5
--- /dev/null
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.zeppelin.interpreter.InterpreterContextRunner;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class InterpreterContextRunnerPool {
+ Logger logger = LoggerFactory.getLogger(InterpreterContextRunnerPool.class);
+ private Map<String, List<InterpreterContextRunner>> interpreterContextRunners;
+
+ public InterpreterContextRunnerPool() {
+ interpreterContextRunners = new HashMap<>();
+
+ }
+
+ // add runner
+ public void add(String noteId, InterpreterContextRunner runner) {
+ synchronized (interpreterContextRunners) {
+ if (!interpreterContextRunners.containsKey(noteId)) {
+ interpreterContextRunners.put(noteId, new LinkedList<InterpreterContextRunner>());
+ }
+
+ interpreterContextRunners.get(noteId).add(runner);
+ }
+ }
+
+ // replace all runners to noteId
+ public void addAll(String noteId, List<InterpreterContextRunner> runners) {
+ synchronized (interpreterContextRunners) {
+ if (!interpreterContextRunners.containsKey(noteId)) {
+ interpreterContextRunners.put(noteId, new LinkedList<InterpreterContextRunner>());
+ }
+
+ interpreterContextRunners.get(noteId).addAll(runners);
+ }
+ }
+
+ public void clear(String noteId) {
+ synchronized (interpreterContextRunners) {
+ interpreterContextRunners.remove(noteId);
+ }
+ }
+
+
+ public void run(String noteId, String paragraphId) {
+ synchronized (interpreterContextRunners) {
+ List<InterpreterContextRunner> list = interpreterContextRunners.get(noteId);
+ if (list != null) {
+ for (InterpreterContextRunner r : list) {
+ if (noteId.equals(r.getNoteId()) && paragraphId.equals(r.getParagraphId())) {
+ logger.info("run paragraph {} on note {} from InterpreterContext",
+ r.getParagraphId(), r.getNoteId());
+ r.run();
+ return;
+ }
+ }
+ }
+
+ throw new InterpreterException("Can not run paragraph " + paragraphId + " on " + noteId);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObject.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObject.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObject.java
new file mode 100644
index 0000000..62c8efd
--- /dev/null
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObject.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote;
+
+import org.apache.zeppelin.display.AngularObject;
+import org.apache.zeppelin.display.AngularObjectListener;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.ManagedInterpreterGroup;
+
+/**
+ * Proxy for AngularObject that exists in remote interpreter process
+ */
+public class RemoteAngularObject extends AngularObject {
+
+ private transient ManagedInterpreterGroup interpreterGroup;
+
+ RemoteAngularObject(String name, Object o, String noteId, String paragraphId,
+ ManagedInterpreterGroup interpreterGroup,
+ AngularObjectListener listener) {
+ super(name, o, noteId, paragraphId, listener);
+ this.interpreterGroup = interpreterGroup;
+ }
+
+ @Override
+ public void set(Object o, boolean emit) {
+ set(o, emit, true);
+ }
+
+ public void set(Object o, boolean emitWeb, boolean emitRemoteProcess) {
+ super.set(o, emitWeb);
+
+ if (emitRemoteProcess) {
+ // send updated value to remote interpreter
+ interpreterGroup.getRemoteInterpreterProcess().
+ updateRemoteAngularObject(
+ getName(), getNoteId(), getParagraphId(), o);
+ }
+ }
+}
[7/9] zeppelin git commit: [ZEPPELIN-2627] Interpreter Component
Refactoring
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
index 9403b4f..f020919 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
@@ -17,288 +17,31 @@
package org.apache.zeppelin.interpreter;
-import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
-import org.apache.commons.lang.NullArgumentException;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
-import org.apache.zeppelin.dep.DependencyResolver;
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.display.AngularObjectRegistryListener;
-import org.apache.zeppelin.helium.ApplicationEventListener;
-import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.sonatype.aether.RepositoryException;
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-import java.util.Properties;
/**
+ * //TODO(zjffdu) considering to move to InterpreterSettingManager
+ *
* Manage interpreters.
*/
-public class InterpreterFactory implements InterpreterGroupFactory {
- private static final Logger logger = LoggerFactory.getLogger(InterpreterFactory.class);
-
- private Map<String, URLClassLoader> cleanCl =
- Collections.synchronizedMap(new HashMap<String, URLClassLoader>());
-
- private ZeppelinConfiguration conf;
+public class InterpreterFactory {
+ private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterFactory.class);
private final InterpreterSettingManager interpreterSettingManager;
- private AngularObjectRegistryListener angularObjectRegistryListener;
- private final RemoteInterpreterProcessListener remoteInterpreterProcessListener;
- private final ApplicationEventListener appEventListener;
-
- private boolean shiroEnabled;
-
- private Map<String, String> env = new HashMap<>();
-
- private Interpreter devInterpreter;
-
- public InterpreterFactory(ZeppelinConfiguration conf,
- AngularObjectRegistryListener angularObjectRegistryListener,
- RemoteInterpreterProcessListener remoteInterpreterProcessListener,
- ApplicationEventListener appEventListener, DependencyResolver depResolver,
- boolean shiroEnabled, InterpreterSettingManager interpreterSettingManager)
- throws InterpreterException, IOException, RepositoryException {
- this.conf = conf;
- this.angularObjectRegistryListener = angularObjectRegistryListener;
- this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
- this.appEventListener = appEventListener;
- this.shiroEnabled = shiroEnabled;
+ public InterpreterFactory(InterpreterSettingManager interpreterSettingManager) {
this.interpreterSettingManager = interpreterSettingManager;
- //TODO(jl): Fix it not to use InterpreterGroupFactory
- interpreterSettingManager.setInterpreterGroupFactory(this);
-
- logger.info("shiroEnabled: {}", shiroEnabled);
- }
-
- /**
- * @param id interpreterGroup id. Combination of interpreterSettingId + noteId/userId/shared
- * depends on interpreter mode
- */
- @Override
- public InterpreterGroup createInterpreterGroup(String id, InterpreterOption option)
- throws InterpreterException, NullArgumentException {
-
- //When called from REST API without option we receive NPE
- if (option == null) {
- throw new NullArgumentException("option");
- }
-
- AngularObjectRegistry angularObjectRegistry;
-
- InterpreterGroup interpreterGroup = new InterpreterGroup(id);
- if (option.isRemote()) {
- angularObjectRegistry =
- new RemoteAngularObjectRegistry(id, angularObjectRegistryListener, interpreterGroup);
- } else {
- angularObjectRegistry = new AngularObjectRegistry(id, angularObjectRegistryListener);
-
- // TODO(moon) : create distributed resource pool for local interpreters and set
- }
-
- interpreterGroup.setAngularObjectRegistry(angularObjectRegistry);
- return interpreterGroup;
- }
-
- public void createInterpretersForNote(InterpreterSetting interpreterSetting, String user,
- String noteId, String interpreterSessionKey) {
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId);
- InterpreterOption option = interpreterSetting.getOption();
- Properties properties = interpreterSetting.getFlatProperties();
- // if interpreters are already there, wait until they're being removed
- synchronized (interpreterGroup) {
- long interpreterRemovalWaitStart = System.nanoTime();
- // interpreter process supposed to be terminated by RemoteInterpreterProcess.dereference()
- // in ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT msec. However, if termination of the process and
- // removal from interpreter group take too long, throw an error.
- long minTimeout = 10L * 1000 * 1000000; // 10 sec
- long interpreterRemovalWaitTimeout = Math.max(minTimeout,
- conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT) * 1000000L * 2);
- while (interpreterGroup.containsKey(interpreterSessionKey)) {
- if (System.nanoTime() - interpreterRemovalWaitStart > interpreterRemovalWaitTimeout) {
- throw new InterpreterException("Can not create interpreter");
- }
- try {
- interpreterGroup.wait(1000);
- } catch (InterruptedException e) {
- logger.debug(e.getMessage(), e);
- }
- }
- }
-
- logger.info("Create interpreter instance {} for note {}", interpreterSetting.getName(), noteId);
-
- List<InterpreterInfo> interpreterInfos = interpreterSetting.getInterpreterInfos();
- String path = interpreterSetting.getPath();
- InterpreterRunner runner = interpreterSetting.getInterpreterRunner();
- Interpreter interpreter;
- for (InterpreterInfo info : interpreterInfos) {
- if (option.isRemote()) {
- if (option.isExistingProcess()) {
- interpreter =
- connectToRemoteRepl(interpreterSessionKey, info.getClassName(), option.getHost(),
- option.getPort(), properties, interpreterSetting.getId(), user,
- option.isUserImpersonate);
- } else {
- interpreter = createRemoteRepl(path, interpreterSessionKey, info.getClassName(),
- properties, interpreterSetting.getId(), user, option.isUserImpersonate(), runner);
- }
- } else {
- interpreter = createRepl(interpreterSetting.getPath(), info.getClassName(), properties);
- }
-
- synchronized (interpreterGroup) {
- List<Interpreter> interpreters = interpreterGroup.get(interpreterSessionKey);
- if (null == interpreters) {
- interpreters = new ArrayList<>();
- interpreterGroup.put(interpreterSessionKey, interpreters);
- }
- if (info.isDefaultInterpreter()) {
- interpreters.add(0, interpreter);
- } else {
- interpreters.add(interpreter);
- }
- }
- logger.info("Interpreter {} {} created", interpreter.getClassName(), interpreter.hashCode());
- interpreter.setInterpreterGroup(interpreterGroup);
- }
- }
-
- private Interpreter createRepl(String dirName, String className, Properties property)
- throws InterpreterException {
- logger.info("Create repl {} from {}", className, dirName);
-
- ClassLoader oldcl = Thread.currentThread().getContextClassLoader();
- try {
-
- URLClassLoader ccl = cleanCl.get(dirName);
- if (ccl == null) {
- // classloader fallback
- ccl = URLClassLoader.newInstance(new URL[]{}, oldcl);
- }
-
- boolean separateCL = true;
- try { // check if server's classloader has driver already.
- Class cls = this.getClass().forName(className);
- if (cls != null) {
- separateCL = false;
- }
- } catch (Exception e) {
- logger.error("exception checking server classloader driver", e);
- }
-
- URLClassLoader cl;
-
- if (separateCL == true) {
- cl = URLClassLoader.newInstance(new URL[]{}, ccl);
- } else {
- cl = ccl;
- }
- Thread.currentThread().setContextClassLoader(cl);
-
- Class<Interpreter> replClass = (Class<Interpreter>) cl.loadClass(className);
- Constructor<Interpreter> constructor =
- replClass.getConstructor(new Class[]{Properties.class});
- Interpreter repl = constructor.newInstance(property);
- repl.setClassloaderUrls(ccl.getURLs());
- LazyOpenInterpreter intp = new LazyOpenInterpreter(new ClassloaderInterpreter(repl, cl));
- return intp;
- } catch (SecurityException e) {
- throw new InterpreterException(e);
- } catch (NoSuchMethodException e) {
- throw new InterpreterException(e);
- } catch (IllegalArgumentException e) {
- throw new InterpreterException(e);
- } catch (InstantiationException e) {
- throw new InterpreterException(e);
- } catch (IllegalAccessException e) {
- throw new InterpreterException(e);
- } catch (InvocationTargetException e) {
- throw new InterpreterException(e);
- } catch (ClassNotFoundException e) {
- throw new InterpreterException(e);
- } finally {
- Thread.currentThread().setContextClassLoader(oldcl);
- }
- }
-
- private Interpreter connectToRemoteRepl(String interpreterSessionKey, String className,
- String host, int port, Properties property, String interpreterSettingId, String userName,
- Boolean isUserImpersonate) {
- int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
- int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE);
- String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + interpreterSettingId;
- LazyOpenInterpreter intp = new LazyOpenInterpreter(
- new RemoteInterpreter(property, interpreterSessionKey, className, host, port, localRepoPath,
- connectTimeout, maxPoolSize, remoteInterpreterProcessListener, appEventListener,
- userName, isUserImpersonate, conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT)));
- return intp;
- }
-
- Interpreter createRemoteRepl(String interpreterPath, String interpreterSessionKey,
- String className, Properties property, String interpreterSettingId,
- String userName, Boolean isUserImpersonate, InterpreterRunner interpreterRunner) {
- int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
- String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + interpreterSettingId;
- int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE);
- String interpreterRunnerPath;
- String interpreterGroupName = interpreterSettingManager.get(interpreterSettingId).getName();
- if (null != interpreterRunner) {
- interpreterRunnerPath = interpreterRunner.getPath();
- Path p = Paths.get(interpreterRunnerPath);
- if (!p.isAbsolute()) {
- interpreterRunnerPath = Joiner.on(File.separator)
- .join(interpreterPath, interpreterRunnerPath);
- }
- } else {
- interpreterRunnerPath = conf.getInterpreterRemoteRunnerPath();
- }
-
- RemoteInterpreter remoteInterpreter =
- new RemoteInterpreter(property, interpreterSessionKey, className,
- interpreterRunnerPath, interpreterPath, localRepoPath, connectTimeout, maxPoolSize,
- remoteInterpreterProcessListener, appEventListener, userName, isUserImpersonate,
- conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT), interpreterGroupName);
- remoteInterpreter.addEnv(env);
-
- return new LazyOpenInterpreter(remoteInterpreter);
- }
-
- private List<Interpreter> createOrGetInterpreterList(String user, String noteId,
- InterpreterSetting setting) {
- InterpreterGroup interpreterGroup = setting.getInterpreterGroup(user, noteId);
- synchronized (interpreterGroup) {
- String interpreterSessionKey =
- interpreterSettingManager.getInterpreterSessionKey(user, noteId, setting);
- if (!interpreterGroup.containsKey(interpreterSessionKey)) {
- createInterpretersForNote(setting, user, noteId, interpreterSessionKey);
- }
- return interpreterGroup.get(interpreterSessionKey);
- }
}
private InterpreterSetting getInterpreterSettingByGroup(List<InterpreterSetting> settings,
String group) {
- Preconditions.checkNotNull(group, "group should be not null");
+ Preconditions.checkNotNull(group, "group should be not null");
for (InterpreterSetting setting : settings) {
if (group.equals(setting.getName())) {
return setting;
@@ -307,80 +50,41 @@ public class InterpreterFactory implements InterpreterGroupFactory {
return null;
}
- private String getInterpreterClassFromInterpreterSetting(InterpreterSetting setting,
- String name) {
- Preconditions.checkNotNull(name, "name should be not null");
-
- for (InterpreterInfo info : setting.getInterpreterInfos()) {
- String infoName = info.getName();
- if (null != info.getName() && name.equals(infoName)) {
- return info.getClassName();
- }
- }
- return null;
- }
-
- private Interpreter getInterpreter(String user, String noteId, InterpreterSetting setting,
- String name) {
- Preconditions.checkNotNull(noteId, "noteId should be not null");
- Preconditions.checkNotNull(setting, "setting should be not null");
- Preconditions.checkNotNull(name, "name should be not null");
-
- String className;
- if (null != (className = getInterpreterClassFromInterpreterSetting(setting, name))) {
- List<Interpreter> interpreterGroup = createOrGetInterpreterList(user, noteId, setting);
- for (Interpreter interpreter : interpreterGroup) {
- if (className.equals(interpreter.getClassName())) {
- return interpreter;
- }
- }
- }
- return null;
- }
-
public Interpreter getInterpreter(String user, String noteId, String replName) {
List<InterpreterSetting> settings = interpreterSettingManager.getInterpreterSettings(noteId);
InterpreterSetting setting;
Interpreter interpreter;
if (settings == null || settings.size() == 0) {
+ LOGGER.error("No interpreter is binded to this note: " + noteId);
return null;
}
- if (replName == null || replName.trim().length() == 0) {
- // get default settings (first available)
- // TODO(jl): Fix it in case of returning null
- InterpreterSetting defaultSettings = interpreterSettingManager
- .getDefaultInterpreterSetting(settings);
- return createOrGetInterpreterList(user, noteId, defaultSettings).get(0);
+ if (StringUtils.isBlank(replName)) {
+ // Get the default interpreter of the first interpreter binding
+ InterpreterSetting defaultSetting = settings.get(0);
+ return defaultSetting.getDefaultInterpreter(user, noteId);
}
String[] replNameSplit = replName.split("\\.");
if (replNameSplit.length == 2) {
- String group = null;
- String name = null;
- group = replNameSplit[0];
- name = replNameSplit[1];
-
+ String group = replNameSplit[0];
+ String name = replNameSplit[1];
setting = getInterpreterSettingByGroup(settings, group);
-
if (null != setting) {
- interpreter = getInterpreter(user, noteId, setting, name);
-
+ interpreter = setting.getInterpreter(user, noteId, name);
if (null != interpreter) {
return interpreter;
}
}
-
throw new InterpreterException(replName + " interpreter not found");
} else {
// first assume replName is 'name' of interpreter. ('groupName' is ommitted)
// search 'name' from first (default) interpreter group
// TODO(jl): Handle with noteId to support defaultInterpreter per note.
- setting = interpreterSettingManager.getDefaultInterpreterSetting(settings);
-
- interpreter = getInterpreter(user, noteId, setting, replName);
+ setting = settings.get(0);
+ interpreter = setting.getInterpreter(user, noteId, replName);
if (null != interpreter) {
return interpreter;
@@ -391,33 +95,17 @@ public class InterpreterFactory implements InterpreterGroupFactory {
setting = getInterpreterSettingByGroup(settings, replName);
if (null != setting) {
- List<Interpreter> interpreters = createOrGetInterpreterList(user, noteId, setting);
- if (null != interpreters) {
- return interpreters.get(0);
- }
+ return setting.getDefaultInterpreter(user, noteId);
}
// Support the legacy way to use it
for (InterpreterSetting s : settings) {
if (s.getGroup().equals(replName)) {
- List<Interpreter> interpreters = createOrGetInterpreterList(user, noteId, s);
- if (null != interpreters) {
- return interpreters.get(0);
- }
+ return setting.getDefaultInterpreter(user, noteId);
}
}
}
-
+ //TODO(zjffdu) throw InterpreterException instead of return null
return null;
}
-
- public Map<String, String> getEnv() {
- return env;
- }
-
- public void setEnv(Map<String, String> env) {
- this.env = env;
- }
-
-
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroupFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroupFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroupFactory.java
deleted file mode 100644
index 3b9be40..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroupFactory.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.zeppelin.interpreter;
-
-import org.apache.commons.lang.NullArgumentException;
-
-/**
- * Created InterpreterGroup
- */
-public interface InterpreterGroupFactory {
- InterpreterGroup createInterpreterGroup(String interpreterGroupId, InterpreterOption option);
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java
index ca688dc..d7593d5 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java
@@ -19,22 +19,78 @@ package org.apache.zeppelin.interpreter;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.internal.StringMap;
+import org.apache.commons.io.IOUtils;
import org.apache.zeppelin.common.JsonSerializable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.sonatype.aether.repository.RemoteRepository;
-import java.util.List;
-import java.util.Map;
+import java.io.BufferedReader;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.PosixFilePermission;
+import java.util.*;
+
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
/**
*
*/
public class InterpreterInfoSaving implements JsonSerializable {
- private static final Gson gson = new GsonBuilder().setPrettyPrinting().create();
+ private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterInfoSaving.class);
+ private static final Gson gson = new GsonBuilder().setPrettyPrinting().create();
+
+ public Map<String, InterpreterSetting> interpreterSettings = new HashMap<>();
+ public Map<String, List<String>> interpreterBindings = new HashMap<>();
+ public List<RemoteRepository> interpreterRepositories = new ArrayList<>();
+
+ public static InterpreterInfoSaving loadFromFile(Path file) throws IOException {
+ LOGGER.info("Load interpreter setting from file: " + file);
+ InterpreterInfoSaving infoSaving = null;
+ try (BufferedReader json = Files.newBufferedReader(file, StandardCharsets.UTF_8)) {
+ JsonParser jsonParser = new JsonParser();
+ JsonObject jsonObject = jsonParser.parse(json).getAsJsonObject();
+ infoSaving = InterpreterInfoSaving.fromJson(jsonObject.toString());
- public Map<String, InterpreterSetting> interpreterSettings;
- public Map<String, List<String>> interpreterBindings;
- public List<RemoteRepository> interpreterRepositories;
+ if (infoSaving != null && infoSaving.interpreterSettings != null) {
+ for (InterpreterSetting interpreterSetting : infoSaving.interpreterSettings.values()) {
+ // Always use separate interpreter process
+ // While we decided to turn this feature on always (without providing
+ // enable/disable option on GUI).
+ // previously created setting should turn this feature on here.
+ interpreterSetting.getOption().setRemote(true);
+ interpreterSetting.convertPermissionsFromUsersToOwners(
+ jsonObject.getAsJsonObject("interpreterSettings")
+ .getAsJsonObject(interpreterSetting.getId()));
+ }
+ }
+ }
+ return infoSaving == null ? new InterpreterInfoSaving() : infoSaving;
+ }
+
+ public void saveToFile(Path file) throws IOException {
+ if (!Files.exists(file)) {
+ Files.createFile(file);
+ try {
+ Set<PosixFilePermission> permissions = EnumSet.of(OWNER_READ, OWNER_WRITE);
+ Files.setPosixFilePermissions(file, permissions);
+ } catch (UnsupportedOperationException e) {
+ // File system does not support Posix file permissions (likely windows) - continue anyway.
+ LOGGER.warn("unable to setPosixFilePermissions on '{}'.", file);
+ };
+ }
+ LOGGER.info("Save Interpreter Settings to " + file);
+ IOUtils.write(this.toJson(), new FileOutputStream(file.toFile()));
+ }
public String toJson() {
return gson.toJson(this);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
index 752b4e2..9f4cfd4 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
@@ -17,8 +17,39 @@
package org.apache.zeppelin.interpreter;
-import java.util.Arrays;
-import java.util.Collection;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.annotations.SerializedName;
+import com.google.gson.internal.StringMap;
+import org.apache.commons.io.FileUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.dep.Dependency;
+import org.apache.zeppelin.dep.DependencyResolver;
+import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.display.AngularObjectRegistryListener;
+import org.apache.zeppelin.helium.ApplicationEventListener;
+import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterRunningProcess;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@@ -26,104 +57,253 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.zeppelin.dep.Dependency;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.annotations.SerializedName;
-import com.google.gson.internal.StringMap;
-
-import static org.apache.zeppelin.notebook.utility.IdHashes.generateId;
+import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE;
+import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT;
+import static org.apache.zeppelin.util.IdHashes.generateId;
/**
- * Interpreter settings
+ * Represent one InterpreterSetting in the interpreter setting page
*/
public class InterpreterSetting {
- private static final Logger logger = LoggerFactory.getLogger(InterpreterSetting.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterSetting.class);
private static final String SHARED_PROCESS = "shared_process";
+ private static final String SHARED_SESSION = "shared_session";
+ private static final Map<String, Object> DEFAULT_EDITOR = ImmutableMap.of(
+ "language", (Object) "text",
+ "editOnDblClick", false);
+
private String id;
private String name;
- // always be null in case of InterpreterSettingRef
+ // the original interpreter setting template name where it is created from
private String group;
- private transient Map<String, String> infos;
-
- // Map of the note and paragraphs which has runtime infos generated by this interpreter setting.
- // This map is used to clear the infos in paragraph when the interpretersetting is restarted
- private transient Map<String, Set<String>> runtimeInfosToBeCleared;
+ //TODO(zjffdu) make the interpreter.json consistent with interpreter-setting.json
/**
- * properties can be either Map<String, DefaultInterpreterProperty> or
- * Map<String, InterpreterProperty>
+ * properties can be either Properties or Map<String, InterpreterProperty>
* properties should be:
- * - Map<String, InterpreterProperty> when Interpreter instances are saved to
- * `conf/interpreter.json` file
- * - Map<String, DefaultInterpreterProperty> when Interpreters are registered
+ * - Properties when Interpreter instances are saved to `conf/interpreter.json` file
+ * - Map<String, InterpreterProperty> when Interpreters are registered
* : this is needed after https://github.com/apache/zeppelin/pull/1145
* which changed the way of getting default interpreter setting AKA interpreterSettingsRef
+ * Note(mina): In order to simplify the implementation, I chose to change properties
+ * from Properties to Object instead of creating new classes.
*/
- private Object properties;
+ private Object properties = new Properties();
+
private Status status;
private String errorReason;
@SerializedName("interpreterGroup")
private List<InterpreterInfo> interpreterInfos;
- private final transient Map<String, InterpreterGroup> interpreterGroupRef = new HashMap<>();
- private List<Dependency> dependencies = new LinkedList<>();
- private InterpreterOption option;
- private transient String path;
+
+ private List<Dependency> dependencies = new ArrayList<>();
+ private InterpreterOption option = new InterpreterOption(true);
@SerializedName("runner")
private InterpreterRunner interpreterRunner;
- @Deprecated
- private transient InterpreterGroupFactory interpreterGroupFactory;
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ private transient InterpreterSettingManager interpreterSettingManager;
+ private transient String interpreterDir;
+ private final transient Map<String, ManagedInterpreterGroup> interpreterGroups =
+ new ConcurrentHashMap<>();
private final transient ReentrantReadWriteLock.ReadLock interpreterGroupReadLock;
private final transient ReentrantReadWriteLock.WriteLock interpreterGroupWriteLock;
+ private transient AngularObjectRegistryListener angularObjectRegistryListener;
+ private transient RemoteInterpreterProcessListener remoteInterpreterProcessListener;
+ private transient ApplicationEventListener appEventListener;
+ private transient DependencyResolver dependencyResolver;
+
+ private transient Map<String, String> infos;
+
+ // Map of the note and paragraphs which has runtime infos generated by this interpreter setting.
+ // This map is used to clear the infos in paragraph when the interpretersetting is restarted
+ private transient Map<String, Set<String>> runtimeInfosToBeCleared;
+
+ private transient ZeppelinConfiguration conf = new ZeppelinConfiguration();
+
+ private transient Map<String, URLClassLoader> cleanCl =
+ Collections.synchronizedMap(new HashMap<String, URLClassLoader>());
+ ///////////////////////////////////////////////////////////////////////////////////////////
+
+
+ /**
+ * Builder class for InterpreterSetting
+ */
+ public static class Builder {
+ private InterpreterSetting interpreterSetting;
+
+ public Builder() {
+ this.interpreterSetting = new InterpreterSetting();
+ }
+
+ public Builder setId(String id) {
+ interpreterSetting.id = id;
+ return this;
+ }
+
+ public Builder setName(String name) {
+ interpreterSetting.name = name;
+ return this;
+ }
+
+ public Builder setGroup(String group) {
+ interpreterSetting.group = group;
+ return this;
+ }
+
+ public Builder setInterpreterInfos(List<InterpreterInfo> interpreterInfos) {
+ interpreterSetting.interpreterInfos = interpreterInfos;
+ return this;
+ }
+
+ public Builder setProperties(Object properties) {
+ interpreterSetting.properties = properties;
+ return this;
+ }
+
+ public Builder setOption(InterpreterOption option) {
+ interpreterSetting.option = option;
+ return this;
+ }
+
+ public Builder setInterpreterDir(String interpreterDir) {
+ interpreterSetting.interpreterDir = interpreterDir;
+ return this;
+ }
+
+ public Builder setRunner(InterpreterRunner runner) {
+ interpreterSetting.interpreterRunner = runner;
+ return this;
+ }
+
+ public Builder setDependencies(List<Dependency> dependencies) {
+ interpreterSetting.dependencies = dependencies;
+ return this;
+ }
+
+ public Builder setConf(ZeppelinConfiguration conf) {
+ interpreterSetting.conf = conf;
+ return this;
+ }
+
+ public Builder setDependencyResolver(DependencyResolver dependencyResolver) {
+ interpreterSetting.dependencyResolver = dependencyResolver;
+ return this;
+ }
+
+// public Builder setInterpreterRunner(InterpreterRunner runner) {
+// interpreterSetting.interpreterRunner = runner;
+// return this;
+// }
+
+ public Builder setIntepreterSettingManager(
+ InterpreterSettingManager interpreterSettingManager) {
+ interpreterSetting.interpreterSettingManager = interpreterSettingManager;
+ return this;
+ }
+
+ public Builder setRemoteInterpreterProcessListener(RemoteInterpreterProcessListener
+ remoteInterpreterProcessListener) {
+ interpreterSetting.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
+ return this;
+ }
+
+ public Builder setAngularObjectRegistryListener(
+ AngularObjectRegistryListener angularObjectRegistryListener) {
+ interpreterSetting.angularObjectRegistryListener = angularObjectRegistryListener;
+ return this;
+ }
+
+ public Builder setApplicationEventListener(ApplicationEventListener applicationEventListener) {
+ interpreterSetting.appEventListener = applicationEventListener;
+ return this;
+ }
+
+ public InterpreterSetting create() {
+ // post processing
+ interpreterSetting.postProcessing();
+ return interpreterSetting;
+ }
+ }
+
public InterpreterSetting() {
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ this.id = generateId();
interpreterGroupReadLock = lock.readLock();
interpreterGroupWriteLock = lock.writeLock();
}
- public InterpreterSetting(String id, String name, String group,
- List<InterpreterInfo> interpreterInfos, Object properties, List<Dependency> dependencies,
- InterpreterOption option, String path, InterpreterRunner runner) {
- this();
- this.id = id;
- this.name = name;
- this.group = group;
- this.interpreterInfos = interpreterInfos;
- this.properties = properties;
- this.dependencies = dependencies;
- this.option = option;
- this.path = path;
+ void postProcessing() {
this.status = Status.READY;
- this.interpreterRunner = runner;
- }
-
- public InterpreterSetting(String name, String group, List<InterpreterInfo> interpreterInfos,
- Object properties, List<Dependency> dependencies, InterpreterOption option, String path,
- InterpreterRunner runner) {
- this(generateId(), name, group, interpreterInfos, properties, dependencies, option, path,
- runner);
}
/**
- * Create interpreter from interpreterSettingRef
+ * Create interpreter from InterpreterSettingTemplate
*
- * @param o interpreterSetting from interpreterSettingRef
+ * @param o interpreterSetting from InterpreterSettingTemplate
*/
public InterpreterSetting(InterpreterSetting o) {
- this(generateId(), o.getName(), o.getGroup(), o.getInterpreterInfos(), o.getProperties(),
- o.getDependencies(), o.getOption(), o.getPath(), o.getInterpreterRunner());
+ this();
+ this.id = generateId();
+ this.name = o.name;
+ this.group = o.group;
+ this.properties = convertInterpreterProperties(
+ (Map<String, DefaultInterpreterProperty>) o.getProperties());
+ this.interpreterInfos = new ArrayList<>(o.getInterpreterInfos());
+ this.option = InterpreterOption.fromInterpreterOption(o.getOption());
+ this.dependencies = new ArrayList<>(o.getDependencies());
+ this.interpreterDir = o.getInterpreterDir();
+ this.interpreterRunner = o.getInterpreterRunner();
+ this.conf = o.getConf();
+ }
+
+ public AngularObjectRegistryListener getAngularObjectRegistryListener() {
+ return angularObjectRegistryListener;
+ }
+
+ public RemoteInterpreterProcessListener getRemoteInterpreterProcessListener() {
+ return remoteInterpreterProcessListener;
+ }
+
+ public ApplicationEventListener getAppEventListener() {
+ return appEventListener;
+ }
+
+ public DependencyResolver getDependencyResolver() {
+ return dependencyResolver;
+ }
+
+ public InterpreterSettingManager getInterpreterSettingManager() {
+ return interpreterSettingManager;
+ }
+
+ public void setAngularObjectRegistryListener(AngularObjectRegistryListener
+ angularObjectRegistryListener) {
+ this.angularObjectRegistryListener = angularObjectRegistryListener;
+ }
+
+ public void setAppEventListener(ApplicationEventListener appEventListener) {
+ this.appEventListener = appEventListener;
+ }
+
+ public void setRemoteInterpreterProcessListener(RemoteInterpreterProcessListener
+ remoteInterpreterProcessListener) {
+ this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
+ }
+
+ public void setDependencyResolver(DependencyResolver dependencyResolver) {
+ this.dependencyResolver = dependencyResolver;
+ }
+
+ public void setInterpreterSettingManager(InterpreterSettingManager interpreterSettingManager) {
+ this.interpreterSettingManager = interpreterSettingManager;
}
public String getId() {
@@ -138,10 +318,9 @@ public class InterpreterSetting {
return group;
}
- private String getInterpreterProcessKey(String user, String noteId) {
- InterpreterOption option = getOption();
+ private String getInterpreterGroupId(String user, String noteId) {
String key;
- if (getOption().isExistingProcess) {
+ if (option.isExistingProcess) {
key = Constants.EXISTING_PROCESS;
} else if (getOption().isProcess()) {
key = (option.perUserIsolated() ? user : "") + ":" + (option.perNoteIsolated() ? noteId : "");
@@ -149,40 +328,11 @@ public class InterpreterSetting {
key = SHARED_PROCESS;
}
- //logger.debug("getInterpreterProcessKey: {} for InterpreterSetting Id: {}, Name: {}",
- // key, getId(), getName());
- return key;
+ //TODO(zjffdu) we encode interpreter setting id into groupId, this is not a good design
+ return id + ":" + key;
}
- private boolean isEqualInterpreterKeyProcessKey(String refKey, String processKey) {
- InterpreterOption option = getOption();
- int validCount = 0;
- if (getOption().isProcess()
- && !(option.perUserIsolated() == true && option.perNoteIsolated() == true)) {
-
- List<String> processList = Arrays.asList(processKey.split(":"));
- List<String> refList = Arrays.asList(refKey.split(":"));
-
- if (refList.size() <= 1 || processList.size() <= 1) {
- return refKey.equals(processKey);
- }
-
- if (processList.get(0).equals("") || processList.get(0).equals(refList.get(0))) {
- validCount = validCount + 1;
- }
-
- if (processList.get(1).equals("") || processList.get(1).equals(refList.get(1))) {
- validCount = validCount + 1;
- }
-
- return (validCount >= 2);
- } else {
- return refKey.equals(processKey);
- }
- }
-
- String getInterpreterSessionKey(String user, String noteId) {
- InterpreterOption option = getOption();
+ private String getInterpreterSessionId(String user, String noteId) {
String key;
if (option.isExistingProcess()) {
key = Constants.EXISTING_PROCESS;
@@ -193,120 +343,153 @@ public class InterpreterSetting {
} else if (option.perNoteScoped()) {
key = noteId;
} else {
- key = "shared_session";
+ key = SHARED_SESSION;
}
- logger.debug("Interpreter session key: {}, for note: {}, user: {}, InterpreterSetting Name: " +
- "{}", key, noteId, user, getName());
return key;
}
- public InterpreterGroup getInterpreterGroup(String user, String noteId) {
- String key = getInterpreterProcessKey(user, noteId);
- if (!interpreterGroupRef.containsKey(key)) {
- String interpreterGroupId = getId() + ":" + key;
- InterpreterGroup intpGroup =
- interpreterGroupFactory.createInterpreterGroup(interpreterGroupId, getOption());
-
+ public ManagedInterpreterGroup getOrCreateInterpreterGroup(String user, String noteId) {
+ String groupId = getInterpreterGroupId(user, noteId);
+ try {
interpreterGroupWriteLock.lock();
- logger.debug("create interpreter group with groupId:" + interpreterGroupId);
- interpreterGroupRef.put(key, intpGroup);
- interpreterGroupWriteLock.unlock();
+ if (!interpreterGroups.containsKey(groupId)) {
+ LOGGER.info("Create InterpreterGroup with groupId {} for user {} and note {}",
+ groupId, user, noteId);
+ ManagedInterpreterGroup intpGroup = createInterpreterGroup(groupId);
+ interpreterGroups.put(groupId, intpGroup);
+ }
+ return interpreterGroups.get(groupId);
+ } finally {
+ interpreterGroupWriteLock.unlock();;
}
+ }
+
+ void removeInterpreterGroup(String groupId) {
+ this.interpreterGroups.remove(groupId);
+ }
+
+ ManagedInterpreterGroup getInterpreterGroup(String user, String noteId) {
+ String groupId = getInterpreterGroupId(user, noteId);
try {
interpreterGroupReadLock.lock();
- return interpreterGroupRef.get(key);
+ return interpreterGroups.get(groupId);
} finally {
- interpreterGroupReadLock.unlock();
+ interpreterGroupReadLock.unlock();;
}
}
- public Collection<InterpreterGroup> getAllInterpreterGroups() {
+ ManagedInterpreterGroup getInterpreterGroup(String groupId) {
+ return interpreterGroups.get(groupId);
+ }
+
+ @VisibleForTesting
+ public ArrayList<ManagedInterpreterGroup> getAllInterpreterGroups() {
try {
interpreterGroupReadLock.lock();
- return new LinkedList<>(interpreterGroupRef.values());
+ return new ArrayList(interpreterGroups.values());
} finally {
interpreterGroupReadLock.unlock();
}
}
- void closeAndRemoveInterpreterGroup(String noteId, String user) {
- if (user.equals("anonymous")) {
- user = "";
- }
- String processKey = getInterpreterProcessKey(user, noteId);
- String sessionKey = getInterpreterSessionKey(user, noteId);
- List<InterpreterGroup> groupToRemove = new LinkedList<>();
- InterpreterGroup groupItem;
- for (String intpKey : new HashSet<>(interpreterGroupRef.keySet())) {
- if (isEqualInterpreterKeyProcessKey(intpKey, processKey)) {
- interpreterGroupWriteLock.lock();
- // TODO(jl): interpreterGroup has two or more sessionKeys inside it. thus we should not
- // remove interpreterGroup if it has two or more values.
- groupItem = interpreterGroupRef.get(intpKey);
- interpreterGroupWriteLock.unlock();
- groupToRemove.add(groupItem);
- }
- for (InterpreterGroup groupToClose : groupToRemove) {
- // TODO(jl): Fix the logic removing session. Now, it's handled into groupToClose.clsose()
- groupToClose.close(interpreterGroupRef, intpKey, sessionKey);
+ Map<String, Object> getEditorFromSettingByClassName(String className) {
+ for (InterpreterInfo intpInfo : interpreterInfos) {
+ if (className.equals(intpInfo.getClassName())) {
+ if (intpInfo.getEditor() == null) {
+ break;
+ }
+ return intpInfo.getEditor();
}
- groupToRemove.clear();
}
+ return DEFAULT_EDITOR;
+ }
- //Remove session because all interpreters in this session are closed
- //TODO(jl): Change all code to handle interpreter one by one or all at once
-
+ void closeInterpreters(String user, String noteId) {
+ ManagedInterpreterGroup interpreterGroup = getInterpreterGroup(user, noteId);
+ if (interpreterGroup != null) {
+ String sessionId = getInterpreterSessionId(user, noteId);
+ interpreterGroup.close(sessionId);
+ }
}
- void closeAndRemoveAllInterpreterGroups() {
- for (String processKey : new HashSet<>(interpreterGroupRef.keySet())) {
- InterpreterGroup interpreterGroup = interpreterGroupRef.get(processKey);
- for (String sessionKey : new HashSet<>(interpreterGroup.keySet())) {
- interpreterGroup.close(interpreterGroupRef, processKey, sessionKey);
- }
+ public void close() {
+ LOGGER.info("Close InterpreterSetting: " + name);
+ for (ManagedInterpreterGroup intpGroup : interpreterGroups.values()) {
+ intpGroup.close();
}
+ interpreterGroups.clear();
+ this.runtimeInfosToBeCleared = null;
+ this.infos = null;
}
- void shutdownAndRemoveAllInterpreterGroups() {
- for (InterpreterGroup interpreterGroup : interpreterGroupRef.values()) {
- interpreterGroup.shutdown();
+ public void setProperties(Object object) {
+ if (object instanceof StringMap) {
+ StringMap<String> map = (StringMap) properties;
+ Properties newProperties = new Properties();
+ for (String key : map.keySet()) {
+ newProperties.put(key, map.get(key));
+ }
+ this.properties = newProperties;
+ } else {
+ this.properties = object;
}
}
+
public Object getProperties() {
return properties;
}
- public Properties getFlatProperties() {
- Properties p = new Properties();
- if (properties != null) {
- Map<String, InterpreterProperty> propertyMap = (Map<String, InterpreterProperty>) properties;
- for (String key : propertyMap.keySet()) {
- InterpreterProperty tmp = propertyMap.get(key);
- p.put(tmp.getName() != null ? tmp.getName() : key,
- tmp.getValue() != null ? tmp.getValue().toString() : null);
- }
+ @VisibleForTesting
+ public void setProperty(String name, String value) {
+ ((Map<String, InterpreterProperty>) properties).put(name, new InterpreterProperty(name, value));
+ }
+
+ // This method is supposed to be only called by InterpreterSetting
+ // but not InterpreterSetting Template
+ public Properties getJavaProperties() {
+ Properties jProperties = new Properties();
+ Map<String, InterpreterProperty> iProperties = (Map<String, InterpreterProperty>) properties;
+ for (Map.Entry<String, InterpreterProperty> entry : iProperties.entrySet()) {
+ jProperties.setProperty(entry.getKey(), entry.getValue().getValue().toString());
+ }
+
+ if (!jProperties.containsKey("zeppelin.interpreter.output.limit")) {
+ jProperties.setProperty("zeppelin.interpreter.output.limit",
+ conf.getInt(ZEPPELIN_INTERPRETER_OUTPUT_LIMIT) + "");
+ }
+
+ if (!jProperties.containsKey("zeppelin.interpreter.max.poolsize")) {
+ jProperties.setProperty("zeppelin.interpreter.max.poolsize",
+ conf.getInt(ZEPPELIN_INTERPRETER_MAX_POOL_SIZE) + "");
}
- return p;
+
+ String interpreterLocalRepoPath = conf.getInterpreterLocalRepoPath();
+ //TODO(zjffdu) change it to interpreterDir/{interpreter_name}
+ jProperties.setProperty("zeppelin.interpreter.localRepo",
+ interpreterLocalRepoPath + "/" + id);
+ return jProperties;
+ }
+
+ public ZeppelinConfiguration getConf() {
+ return conf;
+ }
+
+ public void setConf(ZeppelinConfiguration conf) {
+ this.conf = conf;
}
public List<Dependency> getDependencies() {
- if (dependencies == null) {
- return new LinkedList<>();
- }
return dependencies;
}
public void setDependencies(List<Dependency> dependencies) {
this.dependencies = dependencies;
+ loadInterpreterDependencies();
}
public InterpreterOption getOption() {
- if (option == null) {
- option = new InterpreterOption();
- }
-
return option;
}
@@ -314,35 +497,32 @@ public class InterpreterSetting {
this.option = option;
}
- public String getPath() {
- return path;
+ public String getInterpreterDir() {
+ return interpreterDir;
}
- public void setPath(String path) {
- this.path = path;
+ public void setInterpreterDir(String interpreterDir) {
+ this.interpreterDir = interpreterDir;
}
public List<InterpreterInfo> getInterpreterInfos() {
return interpreterInfos;
}
- void setInterpreterGroupFactory(InterpreterGroupFactory interpreterGroupFactory) {
- this.interpreterGroupFactory = interpreterGroupFactory;
- }
-
void appendDependencies(List<Dependency> dependencies) {
for (Dependency dependency : dependencies) {
if (!this.dependencies.contains(dependency)) {
this.dependencies.add(dependency);
}
}
+ loadInterpreterDependencies();
}
void setInterpreterOption(InterpreterOption interpreterOption) {
this.option = interpreterOption;
}
- public void setProperties(Map<String, InterpreterProperty> p) {
+ public void setProperties(Properties p) {
this.properties = p;
}
@@ -379,6 +559,10 @@ public class InterpreterSetting {
this.errorReason = errorReason;
}
+ public void setInterpreterInfos(List<InterpreterInfo> interpreterInfos) {
+ this.interpreterInfos = interpreterInfos;
+ }
+
public void setInfos(Map<String, String> infos) {
this.infos = infos;
}
@@ -415,7 +599,236 @@ public class InterpreterSetting {
runtimeInfosToBeCleared = null;
}
- // For backward compatibility of interpreter.json format after ZEPPELIN-2654
+
+ //////////////////////////// IMPORTANT ////////////////////////////////////////////////
+ ///////////////////////////////////////////////////////////////////////////////////////
+ ///////////////////////////////////////////////////////////////////////////////////////
+ // This is the only place to create interpreters. For now we always create multiple interpreter
+ // together (one session). We don't support to create single interpreter yet.
+ List<Interpreter> createInterpreters(String user, String sessionId) {
+ List<Interpreter> interpreters = new ArrayList<>();
+ List<InterpreterInfo> interpreterInfos = getInterpreterInfos();
+ for (InterpreterInfo info : interpreterInfos) {
+ Interpreter interpreter = null;
+ if (option.isRemote()) {
+ interpreter = new RemoteInterpreter(getJavaProperties(), sessionId,
+ info.getClassName(), user);
+ } else {
+ interpreter = createLocalInterpreter(info.getClassName());
+ }
+
+ if (info.isDefaultInterpreter()) {
+ interpreters.add(0, interpreter);
+ } else {
+ interpreters.add(interpreter);
+ }
+ LOGGER.info("Interpreter {} created for user: {}, sessionId: {}",
+ interpreter.getClassName(), user, sessionId);
+ }
+ return interpreters;
+ }
+
+ // Create Interpreter in ZeppelinServer for non-remote mode
+ private Interpreter createLocalInterpreter(String className)
+ throws InterpreterException {
+ LOGGER.info("Create Local Interpreter {} from {}", className, interpreterDir);
+
+ ClassLoader oldcl = Thread.currentThread().getContextClassLoader();
+ try {
+
+ URLClassLoader ccl = cleanCl.get(interpreterDir);
+ if (ccl == null) {
+ // classloader fallback
+ ccl = URLClassLoader.newInstance(new URL[]{}, oldcl);
+ }
+
+ boolean separateCL = true;
+ try { // check if server's classloader has driver already.
+ Class cls = this.getClass().forName(className);
+ if (cls != null) {
+ separateCL = false;
+ }
+ } catch (Exception e) {
+ LOGGER.error("exception checking server classloader driver", e);
+ }
+
+ URLClassLoader cl;
+
+ if (separateCL == true) {
+ cl = URLClassLoader.newInstance(new URL[]{}, ccl);
+ } else {
+ cl = ccl;
+ }
+ Thread.currentThread().setContextClassLoader(cl);
+
+ Class<Interpreter> replClass = (Class<Interpreter>) cl.loadClass(className);
+ Constructor<Interpreter> constructor =
+ replClass.getConstructor(new Class[]{Properties.class});
+ Interpreter repl = constructor.newInstance(getJavaProperties());
+ repl.setClassloaderUrls(ccl.getURLs());
+ LazyOpenInterpreter intp = new LazyOpenInterpreter(new ClassloaderInterpreter(repl, cl));
+ return intp;
+ } catch (SecurityException e) {
+ throw new InterpreterException(e);
+ } catch (NoSuchMethodException e) {
+ throw new InterpreterException(e);
+ } catch (IllegalArgumentException e) {
+ throw new InterpreterException(e);
+ } catch (InstantiationException e) {
+ throw new InterpreterException(e);
+ } catch (IllegalAccessException e) {
+ throw new InterpreterException(e);
+ } catch (InvocationTargetException e) {
+ throw new InterpreterException(e);
+ } catch (ClassNotFoundException e) {
+ throw new InterpreterException(e);
+ } finally {
+ Thread.currentThread().setContextClassLoader(oldcl);
+ }
+ }
+
+ RemoteInterpreterProcess createInterpreterProcess() {
+ RemoteInterpreterProcess remoteInterpreterProcess = null;
+ int connectTimeout =
+ conf.getInt(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
+ String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + id;
+ if (option.isExistingProcess()) {
+ // TODO(zjffdu) remove the existing process approach seems no one is using this.
+ // use the existing process
+ remoteInterpreterProcess = new RemoteInterpreterRunningProcess(
+ connectTimeout,
+ remoteInterpreterProcessListener,
+ appEventListener,
+ option.getHost(),
+ option.getPort());
+ } else {
+ // create new remote process
+ remoteInterpreterProcess = new RemoteInterpreterManagedProcess(
+ interpreterRunner != null ? interpreterRunner.getPath() :
+ conf.getInterpreterRemoteRunnerPath(), interpreterDir, localRepoPath,
+ getEnvFromInterpreterProperty(getJavaProperties()), connectTimeout,
+ remoteInterpreterProcessListener, appEventListener, group);
+ }
+ return remoteInterpreterProcess;
+ }
+
+ private Map<String, String> getEnvFromInterpreterProperty(Properties property) {
+ Map<String, String> env = new HashMap<>();
+ for (Object key : property.keySet()) {
+ if (RemoteInterpreterUtils.isEnvString((String) key)) {
+ env.put((String) key, property.getProperty((String) key));
+ }
+ }
+ return env;
+ }
+
+ private List<Interpreter> getOrCreateSession(String user, String noteId) {
+ ManagedInterpreterGroup interpreterGroup = getOrCreateInterpreterGroup(user, noteId);
+ Preconditions.checkNotNull(interpreterGroup, "No InterpreterGroup existed for user {}, " +
+ "noteId {}", user, noteId);
+ String sessionId = getInterpreterSessionId(user, noteId);
+ return interpreterGroup.getOrCreateSession(user, sessionId);
+ }
+
+ public Interpreter getDefaultInterpreter(String user, String noteId) {
+ return getOrCreateSession(user, noteId).get(0);
+ }
+
+ public Interpreter getInterpreter(String user, String noteId, String replName) {
+ Preconditions.checkNotNull(noteId, "noteId should be not null");
+ Preconditions.checkNotNull(replName, "replName should be not null");
+
+ String className = getInterpreterClassFromInterpreterSetting(replName);
+ if (className == null) {
+ return null;
+ }
+ List<Interpreter> interpreters = getOrCreateSession(user, noteId);
+ for (Interpreter interpreter : interpreters) {
+ if (className.equals(interpreter.getClassName())) {
+ return interpreter;
+ }
+ }
+ return null;
+ }
+
+ private String getInterpreterClassFromInterpreterSetting(String replName) {
+ Preconditions.checkNotNull(replName, "replName should be not null");
+
+ for (InterpreterInfo info : interpreterInfos) {
+ String infoName = info.getName();
+ if (null != info.getName() && replName.equals(infoName)) {
+ return info.getClassName();
+ }
+ }
+ return null;
+ }
+
+ private ManagedInterpreterGroup createInterpreterGroup(String groupId)
+ throws InterpreterException {
+ AngularObjectRegistry angularObjectRegistry;
+ ManagedInterpreterGroup interpreterGroup = new ManagedInterpreterGroup(groupId, this);
+ if (option.isRemote()) {
+ angularObjectRegistry =
+ new RemoteAngularObjectRegistry(groupId, angularObjectRegistryListener, interpreterGroup);
+ } else {
+ angularObjectRegistry = new AngularObjectRegistry(id, angularObjectRegistryListener);
+ // TODO(moon) : create distributed resource pool for local interpreters and set
+ }
+
+ interpreterGroup.setAngularObjectRegistry(angularObjectRegistry);
+ return interpreterGroup;
+ }
+
+ private void loadInterpreterDependencies() {
+ setStatus(Status.DOWNLOADING_DEPENDENCIES);
+ setErrorReason(null);
+ Thread t = new Thread() {
+ public void run() {
+ try {
+ // dependencies to prevent library conflict
+ File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + getId());
+ if (localRepoDir.exists()) {
+ try {
+ FileUtils.forceDelete(localRepoDir);
+ } catch (FileNotFoundException e) {
+ LOGGER.info("A file that does not exist cannot be deleted, nothing to worry", e);
+ }
+ }
+
+ // load dependencies
+ List<Dependency> deps = getDependencies();
+ if (deps != null) {
+ for (Dependency d : deps) {
+ File destDir = new File(
+ conf.getRelativeDir(ZeppelinConfiguration.ConfVars.ZEPPELIN_DEP_LOCALREPO));
+
+ if (d.getExclusions() != null) {
+ dependencyResolver.load(d.getGroupArtifactVersion(), d.getExclusions(),
+ new File(destDir, id));
+ } else {
+ dependencyResolver
+ .load(d.getGroupArtifactVersion(), new File(destDir, id));
+ }
+ }
+ }
+
+ setStatus(Status.READY);
+ setErrorReason(null);
+ } catch (Exception e) {
+ LOGGER.error(String.format("Error while downloading repos for interpreter group : %s," +
+ " go to interpreter setting page click on edit and save it again to make " +
+ "this interpreter work properly. : %s",
+ getGroup(), e.getLocalizedMessage()), e);
+ setErrorReason(e.getLocalizedMessage());
+ setStatus(Status.ERROR);
+ }
+ }
+ };
+
+ t.start();
+ }
+
+ //TODO(zjffdu) ugly code, should not use JsonObject as parameter. not readable
public void convertPermissionsFromUsersToOwners(JsonObject jsonObject) {
if (jsonObject != null) {
JsonObject option = jsonObject.getAsJsonObject("option");
@@ -434,26 +847,56 @@ public class InterpreterSetting {
}
// For backward compatibility of interpreter.json format after ZEPPELIN-2403
- public void convertFlatPropertiesToPropertiesWithWidgets() {
- StringMap newProperties = new StringMap();
+ static Map<String, InterpreterProperty> convertInterpreterProperties(Object properties) {
if (properties != null && properties instanceof StringMap) {
+ Map<String, InterpreterProperty> newProperties = new HashMap<>();
StringMap p = (StringMap) properties;
-
for (Object o : p.entrySet()) {
Map.Entry entry = (Map.Entry) o;
if (!(entry.getValue() instanceof StringMap)) {
- StringMap newProperty = new StringMap();
- newProperty.put("name", entry.getKey());
- newProperty.put("value", entry.getValue());
- newProperty.put("type", InterpreterPropertyType.TEXTAREA.getValue());
+ InterpreterProperty newProperty = new InterpreterProperty(
+ entry.getKey().toString(),
+ entry.getValue(),
+ InterpreterPropertyType.STRING.getValue());
newProperties.put(entry.getKey().toString(), newProperty);
} else {
// already converted
- return;
+ return (Map<String, InterpreterProperty>) properties;
}
}
-
- this.properties = newProperties;
+ return newProperties;
+
+ } else if (properties instanceof Map) {
+ Map<String, Object> dProperties =
+ (Map<String, Object>) properties;
+ Map<String, InterpreterProperty> newProperties = new HashMap<>();
+ for (String key : dProperties.keySet()) {
+ Object value = dProperties.get(key);
+ if (value instanceof InterpreterProperty) {
+ return (Map<String, InterpreterProperty>) properties;
+ } else if (value instanceof StringMap) {
+ StringMap stringMap = (StringMap) value;
+ InterpreterProperty newProperty = new InterpreterProperty(
+ key,
+ stringMap.get("value"),
+ stringMap.containsKey("type") ? stringMap.get("type").toString() : "string");
+
+ newProperties.put(newProperty.getName(), newProperty);
+ } else if (value instanceof DefaultInterpreterProperty){
+ DefaultInterpreterProperty dProperty = (DefaultInterpreterProperty) value;
+ InterpreterProperty property = new InterpreterProperty(
+ key,
+ dProperty.getValue(),
+ dProperty.getType() != null ? dProperty.getType() : "string"
+ // in case user forget to specify type in interpreter-setting.json
+ );
+ newProperties.put(key, property);
+ } else {
+ throw new RuntimeException("Can not convert this type of property: " + value.getClass());
+ }
+ }
+ return newProperties;
}
+ throw new RuntimeException("Can not convert this type: " + properties.getClass());
}
}
[5/9] zeppelin git commit: [ZEPPELIN-2627] Interpreter Component
Refactoring
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java
index 0ac7116..924901b 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java
@@ -17,29 +17,28 @@
package org.apache.zeppelin.interpreter.remote;
-import java.util.List;
-
-import org.apache.thrift.TException;
+import com.google.gson.Gson;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.ManagedInterpreterGroup;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.gson.Gson;
+import java.util.List;
/**
* Proxy for AngularObjectRegistry that exists in remote interpreter process
*/
public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
Logger logger = LoggerFactory.getLogger(RemoteAngularObjectRegistry.class);
- private InterpreterGroup interpreterGroup;
+ private ManagedInterpreterGroup interpreterGroup;
public RemoteAngularObjectRegistry(String interpreterId,
- AngularObjectRegistryListener listener,
- InterpreterGroup interpreterGroup) {
+ AngularObjectRegistryListener listener,
+ ManagedInterpreterGroup interpreterGroup) {
super(interpreterId, listener);
this.interpreterGroup = interpreterGroup;
}
@@ -56,31 +55,29 @@ public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
* @param noteId
* @return
*/
- public AngularObject addAndNotifyRemoteProcess(String name, Object o, String noteId, String
- paragraphId) {
- Gson gson = new Gson();
+ public AngularObject addAndNotifyRemoteProcess(final String name,
+ final Object o,
+ final String noteId,
+ final String paragraphId) {
+
RemoteInterpreterProcess remoteInterpreterProcess = getRemoteInterpreterProcess();
if (!remoteInterpreterProcess.isRunning()) {
return super.add(name, o, noteId, paragraphId, true);
}
- Client client = null;
- boolean broken = false;
- try {
- client = remoteInterpreterProcess.getClient();
- client.angularObjectAdd(name, noteId, paragraphId, gson.toJson(o));
- return super.add(name, o, noteId, paragraphId, true);
- } catch (TException e) {
- broken = true;
- logger.error("Error", e);
- } catch (Exception e) {
- logger.error("Error", e);
- } finally {
- if (client != null) {
- remoteInterpreterProcess.releaseClient(client, broken);
- }
- }
- return null;
+ remoteInterpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ Gson gson = new Gson();
+ client.angularObjectAdd(name, noteId, paragraphId, gson.toJson(o));
+ return null;
+ }
+ }
+ );
+
+ return super.add(name, o, noteId, paragraphId, true);
+
}
/**
@@ -91,30 +88,24 @@ public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
* @param paragraphId
* @return
*/
- public AngularObject removeAndNotifyRemoteProcess(String name, String noteId, String
- paragraphId) {
+ public AngularObject removeAndNotifyRemoteProcess(final String name,
+ final String noteId,
+ final String paragraphId) {
RemoteInterpreterProcess remoteInterpreterProcess = getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null || !remoteInterpreterProcess.isRunning()) {
return super.remove(name, noteId, paragraphId);
}
-
- Client client = null;
- boolean broken = false;
- try {
- client = remoteInterpreterProcess.getClient();
- client.angularObjectRemove(name, noteId, paragraphId);
- return super.remove(name, noteId, paragraphId);
- } catch (TException e) {
- broken = true;
- logger.error("Error", e);
- } catch (Exception e) {
- logger.error("Error", e);
- } finally {
- if (client != null) {
- remoteInterpreterProcess.releaseClient(client, broken);
+ remoteInterpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ client.angularObjectRemove(name, noteId, paragraphId);
+ return null;
+ }
}
- }
- return null;
+ );
+
+ return super.remove(name, noteId, paragraphId);
}
public void removeAllAndNotifyRemoteProcess(String noteId, String paragraphId) {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
index 12e0caa..54bf9e1 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
@@ -17,160 +17,68 @@
package org.apache.zeppelin.interpreter.remote;
-import java.util.*;
-
-import org.apache.commons.lang3.StringUtils;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
import org.apache.thrift.TException;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
-import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.display.Input;
-import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterContextRunner;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.ManagedInterpreterGroup;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
+import org.apache.zeppelin.scheduler.Job;
+import org.apache.zeppelin.scheduler.RemoteScheduler;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
/**
* Proxy for Interpreter instance that runs on separate process
*/
public class RemoteInterpreter extends Interpreter {
- private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreter.class);
-
- private final RemoteInterpreterProcessListener remoteInterpreterProcessListener;
- private final ApplicationEventListener applicationEventListener;
- private Gson gson = new Gson();
- private String interpreterRunner;
- private String interpreterPath;
- private String localRepoPath;
+ private static final Logger LOGGER = LoggerFactory.getLogger(RemoteInterpreter.class);
+ private static final Gson gson = new Gson();
+
+
private String className;
- private String sessionKey;
- private FormType formType;
- private boolean initialized;
- private Map<String, String> env;
- private int connectTimeout;
- private int maxPoolSize;
- private String host;
- private int port;
+ private String sessionId;
private String userName;
- private Boolean isUserImpersonate;
- private int outputLimit = Constants.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT;
- private String interpreterGroupName;
-
- /**
- * Remote interpreter and manage interpreter process
- */
- public RemoteInterpreter(Properties property, String sessionKey, String className,
- String interpreterRunner, String interpreterPath, String localRepoPath, int connectTimeout,
- int maxPoolSize, RemoteInterpreterProcessListener remoteInterpreterProcessListener,
- ApplicationEventListener appListener, String userName, Boolean isUserImpersonate,
- int outputLimit, String interpreterGroupName) {
- super(property);
- this.sessionKey = sessionKey;
- this.className = className;
- initialized = false;
- this.interpreterRunner = interpreterRunner;
- this.interpreterPath = interpreterPath;
- this.localRepoPath = localRepoPath;
- env = getEnvFromInterpreterProperty(property);
- this.connectTimeout = connectTimeout;
- this.maxPoolSize = maxPoolSize;
- this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
- this.applicationEventListener = appListener;
- this.userName = userName;
- this.isUserImpersonate = isUserImpersonate;
- this.outputLimit = outputLimit;
- this.interpreterGroupName = interpreterGroupName;
- }
+ private FormType formType;
+ private RemoteInterpreterProcess interpreterProcess;
+ private volatile boolean isOpened = false;
+ private volatile boolean isCreated = false;
/**
- * Connect to existing process
+ * Remote interpreter and manage interpreter process
*/
- public RemoteInterpreter(Properties property, String sessionKey, String className, String host,
- int port, String localRepoPath, int connectTimeout, int maxPoolSize,
- RemoteInterpreterProcessListener remoteInterpreterProcessListener,
- ApplicationEventListener appListener, String userName, Boolean isUserImpersonate,
- int outputLimit) {
- super(property);
- this.sessionKey = sessionKey;
+ public RemoteInterpreter(Properties properties,
+ String sessionId,
+ String className,
+ String userName) {
+ super(properties);
+ this.sessionId = sessionId;
this.className = className;
- initialized = false;
- this.host = host;
- this.port = port;
- this.localRepoPath = localRepoPath;
- this.connectTimeout = connectTimeout;
- this.maxPoolSize = maxPoolSize;
- this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
- this.applicationEventListener = appListener;
this.userName = userName;
- this.isUserImpersonate = isUserImpersonate;
- this.outputLimit = outputLimit;
- }
-
-
- // VisibleForTesting
- public RemoteInterpreter(Properties property, String sessionKey, String className,
- String interpreterRunner, String interpreterPath, String localRepoPath,
- Map<String, String> env, int connectTimeout,
- RemoteInterpreterProcessListener remoteInterpreterProcessListener,
- ApplicationEventListener appListener, String userName, Boolean isUserImpersonate) {
- super(property);
- this.className = className;
- this.sessionKey = sessionKey;
- this.interpreterRunner = interpreterRunner;
- this.interpreterPath = interpreterPath;
- this.localRepoPath = localRepoPath;
- env.putAll(getEnvFromInterpreterProperty(property));
- this.env = env;
- this.connectTimeout = connectTimeout;
- this.maxPoolSize = 10;
- this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
- this.applicationEventListener = appListener;
- this.userName = userName;
- this.isUserImpersonate = isUserImpersonate;
- }
-
- private Map<String, String> getEnvFromInterpreterProperty(Properties property) {
- Map<String, String> env = new HashMap<String, String>();
- StringBuilder sparkConfBuilder = new StringBuilder();
- for (String key : property.stringPropertyNames()) {
- if (RemoteInterpreterUtils.isEnvString(key)) {
- env.put(key, property.getProperty(key));
- }
- if (key.equals("master")) {
- sparkConfBuilder.append(" --master " + property.getProperty("master"));
- }
- if (isSparkConf(key, property.getProperty(key))) {
- sparkConfBuilder.append(" --conf " + key + "=" +
- toShellFormat(property.getProperty(key)));
- }
- }
- env.put("ZEPPELIN_SPARK_CONF", sparkConfBuilder.toString());
- return env;
}
- private String toShellFormat(String value) {
- if (value.contains("\'") && value.contains("\"")) {
- throw new RuntimeException("Spark property value could not contain both \" and '");
- } else if (value.contains("\'")) {
- return "\"" + value + "\"";
- } else {
- return "\'" + value + "\'";
- }
- }
-
- static boolean isSparkConf(String key, String value) {
- return !StringUtils.isEmpty(key) && key.startsWith("spark.") && !StringUtils.isEmpty(value);
+ public boolean isOpened() {
+ return isOpened;
}
@Override
@@ -178,202 +86,113 @@ public class RemoteInterpreter extends Interpreter {
return className;
}
- private boolean connectToExistingProcess() {
- return host != null && port > 0;
+ public String getSessionId() {
+ return this.sessionId;
}
- public RemoteInterpreterProcess getInterpreterProcess() {
- InterpreterGroup intpGroup = getInterpreterGroup();
- if (intpGroup == null) {
- return null;
+ public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess() {
+ if (this.interpreterProcess != null) {
+ return this.interpreterProcess;
}
-
- synchronized (intpGroup) {
- if (intpGroup.getRemoteInterpreterProcess() == null) {
- RemoteInterpreterProcess remoteProcess;
- if (connectToExistingProcess()) {
- remoteProcess = new RemoteInterpreterRunningProcess(
- connectTimeout,
- remoteInterpreterProcessListener,
- applicationEventListener,
- host,
- port);
- } else {
- // create new remote process
- remoteProcess = new RemoteInterpreterManagedProcess(
- interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout,
- remoteInterpreterProcessListener, applicationEventListener, interpreterGroupName);
- }
-
- intpGroup.setRemoteInterpreterProcess(remoteProcess);
+ ManagedInterpreterGroup intpGroup = getInterpreterGroup();
+ this.interpreterProcess = intpGroup.getOrCreateInterpreterProcess();
+ synchronized (interpreterProcess) {
+ if (!interpreterProcess.isRunning()) {
+ interpreterProcess.start(userName, false);
+ interpreterProcess.getRemoteInterpreterEventPoller()
+ .setInterpreterProcess(interpreterProcess);
+ interpreterProcess.getRemoteInterpreterEventPoller().setInterpreterGroup(intpGroup);
+ interpreterProcess.getRemoteInterpreterEventPoller().start();
}
-
- return intpGroup.getRemoteInterpreterProcess();
}
+ return interpreterProcess;
}
- public synchronized void init() {
- if (initialized == true) {
- return;
- }
-
- RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
-
- final InterpreterGroup interpreterGroup = getInterpreterGroup();
-
- interpreterProcess.setMaxPoolSize(
- Math.max(this.maxPoolSize, interpreterProcess.getMaxPoolSize()));
- String groupId = interpreterGroup.getId();
-
- synchronized (interpreterProcess) {
- Client client = null;
- try {
- client = interpreterProcess.getClient();
- } catch (Exception e1) {
- throw new InterpreterException(e1);
- }
-
- boolean broken = false;
- try {
- logger.info("Create remote interpreter {}", getClassName());
- if (localRepoPath != null) {
- property.put("zeppelin.interpreter.localRepo", localRepoPath);
- }
+ public ManagedInterpreterGroup getInterpreterGroup() {
+ return (ManagedInterpreterGroup) super.getInterpreterGroup();
+ }
- property.put("zeppelin.interpreter.output.limit", Integer.toString(outputLimit));
- client.createInterpreter(groupId, sessionKey,
- getClassName(), (Map) property, userName);
- // Push angular object loaded from JSON file to remote interpreter
- if (!interpreterGroup.isAngularRegistryPushed()) {
- pushAngularObjectRegistryToRemote(client);
- interpreterGroup.setAngularRegistryPushed(true);
+ @Override
+ public void open() {
+ synchronized (this) {
+ if (!isOpened) {
+ // create all the interpreters of the same session first, then Open the internal interpreter
+ // of this RemoteInterpreter.
+ // The why we we create all the interpreter of the session is because some interpreter
+ // depends on other interpreter. e.g. PySparkInterpreter depends on SparkInterpreter.
+ // also see method Interpreter.getInterpreterInTheSameSessionByClassName
+ for (Interpreter interpreter : getInterpreterGroup()
+ .getOrCreateSession(userName, sessionId)) {
+ ((RemoteInterpreter) interpreter).internal_create();
}
- } catch (TException e) {
- logger.error("Failed to create interpreter: {}", getClassName());
- throw new InterpreterException(e);
- } finally {
- // TODO(jongyoul): Fixed it when not all of interpreter in same interpreter group are broken
- interpreterProcess.releaseClient(client, broken);
+ interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ LOGGER.info("Open RemoteInterpreter {}", getClassName());
+ // open interpreter here instead of in the jobRun method in RemoteInterpreterServer
+ // client.open(sessionId, className);
+ // Push angular object loaded from JSON file to remote interpreter
+ synchronized (getInterpreterGroup()) {
+ if (!getInterpreterGroup().isAngularRegistryPushed()) {
+ pushAngularObjectRegistryToRemote(client);
+ getInterpreterGroup().setAngularRegistryPushed(true);
+ }
+ }
+ return null;
+ }
+ });
+ isOpened = true;
}
}
- initialized = true;
}
-
- @Override
- public void open() {
- InterpreterGroup interpreterGroup = getInterpreterGroup();
-
- synchronized (interpreterGroup) {
- // initialize all interpreters in this interpreter group
- List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
- // TODO(jl): this open method is called by LazyOpenInterpreter.open(). It, however,
- // initializes all of interpreters with same sessionKey. But LazyOpenInterpreter assumes if it
- // doesn't call open method, it's not open. It causes problem while running intp.close()
- // In case of Spark, this method initializes all of interpreters and init() method increases
- // reference count of RemoteInterpreterProcess. But while closing this interpreter group, all
- // other interpreters doesn't do anything because those LazyInterpreters aren't open.
- // But for now, we have to initialise all of interpreters for some reasons.
- // See Interpreter.getInterpreterInTheSameSessionByClassName(String)
- RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
- if (!initialized) {
- // reference per session
- interpreterProcess.reference(interpreterGroup, userName, isUserImpersonate);
- }
- for (Interpreter intp : new ArrayList<>(interpreters)) {
- Interpreter p = intp;
- while (p instanceof WrappedInterpreter) {
- p = ((WrappedInterpreter) p).getInnerInterpreter();
- }
- try {
- ((RemoteInterpreter) p).init();
- } catch (InterpreterException e) {
- logger.error("Failed to initialize interpreter: {}. Remove it from interpreterGroup",
- p.getClassName());
- interpreters.remove(p);
- }
+ private void internal_create() {
+ synchronized (this) {
+ if (!isCreated) {
+ RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
+ interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ LOGGER.info("Create RemoteInterpreter {}", getClassName());
+ client.createInterpreter(getInterpreterGroup().getId(), sessionId,
+ className, (Map) property, userName);
+ return null;
+ }
+ });
+ isCreated = true;
}
}
}
+
@Override
public void close() {
- InterpreterGroup interpreterGroup = getInterpreterGroup();
- synchronized (interpreterGroup) {
- // close all interpreters in this session
- List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
- // TODO(jl): this open method is called by LazyOpenInterpreter.open(). It, however,
- // initializes all of interpreters with same sessionKey. But LazyOpenInterpreter assumes if it
- // doesn't call open method, it's not open. It causes problem while running intp.close()
- // In case of Spark, this method initializes all of interpreters and init() method increases
- // reference count of RemoteInterpreterProcess. But while closing this interpreter group, all
- // other interpreters doesn't do anything because those LazyInterpreters aren't open.
- // But for now, we have to initialise all of interpreters for some reasons.
- // See Interpreter.getInterpreterInTheSameSessionByClassName(String)
- if (initialized) {
- // dereference per session
- getInterpreterProcess().dereference();
- }
- for (Interpreter intp : new ArrayList<>(interpreters)) {
- Interpreter p = intp;
- while (p instanceof WrappedInterpreter) {
- p = ((WrappedInterpreter) p).getInnerInterpreter();
- }
- try {
- ((RemoteInterpreter) p).closeInterpreter();
- } catch (InterpreterException e) {
- logger.error("Failed to initialize interpreter: {}. Remove it from interpreterGroup",
- p.getClassName());
- interpreters.remove(p);
+ if (isOpened) {
+ RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
+ interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ client.close(sessionId, className);
+ return null;
}
- }
- }
- }
-
- public void closeInterpreter() {
- if (this.initialized == false) {
- return;
- }
- RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
- Client client = null;
- boolean broken = false;
- try {
- client = interpreterProcess.getClient();
- if (client != null) {
- client.close(sessionKey, className);
- }
- } catch (TException e) {
- broken = true;
- throw new InterpreterException(e);
- } catch (Exception e1) {
- throw new InterpreterException(e1);
- } finally {
- if (client != null) {
- interpreterProcess.releaseClient(client, broken);
- }
- this.initialized = false;
+ });
+ isOpened = false;
+ } else {
+ LOGGER.warn("close is called when RemoterInterpreter is not opened for " + className);
}
}
@Override
- public InterpreterResult interpret(String st, InterpreterContext context) {
- if (logger.isDebugEnabled()) {
- logger.debug("st:\n{}", st);
- }
-
- FormType form = getFormType();
- RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
- Client client = null;
- try {
- client = interpreterProcess.getClient();
- } catch (Exception e1) {
- throw new InterpreterException(e1);
+ public InterpreterResult interpret(final String st, final InterpreterContext context) {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("st:\n{}", st);
}
+ final FormType form = getFormType();
+ RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
InterpreterContextRunnerPool interpreterContextRunnerPool = interpreterProcess
.getInterpreterContextRunnerPool();
-
List<InterpreterContextRunner> runners = context.getRunners();
if (runners != null && runners.size() != 0) {
// assume all runners in this InterpreterContext have the same note id
@@ -382,165 +201,153 @@ public class RemoteInterpreter extends Interpreter {
interpreterContextRunnerPool.clear(noteId);
interpreterContextRunnerPool.addAll(noteId, runners);
}
+ return interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<InterpreterResult>() {
+ @Override
+ public InterpreterResult call(Client client) throws Exception {
+
+ RemoteInterpreterResult remoteResult = client.interpret(
+ sessionId, className, st, convert(context));
+ Map<String, Object> remoteConfig = (Map<String, Object>) gson.fromJson(
+ remoteResult.getConfig(), new TypeToken<Map<String, Object>>() {
+ }.getType());
+ context.getConfig().clear();
+ context.getConfig().putAll(remoteConfig);
+ GUI currentGUI = context.getGui();
+ if (form == FormType.NATIVE) {
+ GUI remoteGui = GUI.fromJson(remoteResult.getGui());
+ currentGUI.clear();
+ currentGUI.setParams(remoteGui.getParams());
+ currentGUI.setForms(remoteGui.getForms());
+ } else if (form == FormType.SIMPLE) {
+ final Map<String, Input> currentForms = currentGUI.getForms();
+ final Map<String, Object> currentParams = currentGUI.getParams();
+ final GUI remoteGUI = GUI.fromJson(remoteResult.getGui());
+ final Map<String, Input> remoteForms = remoteGUI.getForms();
+ final Map<String, Object> remoteParams = remoteGUI.getParams();
+ currentForms.putAll(remoteForms);
+ currentParams.putAll(remoteParams);
+ }
+
+ InterpreterResult result = convert(remoteResult);
+ return result;
+ }
+ }
+ );
- boolean broken = false;
- try {
-
- final GUI currentGUI = context.getGui();
- RemoteInterpreterResult remoteResult = client.interpret(
- sessionKey, className, st, convert(context));
-
- Map<String, Object> remoteConfig = (Map<String, Object>) gson.fromJson(
- remoteResult.getConfig(), new TypeToken<Map<String, Object>>() {
- }.getType());
- context.getConfig().clear();
- context.getConfig().putAll(remoteConfig);
-
- if (form == FormType.NATIVE) {
- GUI remoteGui = GUI.fromJson(remoteResult.getGui());
- currentGUI.clear();
- currentGUI.setParams(remoteGui.getParams());
- currentGUI.setForms(remoteGui.getForms());
- } else if (form == FormType.SIMPLE) {
- final Map<String, Input> currentForms = currentGUI.getForms();
- final Map<String, Object> currentParams = currentGUI.getParams();
- final GUI remoteGUI = GUI.fromJson(remoteResult.getGui());
- final Map<String, Input> remoteForms = remoteGUI.getForms();
- final Map<String, Object> remoteParams = remoteGUI.getParams();
- currentForms.putAll(remoteForms);
- currentParams.putAll(remoteParams);
- }
-
- InterpreterResult result = convert(remoteResult);
- return result;
- } catch (TException e) {
- broken = true;
- throw new InterpreterException(e);
- } finally {
- interpreterProcess.releaseClient(client, broken);
- }
}
@Override
- public void cancel(InterpreterContext context) {
- RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
- Client client = null;
- try {
- client = interpreterProcess.getClient();
- } catch (Exception e1) {
- throw new InterpreterException(e1);
- }
-
- boolean broken = false;
- try {
- client.cancel(sessionKey, className, convert(context));
- } catch (TException e) {
- broken = true;
- throw new InterpreterException(e);
- } finally {
- interpreterProcess.releaseClient(client, broken);
+ public void cancel(final InterpreterContext context) {
+ if (!isOpened) {
+ LOGGER.warn("Cancel is called when RemoterInterpreter is not opened for " + className);
+ return;
}
+ RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
+ interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ client.cancel(sessionId, className, convert(context));
+ return null;
+ }
+ });
}
@Override
public FormType getFormType() {
- open();
-
if (formType != null) {
return formType;
}
- RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
- Client client = null;
- try {
- client = interpreterProcess.getClient();
- } catch (Exception e1) {
- throw new InterpreterException(e1);
- }
-
- boolean broken = false;
- try {
- formType = FormType.valueOf(client.getFormType(sessionKey, className));
- return formType;
- } catch (TException e) {
- broken = true;
- throw new InterpreterException(e);
- } finally {
- interpreterProcess.releaseClient(client, broken);
+ // it is possible to call getFormType before it is opened
+ synchronized (this) {
+ if (!isOpened) {
+ open();
+ }
}
+ RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
+ FormType type = interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<FormType>() {
+ @Override
+ public FormType call(Client client) throws Exception {
+ formType = FormType.valueOf(client.getFormType(sessionId, className));
+ return formType;
+ }
+ });
+ return type;
}
@Override
- public int getProgress(InterpreterContext context) {
- RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
- if (interpreterProcess == null || !interpreterProcess.isRunning()) {
+ public int getProgress(final InterpreterContext context) {
+ if (!isOpened) {
+ LOGGER.warn("getProgress is called when RemoterInterpreter is not opened for " + className);
return 0;
}
-
- Client client = null;
- try {
- client = interpreterProcess.getClient();
- } catch (Exception e1) {
- throw new InterpreterException(e1);
- }
-
- boolean broken = false;
- try {
- return client.getProgress(sessionKey, className, convert(context));
- } catch (TException e) {
- broken = true;
- throw new InterpreterException(e);
- } finally {
- interpreterProcess.releaseClient(client, broken);
- }
+ RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
+ return interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<Integer>() {
+ @Override
+ public Integer call(Client client) throws Exception {
+ return client.getProgress(sessionId, className, convert(context));
+ }
+ });
}
@Override
- public List<InterpreterCompletion> completion(String buf, int cursor,
- InterpreterContext interpreterContext) {
- RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
- Client client = null;
- try {
- client = interpreterProcess.getClient();
- } catch (Exception e1) {
- throw new InterpreterException(e1);
+ public List<InterpreterCompletion> completion(final String buf, final int cursor,
+ final InterpreterContext interpreterContext) {
+ if (!isOpened) {
+ LOGGER.warn("completion is called when RemoterInterpreter is not opened for " + className);
+ return new ArrayList<>();
}
+ RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
+ return interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<List<InterpreterCompletion>>() {
+ @Override
+ public List<InterpreterCompletion> call(Client client) throws Exception {
+ return client.completion(sessionId, className, buf, cursor,
+ convert(interpreterContext));
+ }
+ });
+ }
- boolean broken = false;
- try {
- List completion = client.completion(sessionKey, className, buf, cursor,
- convert(interpreterContext));
- return completion;
- } catch (TException e) {
- broken = true;
- throw new InterpreterException(e);
- } finally {
- interpreterProcess.releaseClient(client, broken);
+ public String getStatus(final String jobId) {
+ if (!isOpened) {
+ LOGGER.warn("getStatus is called when RemoteInterpreter is not opened for " + className);
+ return Job.Status.UNKNOWN.name();
}
+ RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
+ return interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<String>() {
+ @Override
+ public String call(Client client) throws Exception {
+ return client.getStatus(sessionId, jobId);
+ }
+ });
}
+ //TODO(zjffdu) Share the Scheduler in the same session or in the same InterpreterGroup ?
@Override
public Scheduler getScheduler() {
- int maxConcurrency = maxPoolSize;
- RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
- if (interpreterProcess == null) {
- return null;
- } else {
- return SchedulerFactory.singleton().createOrGetRemoteScheduler(
- RemoteInterpreter.class.getName() + sessionKey + interpreterProcess.hashCode(),
- sessionKey, interpreterProcess, maxConcurrency);
- }
- }
-
- private String getInterpreterGroupKey(InterpreterGroup interpreterGroup) {
- return interpreterGroup.getId();
+ int maxConcurrency = Integer.parseInt(
+ property.getProperty("zeppelin.interpreter.max.poolsize",
+ ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE.getIntValue() + ""));
+
+ Scheduler s = new RemoteScheduler(
+ RemoteInterpreter.class.getName() + "-" + sessionId,
+ SchedulerFactory.singleton().getExecutor(),
+ sessionId,
+ this,
+ SchedulerFactory.singleton(),
+ maxConcurrency);
+ return SchedulerFactory.singleton().createOrGetScheduler(s);
}
private RemoteInterpreterContext convert(InterpreterContext ic) {
return new RemoteInterpreterContext(ic.getNoteId(), ic.getParagraphId(), ic.getReplName(),
- ic.getParagraphTitle(), ic.getParagraphText(), ic.getAuthenticationInfo().toJson(),
- gson.toJson(ic.getConfig()), ic.getGui().toJson(), gson.toJson(ic.getRunners()));
+ ic.getParagraphTitle(), ic.getParagraphText(), gson.toJson(ic.getAuthenticationInfo()),
+ gson.toJson(ic.getConfig()), gson.toJson(ic.getGui()), gson.toJson(ic.getRunners()));
}
private InterpreterResult convert(RemoteInterpreterResult result) {
@@ -557,41 +364,25 @@ public class RemoteInterpreter extends Interpreter {
/**
* Push local angular object registry to
* remote interpreter. This method should be
- * call ONLY inside the init() method
+ * call ONLY once when the first Interpreter is created
*/
- void pushAngularObjectRegistryToRemote(Client client) throws TException {
+ private void pushAngularObjectRegistryToRemote(Client client) throws TException {
final AngularObjectRegistry angularObjectRegistry = this.getInterpreterGroup()
.getAngularObjectRegistry();
-
if (angularObjectRegistry != null && angularObjectRegistry.getRegistry() != null) {
final Map<String, Map<String, AngularObject>> registry = angularObjectRegistry
.getRegistry();
-
- logger.info("Push local angular object registry from ZeppelinServer to" +
+ LOGGER.info("Push local angular object registry from ZeppelinServer to" +
" remote interpreter group {}", this.getInterpreterGroup().getId());
-
final java.lang.reflect.Type registryType = new TypeToken<Map<String,
Map<String, AngularObject>>>() {
}.getType();
-
- Gson gson = new Gson();
client.angularRegistryPush(gson.toJson(registry, registryType));
}
}
- public Map<String, String> getEnv() {
- return env;
- }
-
- public void addEnv(Map<String, String> env) {
- if (this.env == null) {
- this.env = new HashMap<>();
- }
- this.env.putAll(env);
- }
-
- //Only for test
- public String getInterpreterRunner() {
- return interpreterRunner;
+ @Override
+ public String toString() {
+ return "RemoteInterpreter_" + className + "_" + sessionId;
}
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
new file mode 100644
index 0000000..ca23bcf
--- /dev/null
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
@@ -0,0 +1,525 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import org.apache.thrift.TException;
+import org.apache.zeppelin.display.AngularObject;
+import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.helium.ApplicationEventListener;
+import org.apache.zeppelin.interpreter.InterpreterContextRunner;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.ManagedInterpreterGroup;
+import org.apache.zeppelin.interpreter.RemoteZeppelinServerResource;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
+import org.apache.zeppelin.interpreter.thrift.ZeppelinServerResourceParagraphRunner;
+import org.apache.zeppelin.resource.Resource;
+import org.apache.zeppelin.resource.ResourceId;
+import org.apache.zeppelin.resource.ResourcePool;
+import org.apache.zeppelin.resource.ResourceSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Processes message from RemoteInterpreter process
+ */
+public class RemoteInterpreterEventPoller extends Thread {
+ private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterEventPoller.class);
+ private final ScheduledExecutorService appendService =
+ Executors.newSingleThreadScheduledExecutor();
+ private final RemoteInterpreterProcessListener listener;
+ private final ApplicationEventListener appListener;
+
+ private volatile boolean shutdown;
+
+ private RemoteInterpreterProcess interpreterProcess;
+ private ManagedInterpreterGroup interpreterGroup;
+
+ Gson gson = new Gson();
+
+ public RemoteInterpreterEventPoller(
+ RemoteInterpreterProcessListener listener,
+ ApplicationEventListener appListener) {
+ this.listener = listener;
+ this.appListener = appListener;
+ shutdown = false;
+ }
+
+ public void setInterpreterProcess(RemoteInterpreterProcess interpreterProcess) {
+ this.interpreterProcess = interpreterProcess;
+ }
+
+ public void setInterpreterGroup(ManagedInterpreterGroup interpreterGroup) {
+ this.interpreterGroup = interpreterGroup;
+ }
+
+ @Override
+ public void run() {
+ AppendOutputRunner runner = new AppendOutputRunner(listener);
+ ScheduledFuture<?> appendFuture = appendService.scheduleWithFixedDelay(
+ runner, 0, AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
+
+ while (!shutdown) {
+ // wait and retry
+ if (!interpreterProcess.isRunning()) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ // nothing to do
+ }
+ continue;
+ }
+
+ RemoteInterpreterEvent event = interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<RemoteInterpreterEvent>() {
+ @Override
+ public RemoteInterpreterEvent call(Client client) throws Exception {
+ return client.getEvent();
+ }
+ }
+ );
+
+ AngularObjectRegistry angularObjectRegistry = interpreterGroup.getAngularObjectRegistry();
+
+ try {
+ if (event.getType() != RemoteInterpreterEventType.NO_OP) {
+ logger.debug("Receive message from RemoteInterpreter Process: " + event.toString());
+ }
+ if (event.getType() == RemoteInterpreterEventType.NO_OP) {
+ continue;
+ } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_ADD) {
+ AngularObject angularObject = AngularObject.fromJson(event.getData());
+ angularObjectRegistry.add(angularObject.getName(),
+ angularObject.get(), angularObject.getNoteId(), angularObject.getParagraphId());
+ } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_UPDATE) {
+ AngularObject angularObject = AngularObject.fromJson(event.getData());
+ AngularObject localAngularObject = angularObjectRegistry.get(
+ angularObject.getName(), angularObject.getNoteId(), angularObject.getParagraphId());
+ if (localAngularObject instanceof RemoteAngularObject) {
+ // to avoid ping-pong loop
+ ((RemoteAngularObject) localAngularObject).set(
+ angularObject.get(), true, false);
+ } else {
+ localAngularObject.set(angularObject.get());
+ }
+ } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_REMOVE) {
+ AngularObject angularObject = AngularObject.fromJson(event.getData());
+ angularObjectRegistry.remove(angularObject.getName(), angularObject.getNoteId(),
+ angularObject.getParagraphId());
+ } else if (event.getType() == RemoteInterpreterEventType.RUN_INTERPRETER_CONTEXT_RUNNER) {
+ InterpreterContextRunner runnerFromRemote = gson.fromJson(
+ event.getData(), RemoteInterpreterContextRunner.class);
+
+ listener.onRemoteRunParagraph(
+ runnerFromRemote.getNoteId(), runnerFromRemote.getParagraphId());
+
+ } else if (event.getType() == RemoteInterpreterEventType.RESOURCE_POOL_GET_ALL) {
+ ResourceSet resourceSet = getAllResourcePoolExcept();
+ sendResourcePoolResponseGetAll(resourceSet);
+ } else if (event.getType() == RemoteInterpreterEventType.RESOURCE_GET) {
+ String resourceIdString = event.getData();
+ ResourceId resourceId = ResourceId.fromJson(resourceIdString);
+ logger.debug("RESOURCE_GET {} {}", resourceId.getResourcePoolId(), resourceId.getName());
+ Object o = getResource(resourceId);
+ sendResourceResponseGet(resourceId, o);
+ } else if (event.getType() == RemoteInterpreterEventType.RESOURCE_INVOKE_METHOD) {
+ String message = event.getData();
+ InvokeResourceMethodEventMessage invokeMethodMessage =
+ InvokeResourceMethodEventMessage.fromJson(message);
+ Object ret = invokeResourceMethod(invokeMethodMessage);
+ sendInvokeMethodResult(invokeMethodMessage, ret);
+ } else if (event.getType() == RemoteInterpreterEventType.OUTPUT_APPEND) {
+ // on output append
+ Map<String, String> outputAppend = gson.fromJson(
+ event.getData(), new TypeToken<Map<String, Object>>() {}.getType());
+ String noteId = (String) outputAppend.get("noteId");
+ String paragraphId = (String) outputAppend.get("paragraphId");
+ int index = Integer.parseInt(outputAppend.get("index"));
+ String outputToAppend = (String) outputAppend.get("data");
+
+ String appId = (String) outputAppend.get("appId");
+
+ if (appId == null) {
+ runner.appendBuffer(noteId, paragraphId, index, outputToAppend);
+ } else {
+ appListener.onOutputAppend(noteId, paragraphId, index, appId, outputToAppend);
+ }
+ } else if (event.getType() == RemoteInterpreterEventType.OUTPUT_UPDATE_ALL) {
+ Map<String, Object> outputUpdate = gson.fromJson(
+ event.getData(), new TypeToken<Map<String, Object>>() {}.getType());
+ String noteId = (String) outputUpdate.get("noteId");
+ String paragraphId = (String) outputUpdate.get("paragraphId");
+
+ // clear the output
+ List<Map<String, String>> messages =
+ (List<Map<String, String>>) outputUpdate.get("messages");
+
+ if (messages != null) {
+ listener.onOutputClear(noteId, paragraphId);
+ for (int i = 0; i < messages.size(); i++) {
+ Map<String, String> m = messages.get(i);
+ InterpreterResult.Type type =
+ InterpreterResult.Type.valueOf((String) m.get("type"));
+ String outputToUpdate = (String) m.get("data");
+
+ listener.onOutputUpdated(noteId, paragraphId, i, type, outputToUpdate);
+ }
+ }
+ } else if (event.getType() == RemoteInterpreterEventType.OUTPUT_UPDATE) {
+ // on output update
+ Map<String, String> outputAppend = gson.fromJson(
+ event.getData(), new TypeToken<Map<String, Object>>() {}.getType());
+ String noteId = (String) outputAppend.get("noteId");
+ String paragraphId = (String) outputAppend.get("paragraphId");
+ int index = Integer.parseInt(outputAppend.get("index"));
+ InterpreterResult.Type type =
+ InterpreterResult.Type.valueOf((String) outputAppend.get("type"));
+ String outputToUpdate = (String) outputAppend.get("data");
+ String appId = (String) outputAppend.get("appId");
+
+ if (appId == null) {
+ listener.onOutputUpdated(noteId, paragraphId, index, type, outputToUpdate);
+ } else {
+ appListener.onOutputUpdated(noteId, paragraphId, index, appId, type, outputToUpdate);
+ }
+ } else if (event.getType() == RemoteInterpreterEventType.APP_STATUS_UPDATE) {
+ // on output update
+ Map<String, String> appStatusUpdate = gson.fromJson(
+ event.getData(), new TypeToken<Map<String, String>>() {}.getType());
+
+ String noteId = appStatusUpdate.get("noteId");
+ String paragraphId = appStatusUpdate.get("paragraphId");
+ String appId = appStatusUpdate.get("appId");
+ String status = appStatusUpdate.get("status");
+
+ appListener.onStatusChange(noteId, paragraphId, appId, status);
+ } else if (event.getType() == RemoteInterpreterEventType.REMOTE_ZEPPELIN_SERVER_RESOURCE) {
+ RemoteZeppelinServerResource reqResourceBody = RemoteZeppelinServerResource.fromJson(
+ event.getData());
+ progressRemoteZeppelinControlEvent(
+ reqResourceBody.getResourceType(), listener, reqResourceBody);
+
+ } else if (event.getType() == RemoteInterpreterEventType.META_INFOS) {
+ Map<String, String> metaInfos = gson.fromJson(event.getData(),
+ new TypeToken<Map<String, String>>() {
+ }.getType());
+ String settingId = RemoteInterpreterUtils.
+ getInterpreterSettingId(interpreterGroup.getId());
+ listener.onMetaInfosReceived(settingId, metaInfos);
+ } else if (event.getType() == RemoteInterpreterEventType.PARA_INFOS) {
+ Map<String, String> paraInfos = gson.fromJson(event.getData(),
+ new TypeToken<Map<String, String>>() {
+ }.getType());
+ String noteId = paraInfos.get("noteId");
+ String paraId = paraInfos.get("paraId");
+ String settingId = RemoteInterpreterUtils.
+ getInterpreterSettingId(interpreterGroup.getId());
+ if (noteId != null && paraId != null && settingId != null) {
+ listener.onParaInfosReceived(noteId, paraId, settingId, paraInfos);
+ }
+ }
+ logger.debug("Event from remote process {}", event.getType());
+ } catch (Exception e) {
+ logger.error("Can't handle event " + event, e);
+ }
+ }
+ try {
+ clearUnreadEvents(interpreterProcess.getClient());
+ } catch (Exception e1) {
+ logger.error("Can't get RemoteInterpreterEvent", e1);
+ }
+ if (appendFuture != null) {
+ appendFuture.cancel(true);
+ }
+ }
+
+ private void clearUnreadEvents(Client client) throws TException {
+ while (client.getEvent().getType() != RemoteInterpreterEventType.NO_OP) {}
+ }
+
+ private void progressRemoteZeppelinControlEvent(
+ RemoteZeppelinServerResource.Type resourceType,
+ RemoteInterpreterProcessListener remoteWorksEventListener,
+ RemoteZeppelinServerResource reqResourceBody) throws Exception {
+ boolean broken = false;
+ final Gson gson = new Gson();
+ final String eventOwnerKey = reqResourceBody.getOwnerKey();
+ try {
+ if (resourceType == RemoteZeppelinServerResource.Type.PARAGRAPH_RUNNERS) {
+ final List<ZeppelinServerResourceParagraphRunner> remoteRunners = new LinkedList<>();
+
+ ZeppelinServerResourceParagraphRunner reqRunnerContext =
+ new ZeppelinServerResourceParagraphRunner();
+
+ Map<String, Object> reqResourceMap = (Map<String, Object>) reqResourceBody.getData();
+ String noteId = (String) reqResourceMap.get("noteId");
+ String paragraphId = (String) reqResourceMap.get("paragraphId");
+
+ reqRunnerContext.setNoteId(noteId);
+ reqRunnerContext.setParagraphId(paragraphId);
+
+ RemoteInterpreterProcessListener.RemoteWorksEventListener callBackEvent =
+ new RemoteInterpreterProcessListener.RemoteWorksEventListener() {
+
+ @Override
+ public void onFinished(Object resultObject) {
+ if (resultObject != null && resultObject instanceof List) {
+ List<InterpreterContextRunner> runnerList =
+ (List<InterpreterContextRunner>) resultObject;
+ for (InterpreterContextRunner r : runnerList) {
+ remoteRunners.add(
+ new ZeppelinServerResourceParagraphRunner(r.getNoteId(), r.getParagraphId())
+ );
+ }
+
+ final RemoteZeppelinServerResource resResource =
+ new RemoteZeppelinServerResource();
+ resResource.setOwnerKey(eventOwnerKey);
+ resResource.setResourceType(RemoteZeppelinServerResource.Type.PARAGRAPH_RUNNERS);
+ resResource.setData(remoteRunners);
+
+ interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ client.onReceivedZeppelinResource(resResource.toJson());
+ return null;
+ }
+ }
+ );
+ }
+ }
+
+ @Override
+ public void onError() {
+ logger.info("onGetParagraphRunners onError");
+ }
+ };
+
+ remoteWorksEventListener.onGetParagraphRunners(
+ reqRunnerContext.getNoteId(), reqRunnerContext.getParagraphId(), callBackEvent);
+ }
+ } catch (Exception e) {
+ logger.error("Can't get RemoteInterpreterEvent", e);
+ waitQuietly();
+
+ }
+ }
+
+ private void sendResourcePoolResponseGetAll(final ResourceSet resourceSet) {
+ interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ List<String> resourceList = new LinkedList<>();
+ for (Resource r : resourceSet) {
+ resourceList.add(r.toJson());
+ }
+ client.resourcePoolResponseGetAll(resourceList);
+ return null;
+ }
+ }
+ );
+ }
+
+ private ResourceSet getAllResourcePoolExcept() {
+ ResourceSet resourceSet = new ResourceSet();
+ for (ManagedInterpreterGroup intpGroup : interpreterGroup.getInterpreterSetting()
+ .getInterpreterSettingManager().getAllInterpreterGroup()) {
+ if (intpGroup.getId().equals(interpreterGroup.getId())) {
+ continue;
+ }
+
+ RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
+ if (remoteInterpreterProcess == null) {
+ ResourcePool localPool = intpGroup.getResourcePool();
+ if (localPool != null) {
+ resourceSet.addAll(localPool.getAll());
+ }
+ } else if (interpreterProcess.isRunning()) {
+ List<String> resourceList = remoteInterpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<List<String>>() {
+ @Override
+ public List<String> call(Client client) throws Exception {
+ return client.resourcePoolGetAll();
+ }
+ }
+ );
+ for (String res : resourceList) {
+ resourceSet.add(Resource.fromJson(res));
+ }
+ }
+ }
+ return resourceSet;
+ }
+
+ private void sendResourceResponseGet(final ResourceId resourceId, final Object o) {
+ interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ String rid = resourceId.toJson();
+ ByteBuffer obj;
+ if (o == null) {
+ obj = ByteBuffer.allocate(0);
+ } else {
+ obj = Resource.serializeObject(o);
+ }
+ client.resourceResponseGet(rid, obj);
+ return null;
+ }
+ }
+ );
+ }
+
+ private Object getResource(final ResourceId resourceId) {
+ ManagedInterpreterGroup intpGroup = interpreterGroup.getInterpreterSetting()
+ .getInterpreterSettingManager()
+ .getInterpreterGroupById(resourceId.getResourcePoolId());
+ if (intpGroup == null) {
+ return null;
+ }
+ RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
+ ByteBuffer buffer = remoteInterpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<ByteBuffer>() {
+ @Override
+ public ByteBuffer call(Client client) throws Exception {
+ return client.resourceGet(
+ resourceId.getNoteId(),
+ resourceId.getParagraphId(),
+ resourceId.getName());
+ }
+ }
+ );
+
+ try {
+ Object o = Resource.deserializeObject(buffer);
+ return o;
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ }
+ return null;
+ }
+
+ public void sendInvokeMethodResult(final InvokeResourceMethodEventMessage message,
+ final Object o) {
+ interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<Void>() {
+ @Override
+ public Void call(Client client) throws Exception {
+ String invokeMessage = message.toJson();
+ ByteBuffer obj;
+ if (o == null) {
+ obj = ByteBuffer.allocate(0);
+ } else {
+ obj = Resource.serializeObject(o);
+ }
+ client.resourceResponseInvokeMethod(invokeMessage, obj);
+ return null;
+ }
+ }
+ );
+ }
+
+ private Object invokeResourceMethod(final InvokeResourceMethodEventMessage message) {
+ final ResourceId resourceId = message.resourceId;
+ ManagedInterpreterGroup intpGroup = interpreterGroup.getInterpreterSetting()
+ .getInterpreterSettingManager().getInterpreterGroupById(resourceId.getResourcePoolId());
+ if (intpGroup == null) {
+ return null;
+ }
+
+ RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
+ if (remoteInterpreterProcess == null) {
+ ResourcePool localPool = intpGroup.getResourcePool();
+ if (localPool != null) {
+ Resource res = localPool.get(resourceId.getName());
+ if (res != null) {
+ try {
+ return res.invokeMethod(
+ message.methodName,
+ message.getParamTypes(),
+ message.params,
+ message.returnResourceName);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ return null;
+ }
+ } else {
+ // object is null. can't invoke any method
+ logger.error("Can't invoke method {} on null object", message.methodName);
+ return null;
+ }
+ } else {
+ logger.error("no resource pool");
+ return null;
+ }
+ } else if (interpreterProcess.isRunning()) {
+ ByteBuffer res = interpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<ByteBuffer>() {
+ @Override
+ public ByteBuffer call(Client client) throws Exception {
+ return client.resourceInvokeMethod(
+ resourceId.getNoteId(),
+ resourceId.getParagraphId(),
+ resourceId.getName(),
+ message.toJson());
+ }
+ }
+ );
+
+ try {
+ return Resource.deserializeObject(res);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ }
+ return null;
+ }
+ return null;
+ }
+
+ private void waitQuietly() {
+ try {
+ synchronized (this) {
+ wait(1000);
+ }
+ } catch (InterruptedException ignored) {
+ logger.info("Error in RemoteInterpreterEventPoller while waitQuietly : ", ignored);
+ }
+ }
+
+ public void shutdown() {
+ shutdown = true;
+ synchronized (this) {
+ notify();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
index 1fb9b90..19356fb 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
@@ -21,6 +21,7 @@ import org.apache.commons.exec.*;
import org.apache.commons.exec.environment.EnvironmentUtils;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -97,6 +98,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
// start server process
try {
port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces();
+ logger.info("Choose port {} for RemoteInterpreterProcess", port);
} catch (IOException e1) {
throw new InterpreterException(e1);
}
@@ -172,6 +174,17 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
public void stop() {
if (isRunning()) {
logger.info("kill interpreter process");
+ try {
+ callRemoteFunction(new RemoteFunction<Void>() {
+ @Override
+ public Void call(RemoteInterpreterService.Client client) throws Exception {
+ client.shutdown();
+ return null;
+ }
+ });
+ } catch (Exception e) {
+ logger.warn("ignore the exception when shutting down");
+ }
watchdog.destroyProcess();
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
new file mode 100644
index 0000000..d34c538
--- /dev/null
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.interpreter.remote;
+
+import com.google.gson.Gson;
+import org.apache.commons.pool2.impl.GenericObjectPool;
+import org.apache.thrift.TException;
+import org.apache.zeppelin.helium.ApplicationEventListener;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Abstract class for interpreter process
+ */
+public abstract class RemoteInterpreterProcess {
+ private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterProcess.class);
+
+ private GenericObjectPool<Client> clientPool;
+ private final RemoteInterpreterEventPoller remoteInterpreterEventPoller;
+ private final InterpreterContextRunnerPool interpreterContextRunnerPool;
+ private int connectTimeout;
+
+ public RemoteInterpreterProcess(
+ int connectTimeout,
+ RemoteInterpreterProcessListener listener,
+ ApplicationEventListener appListener) {
+ this(new RemoteInterpreterEventPoller(listener, appListener),
+ connectTimeout);
+ this.remoteInterpreterEventPoller.setInterpreterProcess(this);
+ }
+
+ RemoteInterpreterProcess(RemoteInterpreterEventPoller remoteInterpreterEventPoller,
+ int connectTimeout) {
+ this.interpreterContextRunnerPool = new InterpreterContextRunnerPool();
+ this.remoteInterpreterEventPoller = remoteInterpreterEventPoller;
+ this.connectTimeout = connectTimeout;
+ }
+
+ public RemoteInterpreterEventPoller getRemoteInterpreterEventPoller() {
+ return remoteInterpreterEventPoller;
+ }
+
+ public abstract String getHost();
+ public abstract int getPort();
+ public abstract void start(String userName, Boolean isUserImpersonate);
+ public abstract void stop();
+ public abstract boolean isRunning();
+
+ public int getConnectTimeout() {
+ return connectTimeout;
+ }
+
+ public synchronized Client getClient() throws Exception {
+ if (clientPool == null || clientPool.isClosed()) {
+ clientPool = new GenericObjectPool<>(new ClientFactory(getHost(), getPort()));
+ }
+ return clientPool.borrowObject();
+ }
+
+ private void releaseClient(Client client) {
+ releaseClient(client, false);
+ }
+
+ private void releaseClient(Client client, boolean broken) {
+ if (broken) {
+ releaseBrokenClient(client);
+ } else {
+ try {
+ clientPool.returnObject(client);
+ } catch (Exception e) {
+ logger.warn("exception occurred during releasing thrift client", e);
+ }
+ }
+ }
+
+ private void releaseBrokenClient(Client client) {
+ try {
+ clientPool.invalidateObject(client);
+ } catch (Exception e) {
+ logger.warn("exception occurred during releasing thrift client", e);
+ }
+ }
+
+ /**
+ * Called when angular object is updated in client side to propagate
+ * change to the remote process
+ * @param name
+ * @param o
+ */
+ public void updateRemoteAngularObject(String name, String noteId, String paragraphId, Object o) {
+ Client client = null;
+ try {
+ client = getClient();
+ } catch (NullPointerException e) {
+ // remote process not started
+ logger.info("NullPointerException in RemoteInterpreterProcess while " +
+ "updateRemoteAngularObject getClient, remote process not started", e);
+ return;
+ } catch (Exception e) {
+ logger.error("Can't update angular object", e);
+ }
+
+ boolean broken = false;
+ try {
+ Gson gson = new Gson();
+ client.angularObjectUpdate(name, noteId, paragraphId, gson.toJson(o));
+ } catch (TException e) {
+ broken = true;
+ logger.error("Can't update angular object", e);
+ } catch (NullPointerException e) {
+ logger.error("Remote interpreter process not started", e);
+ return;
+ } finally {
+ if (client != null) {
+ releaseClient(client, broken);
+ }
+ }
+ }
+
+ public InterpreterContextRunnerPool getInterpreterContextRunnerPool() {
+ return interpreterContextRunnerPool;
+ }
+
+ public <T> T callRemoteFunction(RemoteFunction<T> func) {
+ Client client = null;
+ boolean broken = false;
+ try {
+ client = getClient();
+ if (client != null) {
+ return func.call(client);
+ }
+ } catch (TException e) {
+ broken = true;
+ throw new InterpreterException(e);
+ } catch (Exception e1) {
+ throw new InterpreterException(e1);
+ } finally {
+ if (client != null) {
+ releaseClient(client, broken);
+ }
+ }
+ return null;
+ }
+
+ /**
+ *
+ * @param <T>
+ */
+ public interface RemoteFunction<T> {
+ T call(Client client) throws Exception;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessListener.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessListener.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessListener.java
new file mode 100644
index 0000000..8b23bf2
--- /dev/null
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessListener.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.interpreter.remote;
+
+import org.apache.zeppelin.interpreter.InterpreterResult;
+
+import java.util.Map;
+
+/**
+ * Event from remoteInterpreterProcess
+ */
+public interface RemoteInterpreterProcessListener {
+ public void onOutputAppend(String noteId, String paragraphId, int index, String output);
+ public void onOutputUpdated(
+ String noteId, String paragraphId, int index, InterpreterResult.Type type, String output);
+ public void onOutputClear(String noteId, String paragraphId);
+ public void onMetaInfosReceived(String settingId, Map<String, String> metaInfos);
+ public void onRemoteRunParagraph(String noteId, String ParagraphID) throws Exception;
+ public void onGetParagraphRunners(
+ String noteId, String paragraphId, RemoteWorksEventListener callback);
+
+ /**
+ * Remote works for Interpreter callback listener
+ */
+ public interface RemoteWorksEventListener {
+ public void onFinished(Object resultObject);
+ public void onError();
+ }
+ public void onParaInfosReceived(String noteId, String paragraphId,
+ String interpreterSettingId, Map<String, String> metaInfos);
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java
index 1505db9..bc71d89 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java
@@ -17,7 +17,6 @@
package org.apache.zeppelin.notebook;
import org.apache.zeppelin.helium.HeliumPackage;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
/**
* Current state of application
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
index 198e278..5a42f37 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
@@ -20,7 +20,6 @@ package org.apache.zeppelin.notebook;
import static java.lang.String.format;
import java.io.IOException;
-import java.io.Serializable;
import java.util.*;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
@@ -28,8 +27,6 @@ import java.util.concurrent.TimeUnit;
import com.google.common.annotations.VisibleForTesting;
import com.google.gson.GsonBuilder;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.common.JsonSerializable;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
@@ -41,7 +38,6 @@ import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.utility.IdHashes;
-import org.apache.zeppelin.resource.ResourcePoolUtils;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.search.SearchService;
@@ -126,11 +122,6 @@ public class Note implements ParagraphJobListener, JsonSerializable {
id = IdHashes.generateId();
}
- private String getDefaultInterpreterName() {
- InterpreterSetting setting = interpreterSettingManager.getDefaultInterpreterSetting(getId());
- return null != setting ? setting.getName() : StringUtils.EMPTY;
- }
-
public boolean isPersonalizedMode() {
Object v = getConfig().get("personalizedMode");
return null != v && "true".equals(v);
@@ -385,7 +376,7 @@ public class Note implements ParagraphJobListener, JsonSerializable {
*/
public Paragraph removeParagraph(String user, String paragraphId) {
removeAllAngularObjectInParagraph(user, paragraphId);
- ResourcePoolUtils.removeResourcesBelongsToParagraph(getId(), paragraphId);
+ interpreterSettingManager.removeResourcesBelongsToParagraph(getId(), paragraphId);
synchronized (paragraphs) {
Iterator<Paragraph> i = paragraphs.iterator();
while (i.hasNext()) {
@@ -690,7 +681,7 @@ public class Note implements ParagraphJobListener, JsonSerializable {
}
for (InterpreterSetting setting : settings) {
- InterpreterGroup intpGroup = setting.getInterpreterGroup(user, id);
+ InterpreterGroup intpGroup = setting.getOrCreateInterpreterGroup(user, id);
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
angularObjects.put(intpGroup.getId(), registry.getAllWithGlobal(id));
}
@@ -705,7 +696,7 @@ public class Note implements ParagraphJobListener, JsonSerializable {
}
for (InterpreterSetting setting : settings) {
- InterpreterGroup intpGroup = setting.getInterpreterGroup(user, id);
+ InterpreterGroup intpGroup = setting.getOrCreateInterpreterGroup(user, id);
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
if (registry instanceof RemoteAngularObjectRegistry) {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
index a0c1dff..4652fcd 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
@@ -18,7 +18,6 @@
package org.apache.zeppelin.notebook;
import java.io.IOException;
-import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -35,10 +34,8 @@ import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Sets;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.stream.JsonReader;
import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.quartz.CronScheduleBuilder;
import org.quartz.CronTrigger;
import org.quartz.JobBuilder;
@@ -56,11 +53,9 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.repo.NotebookRepo.Revision;
import org.apache.zeppelin.notebook.repo.NotebookRepoSync;
-import org.apache.zeppelin.resource.ResourcePoolUtils;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.search.SearchService;
@@ -140,7 +135,7 @@ public class Notebook implements NoteEventListener {
Preconditions.checkNotNull(subject, "AuthenticationInfo should not be null");
Note note;
if (conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING)) {
- note = createNote(interpreterSettingManager.getDefaultInterpreterSettingList(), subject);
+ note = createNote(interpreterSettingManager.getInterpreterSettingIds(), subject);
} else {
note = createNote(null, subject);
}
@@ -270,8 +265,8 @@ public class Notebook implements NoteEventListener {
}
}
- interpreterSettingManager.setInterpreters(user, note.getId(), interpreterSettingIds);
- // comment out while note.getNoteReplLoader().setInterpreters(...) do the same
+ interpreterSettingManager.setInterpreterBinding(user, note.getId(), interpreterSettingIds);
+ // comment out while note.getNoteReplLoader().setInterpreterBinding(...) do the same
// replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds);
}
}
@@ -279,7 +274,7 @@ public class Notebook implements NoteEventListener {
List<String> getBindedInterpreterSettingsIds(String id) {
Note note = getNote(id);
if (note != null) {
- return interpreterSettingManager.getInterpreters(note.getId());
+ return interpreterSettingManager.getInterpreterBinding(note.getId());
} else {
return new LinkedList<>();
}
@@ -313,9 +308,10 @@ public class Notebook implements NoteEventListener {
}
public void moveNoteToTrash(String noteId) {
- for (InterpreterSetting interpreterSetting : interpreterSettingManager
- .getInterpreterSettings(noteId)) {
- interpreterSettingManager.removeInterpretersForNote(interpreterSetting, "", noteId);
+ try {
+ interpreterSettingManager.setInterpreterBinding("", noteId, new ArrayList<String>());
+ } catch (IOException e) {
+ e.printStackTrace();
}
}
@@ -339,7 +335,7 @@ public class Notebook implements NoteEventListener {
// remove from all interpreter instance's angular object registry
for (InterpreterSetting settings : interpreterSettingManager.get()) {
AngularObjectRegistry registry =
- settings.getInterpreterGroup(subject.getUser(), id).getAngularObjectRegistry();
+ settings.getOrCreateInterpreterGroup(subject.getUser(), id).getAngularObjectRegistry();
if (registry instanceof RemoteAngularObjectRegistry) {
// remove paragraph scope object
for (Paragraph p : note.getParagraphs()) {
@@ -374,7 +370,7 @@ public class Notebook implements NoteEventListener {
}
}
- ResourcePoolUtils.removeResourcesBelongsToNote(id);
+ interpreterSettingManager.removeResourcesBelongsToNote(id);
fireNoteRemoveEvent(note);
@@ -521,7 +517,8 @@ public class Notebook implements NoteEventListener {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
List<InterpreterSetting> settings = interpreterSettingManager.get();
for (InterpreterSetting setting : settings) {
- InterpreterGroup intpGroup = setting.getInterpreterGroup(subject.getUser(), note.getId());
+ InterpreterGroup intpGroup = setting.getOrCreateInterpreterGroup(subject.getUser(),
+ note.getId());
if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
String noteId = snapshot.getAngularObject().getNoteId();
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
index 37138e6..161dc30 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
@@ -28,8 +28,6 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.common.JsonSerializable;
@@ -93,10 +91,10 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
// since zeppelin-0.7.0, zeppelin stores multiple results of the paragraph
// see ZEPPELIN-212
- Object results;
+ volatile Object results;
// For backward compatibility of note.json format after ZEPPELIN-212
- Object result;
+ volatile Object result;
private Map<String, ParagraphRuntimeInfo> runtimeInfos;
/**
@@ -157,7 +155,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
}
@Override
- public void setResult(Object results) {
+ public synchronized void setResult(Object results) {
this.results = results;
}
@@ -354,7 +352,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
}
@Override
- public Object getReturn() {
+ public synchronized Object getReturn() {
return results;
}
@@ -401,6 +399,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
logger.error("Can not find interpreter name " + repl);
throw new RuntimeException("Can not find interpreter for " + getRequiredReplName());
}
+ //TODO(zjffdu) check interpreter setting status in interpreter setting itself
InterpreterSetting intp = getInterpreterSettingById(repl.getInterpreterGroup().getId());
while (intp.getStatus().equals(
org.apache.zeppelin.interpreter.InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES)) {
@@ -560,8 +559,10 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) {
InterpreterSetting intpGroup =
interpreterSettingManager.getInterpreterSettings(note.getId()).get(0);
- registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
- resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
+ registry = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
+ .getAngularObjectRegistry();
+ resourcePool = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
+ .getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<>();
@@ -591,8 +592,10 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) {
InterpreterSetting intpGroup =
interpreterSettingManager.getInterpreterSettings(note.getId()).get(0);
- registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
- resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
+ registry = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
+ .getAngularObjectRegistry();
+ resourcePool = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
+ .getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<>();
[8/9] zeppelin git commit: [ZEPPELIN-2627] Interpreter Component
Refactoring
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java
index d0025d8..191902a 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java
@@ -41,6 +41,7 @@ public abstract class Job {
/**
* Job status.
*
+ * UNKNOWN - Job is not found in remote
* READY - Job is not running, ready to run.
* PENDING - Job is submitted to scheduler. but not running yet
* RUNNING - Job is running.
@@ -48,8 +49,8 @@ public abstract class Job {
* ERROR - Job finished run. with error
* ABORT - Job finished by abort
*/
- public static enum Status {
- READY, PENDING, RUNNING, FINISHED, ERROR, ABORT;
+ public enum Status {
+ UNKNOWN, READY, PENDING, RUNNING, FINISHED, ERROR, ABORT;
public boolean isReady() {
return this == READY;
@@ -70,14 +71,14 @@ public abstract class Job {
Date dateCreated;
Date dateStarted;
Date dateFinished;
- Status status;
+ volatile Status status;
static Logger LOGGER = LoggerFactory.getLogger(Job.class);
transient boolean aborted = false;
- private String errorMessage;
- private transient Throwable exception;
+ private volatile String errorMessage;
+ private transient volatile Throwable exception;
private transient JobListener listener;
private long progressUpdateIntervalMs;
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java
deleted file mode 100644
index f9ddc4e..0000000
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java
+++ /dev/null
@@ -1,426 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.scheduler;
-
-import org.apache.thrift.TException;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.InterpreterResult.Code;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
-import org.apache.zeppelin.scheduler.Job.Status;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-
-/**
- * RemoteScheduler runs in ZeppelinServer and proxies Scheduler running on RemoteInterpreter
- */
-public class RemoteScheduler implements Scheduler {
- Logger logger = LoggerFactory.getLogger(RemoteScheduler.class);
-
- List<Job> queue = new LinkedList<>();
- List<Job> running = new LinkedList<>();
- private ExecutorService executor;
- private SchedulerListener listener;
- boolean terminate = false;
- private String name;
- private int maxConcurrency;
- private final String noteId;
- private RemoteInterpreterProcess interpreterProcess;
-
- public RemoteScheduler(String name, ExecutorService executor, String noteId,
- RemoteInterpreterProcess interpreterProcess, SchedulerListener listener,
- int maxConcurrency) {
- this.name = name;
- this.executor = executor;
- this.listener = listener;
- this.noteId = noteId;
- this.interpreterProcess = interpreterProcess;
- this.maxConcurrency = maxConcurrency;
- }
-
- @Override
- public void run() {
- while (terminate == false) {
- Job job = null;
-
- synchronized (queue) {
- if (running.size() >= maxConcurrency || queue.isEmpty() == true) {
- try {
- queue.wait(500);
- } catch (InterruptedException e) {
- logger.error("Exception in RemoteScheduler while run queue.wait", e);
- }
- continue;
- }
-
- job = queue.remove(0);
- running.add(job);
- }
-
- // run
- Scheduler scheduler = this;
- JobRunner jobRunner = new JobRunner(scheduler, job);
- executor.execute(jobRunner);
-
- // wait until it is submitted to the remote
- while (!jobRunner.isJobSubmittedInRemote()) {
- synchronized (queue) {
- try {
- queue.wait(500);
- } catch (InterruptedException e) {
- logger.error("Exception in RemoteScheduler while jobRunner.isJobSubmittedInRemote " +
- "queue.wait", e);
- }
- }
- }
- }
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public Collection<Job> getJobsWaiting() {
- List<Job> ret = new LinkedList<>();
- synchronized (queue) {
- for (Job job : queue) {
- ret.add(job);
- }
- }
- return ret;
- }
-
- @Override
- public Job removeFromWaitingQueue(String jobId) {
- synchronized (queue) {
- Iterator<Job> it = queue.iterator();
- while (it.hasNext()) {
- Job job = it.next();
- if (job.getId().equals(jobId)) {
- it.remove();
- return job;
- }
- }
- }
- return null;
- }
-
- @Override
- public Collection<Job> getJobsRunning() {
- List<Job> ret = new LinkedList<>();
- synchronized (queue) {
- for (Job job : running) {
- ret.add(job);
- }
- }
- return ret;
- }
-
- @Override
- public void submit(Job job) {
- if (terminate) {
- throw new RuntimeException("Scheduler already terminated");
- }
- job.setStatus(Status.PENDING);
-
- synchronized (queue) {
- queue.add(job);
- queue.notify();
- }
- }
-
- public void setMaxConcurrency(int maxConcurrency) {
- this.maxConcurrency = maxConcurrency;
- synchronized (queue) {
- queue.notify();
- }
- }
-
- /**
- * Role of the class is get status info from remote process from PENDING to
- * RUNNING status.
- */
- private class JobStatusPoller extends Thread {
- private long initialPeriodMsec;
- private long initialPeriodCheckIntervalMsec;
- private long checkIntervalMsec;
- private boolean terminate;
- private JobListener listener;
- private Job job;
- Status lastStatus;
-
- public JobStatusPoller(long initialPeriodMsec,
- long initialPeriodCheckIntervalMsec, long checkIntervalMsec, Job job,
- JobListener listener) {
- this.initialPeriodMsec = initialPeriodMsec;
- this.initialPeriodCheckIntervalMsec = initialPeriodCheckIntervalMsec;
- this.checkIntervalMsec = checkIntervalMsec;
- this.job = job;
- this.listener = listener;
- this.terminate = false;
- }
-
- @Override
- public void run() {
- long started = System.currentTimeMillis();
- while (terminate == false) {
- long current = System.currentTimeMillis();
- long interval;
- if (current - started < initialPeriodMsec) {
- interval = initialPeriodCheckIntervalMsec;
- } else {
- interval = checkIntervalMsec;
- }
-
- synchronized (this) {
- try {
- this.wait(interval);
- } catch (InterruptedException e) {
- logger.error("Exception in RemoteScheduler while run this.wait", e);
- }
- }
-
- if (terminate) {
- // terminated by shutdown
- break;
- }
-
- Status newStatus = getStatus();
- if (newStatus == null) { // unknown
- continue;
- }
-
- if (newStatus != Status.READY && newStatus != Status.PENDING) {
- // we don't need more
- break;
- }
- }
- terminate = true;
- }
-
- public void shutdown() {
- terminate = true;
- synchronized (this) {
- this.notify();
- }
- }
-
-
- private Status getLastStatus() {
- if (terminate == true) {
- if (lastStatus != Status.FINISHED &&
- lastStatus != Status.ERROR &&
- lastStatus != Status.ABORT) {
- return Status.FINISHED;
- } else {
- return (lastStatus == null) ? Status.FINISHED : lastStatus;
- }
- } else {
- return (lastStatus == null) ? Status.FINISHED : lastStatus;
- }
- }
-
- public synchronized Job.Status getStatus() {
- if (interpreterProcess.referenceCount() <= 0) {
- return getLastStatus();
- }
-
- Client client;
- try {
- client = interpreterProcess.getClient();
- } catch (Exception e) {
- logger.error("Can't get status information", e);
- lastStatus = Status.ERROR;
- return Status.ERROR;
- }
-
- boolean broken = false;
- try {
- String statusStr = client.getStatus(noteId, job.getId());
- if ("Unknown".equals(statusStr)) {
- // not found this job in the remote schedulers.
- // maybe not submitted, maybe already finished
- //Status status = getLastStatus();
- listener.afterStatusChange(job, null, null);
- return job.getStatus();
- }
- Status status = Status.valueOf(statusStr);
- lastStatus = status;
- listener.afterStatusChange(job, null, status);
- return status;
- } catch (TException e) {
- broken = true;
- logger.error("Can't get status information", e);
- lastStatus = Status.ERROR;
- return Status.ERROR;
- } catch (Exception e) {
- logger.error("Unknown status", e);
- lastStatus = Status.ERROR;
- return Status.ERROR;
- } finally {
- interpreterProcess.releaseClient(client, broken);
- }
- }
- }
-
- private class JobRunner implements Runnable, JobListener {
- private Scheduler scheduler;
- private Job job;
- private boolean jobExecuted;
- boolean jobSubmittedRemotely;
-
- public JobRunner(Scheduler scheduler, Job job) {
- this.scheduler = scheduler;
- this.job = job;
- jobExecuted = false;
- jobSubmittedRemotely = false;
- }
-
- public boolean isJobSubmittedInRemote() {
- return jobSubmittedRemotely;
- }
-
- @Override
- public void run() {
- if (job.isAborted()) {
- synchronized (queue) {
- job.setStatus(Status.ABORT);
- job.aborted = false;
-
- running.remove(job);
- queue.notify();
- }
- jobSubmittedRemotely = true;
-
- return;
- }
-
- JobStatusPoller jobStatusPoller = new JobStatusPoller(1500, 100, 500,
- job, this);
- jobStatusPoller.start();
-
- if (listener != null) {
- listener.jobStarted(scheduler, job);
- }
- job.run();
-
- jobExecuted = true;
- jobSubmittedRemotely = true;
-
- jobStatusPoller.shutdown();
- try {
- jobStatusPoller.join();
- } catch (InterruptedException e) {
- logger.error("JobStatusPoller interrupted", e);
- }
-
- // set job status based on result.
- Status lastStatus = jobStatusPoller.getStatus();
- Object jobResult = job.getReturn();
- if (jobResult != null && jobResult instanceof InterpreterResult) {
- if (((InterpreterResult) jobResult).code() == Code.ERROR) {
- lastStatus = Status.ERROR;
- }
- }
- if (job.getException() != null) {
- lastStatus = Status.ERROR;
- }
-
- synchronized (queue) {
- job.setStatus(lastStatus);
-
- if (listener != null) {
- listener.jobFinished(scheduler, job);
- }
-
- // reset aborted flag to allow retry
- job.aborted = false;
-
- running.remove(job);
- queue.notify();
- }
- }
-
- @Override
- public void onProgressUpdate(Job job, int progress) {
- }
-
- @Override
- public void beforeStatusChange(Job job, Status before, Status after) {
- }
-
- @Override
- public void afterStatusChange(Job job, Status before, Status after) {
- if (after == null) { // unknown. maybe before sumitted remotely, maybe already finished.
- if (jobExecuted) {
- jobSubmittedRemotely = true;
- Object jobResult = job.getReturn();
- if (job.isAborted()) {
- job.setStatus(Status.ABORT);
- } else if (job.getException() != null) {
- job.setStatus(Status.ERROR);
- } else if (jobResult != null && jobResult instanceof InterpreterResult
- && ((InterpreterResult) jobResult).code() == Code.ERROR) {
- job.setStatus(Status.ERROR);
- } else {
- job.setStatus(Status.FINISHED);
- }
- }
- return;
- }
-
-
- // Update remoteStatus
- if (jobExecuted == false) {
- if (after == Status.FINISHED || after == Status.ABORT
- || after == Status.ERROR) {
- // it can be status of last run.
- // so not updating the remoteStatus
- return;
- } else if (after == Status.RUNNING) {
- jobSubmittedRemotely = true;
- }
- } else {
- jobSubmittedRemotely = true;
- }
-
- // status polled by status poller
- if (job.getStatus() != after) {
- job.setStatus(after);
- }
- }
- }
-
- @Override
- public void stop() {
- terminate = true;
- synchronized (queue) {
- queue.notify();
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java
index af52dec..b629ef7 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java
@@ -17,24 +17,21 @@
package org.apache.zeppelin.scheduler;
-import java.util.Collection;
import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * TODO(moon) : add description.
+ * Factory class for creating schedulers
+ *
*/
public class SchedulerFactory implements SchedulerListener {
private static final Logger logger = LoggerFactory.getLogger(SchedulerFactory.class);
- ExecutorService executor;
- Map<String, Scheduler> schedulers = new LinkedHashMap<>();
+ protected ExecutorService executor;
+ protected Map<String, Scheduler> schedulers = new LinkedHashMap<>();
private static SchedulerFactory singleton;
private static Long singletonLock = new Long(0);
@@ -54,17 +51,17 @@ public class SchedulerFactory implements SchedulerListener {
return singleton;
}
- public SchedulerFactory() throws Exception {
- executor = ExecutorFactory.singleton().createOrGet("schedulerFactory", 100);
+ SchedulerFactory() throws Exception {
+ executor = ExecutorFactory.singleton().createOrGet("SchedulerFactory", 100);
}
public void destroy() {
- ExecutorFactory.singleton().shutdown("schedulerFactory");
+ ExecutorFactory.singleton().shutdown("SchedulerFactory");
}
public Scheduler createOrGetFIFOScheduler(String name) {
synchronized (schedulers) {
- if (schedulers.containsKey(name) == false) {
+ if (!schedulers.containsKey(name)) {
Scheduler s = new FIFOScheduler(name, executor, this);
schedulers.put(name, s);
executor.execute(s);
@@ -75,7 +72,7 @@ public class SchedulerFactory implements SchedulerListener {
public Scheduler createOrGetParallelScheduler(String name, int maxConcurrency) {
synchronized (schedulers) {
- if (schedulers.containsKey(name) == false) {
+ if (!schedulers.containsKey(name)) {
Scheduler s = new ParallelScheduler(name, executor, this, maxConcurrency);
schedulers.put(name, s);
executor.execute(s);
@@ -84,60 +81,38 @@ public class SchedulerFactory implements SchedulerListener {
}
}
- public Scheduler createOrGetRemoteScheduler(
- String name,
- String noteId,
- RemoteInterpreterProcess interpreterProcess,
- int maxConcurrency) {
-
+ public Scheduler createOrGetScheduler(Scheduler scheduler) {
synchronized (schedulers) {
- if (schedulers.containsKey(name) == false) {
- Scheduler s = new RemoteScheduler(
- name,
- executor,
- noteId,
- interpreterProcess,
- this,
- maxConcurrency);
- schedulers.put(name, s);
- executor.execute(s);
+ if (!schedulers.containsKey(scheduler.getName())) {
+ schedulers.put(scheduler.getName(), scheduler);
+ executor.execute(scheduler);
}
- return schedulers.get(name);
+ return schedulers.get(scheduler.getName());
}
}
- public Scheduler removeScheduler(String name) {
+ public void removeScheduler(String name) {
synchronized (schedulers) {
Scheduler s = schedulers.remove(name);
if (s != null) {
s.stop();
}
}
- return null;
}
- public Collection<Scheduler> listScheduler(String name) {
- List<Scheduler> s = new LinkedList<>();
- synchronized (schedulers) {
- for (Scheduler ss : schedulers.values()) {
- s.add(ss);
- }
- }
- return s;
+ public ExecutorService getExecutor() {
+ return executor;
}
@Override
public void jobStarted(Scheduler scheduler, Job job) {
- logger.info("Job " + job.getJobName() + " started by scheduler " + scheduler.getName());
+ logger.info("Job " + job.getId() + " started by scheduler " + scheduler.getName());
}
@Override
public void jobFinished(Scheduler scheduler, Job job) {
- logger.info("Job " + job.getJobName() + " finished by scheduler " + scheduler.getName());
+ logger.info("Job " + job.getId() + " finished by scheduler " + scheduler.getName());
}
-
-
-
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/TableDataProxy.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/TableDataProxy.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/TableDataProxy.java
index 8673476..1926528 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/TableDataProxy.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/TableDataProxy.java
@@ -17,7 +17,6 @@
package org.apache.zeppelin.tabledata;
import org.apache.zeppelin.resource.Resource;
-import org.apache.zeppelin.resource.ResourcePoolUtils;
import java.util.Iterator;
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java
new file mode 100644
index 0000000..14c03a1
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.util;
+
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * Generate Tiny ID.
+ */
+public class IdHashes {
+ private static final char[] DICTIONARY = new char[] {'1', '2', '3', '4', '5', '6', '7', '8', '9',
+ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
+ 'W', 'X', 'Y', 'Z'};
+
+ /**
+ * encodes the given string into the base of the dictionary provided in the constructor.
+ *
+ * @param value the number to encode.
+ * @return the encoded string.
+ */
+ private static String encode(Long value) {
+
+ List<Character> result = new ArrayList<>();
+ BigInteger base = new BigInteger("" + DICTIONARY.length);
+ int exponent = 1;
+ BigInteger remaining = new BigInteger(value.toString());
+ while (true) {
+ BigInteger a = base.pow(exponent); // 16^1 = 16
+ BigInteger b = remaining.mod(a); // 119 % 16 = 7 | 112 % 256 = 112
+ BigInteger c = base.pow(exponent - 1);
+ BigInteger d = b.divide(c);
+
+ // if d > dictionary.length, we have a problem. but BigInteger doesnt have
+ // a greater than method :-( hope for the best. theoretically, d is always
+ // an index of the dictionary!
+ result.add(DICTIONARY[d.intValue()]);
+ remaining = remaining.subtract(b); // 119 - 7 = 112 | 112 - 112 = 0
+
+ // finished?
+ if (remaining.equals(BigInteger.ZERO)) {
+ break;
+ }
+
+ exponent++;
+ }
+
+ // need to reverse it, since the start of the list contains the least significant values
+ StringBuffer sb = new StringBuffer();
+ for (int i = result.size() - 1; i >= 0; i--) {
+ sb.append(result.get(i));
+ }
+ return sb.toString();
+ }
+
+ public static String generateId() {
+ return encode(System.currentTimeMillis() + new Random().nextInt());
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/Util.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/Util.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/Util.java
new file mode 100644
index 0000000..6153f49
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/Util.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.util;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.io.IOException;
+import java.util.Properties;
+
+/**
+ * TODO(moon) : add description.
+ */
+public class Util {
+ private static final String PROJECT_PROPERTIES_VERSION_KEY = "version";
+ private static final String GIT_PROPERTIES_COMMIT_ID_KEY = "git.commit.id.abbrev";
+ private static final String GIT_PROPERTIES_COMMIT_TS_KEY = "git.commit.time";
+
+ private static Properties projectProperties;
+ private static Properties gitProperties;
+
+ static {
+ projectProperties = new Properties();
+ gitProperties = new Properties();
+ try {
+ projectProperties.load(Util.class.getResourceAsStream("/project.properties"));
+ gitProperties.load(Util.class.getResourceAsStream("/git.properties"));
+ } catch (IOException e) {
+ //Fail to read project.properties
+ }
+ }
+
+ /**
+ * Get Zeppelin version
+ *
+ * @return Current Zeppelin version
+ */
+ public static String getVersion() {
+ return StringUtils.defaultIfEmpty(projectProperties.getProperty(PROJECT_PROPERTIES_VERSION_KEY),
+ StringUtils.EMPTY);
+ }
+
+ /**
+ * Get Zeppelin Git latest commit id
+ *
+ * @return Latest Zeppelin commit id
+ */
+ public static String getGitCommitId() {
+ return StringUtils.defaultIfEmpty(gitProperties.getProperty(GIT_PROPERTIES_COMMIT_ID_KEY),
+ StringUtils.EMPTY);
+ }
+
+ /**
+ * Get Zeppelin Git latest commit timestamp
+ *
+ * @return Latest Zeppelin commit timestamp
+ */
+ public static String getGitTimestamp() {
+ return StringUtils.defaultIfEmpty(gitProperties.getProperty(GIT_PROPERTIES_COMMIT_TS_KEY),
+ StringUtils.EMPTY);
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DummyInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DummyInterpreter.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DummyInterpreter.java
deleted file mode 100644
index a7a6eb9..0000000
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DummyInterpreter.java
+++ /dev/null
@@ -1,43 +0,0 @@
-package org.apache.zeppelin.interpreter;
-
-import java.util.Properties;
-
-/**
- *
- */
-public class DummyInterpreter extends Interpreter {
-
- public DummyInterpreter(Properties property) {
- super(property);
- }
-
- @Override
- public void open() {
-
- }
-
- @Override
- public void close() {
-
- }
-
- @Override
- public InterpreterResult interpret(String st, InterpreterContext context) {
- return null;
- }
-
- @Override
- public void cancel(InterpreterContext context) {
-
- }
-
- @Override
- public FormType getFormType() {
- return null;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcherTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcherTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcherTest.java
index e376809..f3a30fb 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcherTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcherTest.java
@@ -21,6 +21,7 @@ import static org.junit.Assert.*;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
import org.junit.After;
import org.junit.Before;
@@ -29,7 +30,7 @@ import org.junit.Test;
public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChangeListener {
private File tmpDir;
private File fileChanged;
- private int numChanged;
+ private AtomicInteger numChanged;
private InterpreterOutputChangeWatcher watcher;
@Before
@@ -40,7 +41,7 @@ public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChan
tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
tmpDir.mkdirs();
fileChanged = null;
- numChanged = 0;
+ numChanged = new AtomicInteger(0);
}
@After
@@ -66,7 +67,7 @@ public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChan
@Test
public void test() throws IOException, InterruptedException {
assertNull(fileChanged);
- assertEquals(0, numChanged);
+ assertEquals(0, numChanged.get());
Thread.sleep(1000);
// create new file
@@ -92,14 +93,14 @@ public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChan
}
assertNotNull(fileChanged);
- assertEquals(1, numChanged);
+ assertEquals(1, numChanged.get());
}
@Override
public void fileChanged(File file) {
fileChanged = file;
- numChanged++;
+ numChanged.incrementAndGet();
synchronized(this) {
notify();
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java
index 305268c..31c9225 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java
@@ -24,6 +24,7 @@ import org.junit.Test;
import static org.junit.Assert.assertEquals;
+//TODO(zjffdu) add more test for Interpreter which is a very important class
public class InterpreterTest {
@Test
@@ -85,4 +86,41 @@ public class InterpreterTest {
);
}
+ public static class DummyInterpreter extends Interpreter {
+
+ public DummyInterpreter(Properties property) {
+ super(property);
+ }
+
+ @Override
+ public void open() {
+
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ return null;
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+
+ }
+
+ @Override
+ public FormType getFormType() {
+ return null;
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java
new file mode 100644
index 0000000..5f7426a
--- /dev/null
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote;
+
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertTrue;
+
+public class RemoteInterpreterUtilsTest {
+
+ @Test
+ public void testFindRandomAvailablePortOnAllLocalInterfaces() throws IOException {
+ assertTrue(RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces() > 0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/test/resources/conf/interpreter.json
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/resources/conf/interpreter.json b/zeppelin-interpreter/src/test/resources/conf/interpreter.json
new file mode 100644
index 0000000..45e1d60
--- /dev/null
+++ b/zeppelin-interpreter/src/test/resources/conf/interpreter.json
@@ -0,0 +1,115 @@
+{
+ "interpreterSettings": {
+ "2C3RWCVAG": {
+ "id": "2C3RWCVAG",
+ "name": "test",
+ "group": "test",
+ "properties": {
+ "property_1": "value_1",
+ "property_2": "new_value_2",
+ "property_3": "value_3"
+ },
+ "status": "READY",
+ "interpreterGroup": [
+ {
+ "name": "echo",
+ "class": "org.apache.zeppelin.interpreter.EchoInterpreter",
+ "defaultInterpreter": true,
+ "editor": {
+ "language": "java",
+ "editOnDblClick": false
+ }
+ }
+ ],
+ "dependencies": [],
+ "option": {
+ "remote": true,
+ "port": -1,
+ "perNote": "shared",
+ "perUser": "shared",
+ "isExistingProcess": false,
+ "setPermission": false,
+ "users": [],
+ "isUserImpersonate": false
+ }
+ },
+
+ "2CKWE7B19": {
+ "id": "2CKWE7B19",
+ "name": "test2",
+ "group": "test",
+ "properties": {
+ "property_1": "value_1",
+ "property_2": "new_value_2",
+ "property_3": "value_3"
+ },
+ "status": "READY",
+ "interpreterGroup": [
+ {
+ "name": "echo",
+ "class": "org.apache.zeppelin.interpreter.EchoInterpreter",
+ "defaultInterpreter": true,
+ "editor": {
+ "language": "java",
+ "editOnDblClick": false
+ }
+ }
+ ],
+ "dependencies": [],
+ "option": {
+ "remote": true,
+ "port": -1,
+ "perNote": "shared",
+ "perUser": "shared",
+ "isExistingProcess": false,
+ "setPermission": false,
+ "users": [],
+ "isUserImpersonate": false
+ }
+ }
+ },
+ "interpreterBindings": {
+ "2C6793KRV": [
+ "2C48Y7FSJ",
+ "2C63XW4XE",
+ "2C66GE1VB",
+ "2C5VH924X",
+ "2C4BJDRRZ",
+ "2C3SQSB7V",
+ "2C4HKDCQW",
+ "2C3DR183X",
+ "2C66Z9XPQ",
+ "2C3PTPMUH",
+ "2C69WE69N",
+ "2C5SRRXHM",
+ "2C4ZD49PF",
+ "2C6V3D44K",
+ "2C4UB1UZA",
+ "2C5S1R21W",
+ "2C5DCRVGM",
+ "2C686X8ZH",
+ "2C3RWCVAG",
+ "2C3JKFMJU",
+ "2C3VECEG2"
+ ]
+ },
+ "interpreterRepositories": [
+ {
+ "id": "central",
+ "type": "default",
+ "url": "http://repo1.maven.org/maven2/",
+ "releasePolicy": {
+ "enabled": true,
+ "updatePolicy": "daily",
+ "checksumPolicy": "warn"
+ },
+ "snapshotPolicy": {
+ "enabled": true,
+ "updatePolicy": "daily",
+ "checksumPolicy": "warn"
+ },
+ "mirroredRepositories": [],
+ "repositoryManager": false
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/test/resources/interpreter/test/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/resources/interpreter/test/interpreter-setting.json b/zeppelin-interpreter/src/test/resources/interpreter/test/interpreter-setting.json
new file mode 100644
index 0000000..1ba1b94
--- /dev/null
+++ b/zeppelin-interpreter/src/test/resources/interpreter/test/interpreter-setting.json
@@ -0,0 +1,42 @@
+[
+ {
+ "group": "test",
+ "name": "double_echo",
+ "className": "org.apache.zeppelin.interpreter.DoubleEchoInterpreter",
+ "properties": {
+ "property_1": {
+ "envName": "PROPERTY_1",
+ "propertyName": "property_1",
+ "defaultValue": "value_1",
+ "description": "desc_1"
+ },
+ "property_2": {
+ "envName": "PROPERTY_2",
+ "propertyName": "property_2",
+ "defaultValue": "value_2",
+ "description": "desc_2"
+ }
+ }
+ },
+
+ {
+ "group": "test",
+ "name": "echo",
+ "defaultInterpreter": true,
+ "className": "org.apache.zeppelin.interpreter.EchoInterpreter",
+ "properties": {
+ "property_1": {
+ "envName": "PROPERTY_1",
+ "propertyName": "property_1",
+ "defaultValue": "value_1",
+ "description": "desc_1"
+ },
+ "property_2": {
+ "envName": "PROPERTY_2",
+ "propertyName": "property_2",
+ "defaultValue": "value_2",
+ "description": "desc_2"
+ }
+ }
+ }
+]
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/resources/log4j.properties b/zeppelin-interpreter/src/test/resources/log4j.properties
index d8a7839..6f34691 100644
--- a/zeppelin-interpreter/src/test/resources/log4j.properties
+++ b/zeppelin-interpreter/src/test/resources/log4j.properties
@@ -26,4 +26,6 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c:%L - %m%n
#
# Root logger option
-log4j.rootLogger=INFO, stdout
\ No newline at end of file
+log4j.rootLogger=INFO, stdout
+log4j.logger.org.apache.zeppelin.interpreter=DEBUG
+log4j.logger.org.apache.zeppelin.scheduler=DEBUG
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
index cd0210e..c1dba5c 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
@@ -185,7 +185,7 @@ public class InterpreterRestApi {
String noteId = request == null ? null : request.getNoteId();
if (null == noteId) {
- interpreterSettingManager.close(setting);
+ interpreterSettingManager.close(settingId);
} else {
interpreterSettingManager.restart(settingId, noteId, SecurityUtils.getPrincipal());
}
@@ -208,7 +208,7 @@ public class InterpreterRestApi {
@GET
@ZeppelinApi
public Response listInterpreter(String message) {
- Map<String, InterpreterSetting> m = interpreterSettingManager.getAvailableInterpreterSettings();
+ Map<String, InterpreterSetting> m = interpreterSettingManager.getInterpreterSettingTemplates();
return new JsonResponse<>(Status.OK, "", m).build();
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
index 7453470..c103eeb 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
@@ -31,12 +31,10 @@ import org.apache.shiro.web.env.EnvironmentLoaderListener;
import org.apache.shiro.web.servlet.ShiroFilter;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
-import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.helium.Helium;
import org.apache.zeppelin.helium.HeliumApplicationFactory;
import org.apache.zeppelin.helium.HeliumBundleFactory;
import org.apache.zeppelin.interpreter.InterpreterFactory;
-import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.notebook.Notebook;
@@ -93,13 +91,11 @@ public class ZeppelinServer extends Application {
private NotebookRepoSync notebookRepo;
private NotebookAuthorization notebookAuthorization;
private Credentials credentials;
- private DependencyResolver depResolver;
public ZeppelinServer() throws Exception {
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
- this.depResolver = new DependencyResolver(
- conf.getString(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO));
+
InterpreterOutput.limit = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT);
@@ -129,13 +125,26 @@ public class ZeppelinServer extends Application {
new File(conf.getRelativeDir("zeppelin-web/src/app/spell")));
}
+ this.schedulerFactory = SchedulerFactory.singleton();
+ this.interpreterSettingManager = new InterpreterSettingManager(conf, notebookWsServer,
+ notebookWsServer, notebookWsServer);
+ this.replFactory = new InterpreterFactory(interpreterSettingManager);
+ this.notebookRepo = new NotebookRepoSync(conf);
+ this.noteSearchService = new LuceneSearch();
+ this.notebookAuthorization = NotebookAuthorization.init(conf);
+ this.credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath());
+ notebook = new Notebook(conf,
+ notebookRepo, schedulerFactory, replFactory, interpreterSettingManager, notebookWsServer,
+ noteSearchService, notebookAuthorization, credentials);
+
ZeppelinServer.helium = new Helium(
conf.getHeliumConfPath(),
conf.getHeliumRegistry(),
new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO),
"helium-registry-cache"),
heliumBundleFactory,
- heliumApplicationFactory);
+ heliumApplicationFactory,
+ interpreterSettingManager);
// create bundle
try {
@@ -144,20 +153,6 @@ public class ZeppelinServer extends Application {
LOG.error(e.getMessage(), e);
}
- this.schedulerFactory = new SchedulerFactory();
- this.interpreterSettingManager = new InterpreterSettingManager(conf, depResolver,
- new InterpreterOption(true));
- this.replFactory = new InterpreterFactory(conf, notebookWsServer,
- notebookWsServer, heliumApplicationFactory, depResolver, SecurityUtils.isAuthenticated(),
- interpreterSettingManager);
- this.notebookRepo = new NotebookRepoSync(conf);
- this.noteSearchService = new LuceneSearch();
- this.notebookAuthorization = NotebookAuthorization.init(conf);
- this.credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath());
- notebook = new Notebook(conf,
- notebookRepo, schedulerFactory, replFactory, interpreterSettingManager, notebookWsServer,
- noteSearchService, notebookAuthorization, credentials);
-
// to update notebook from application event from remote process.
heliumApplicationFactory.setNotebook(notebook);
// to update fire websocket event on application event.
@@ -206,7 +201,7 @@ public class ZeppelinServer extends Application {
LOG.info("Shutting down Zeppelin Server ... ");
try {
jettyWebServer.stop();
- notebook.getInterpreterSettingManager().shutdown();
+ notebook.getInterpreterSettingManager().close();
notebook.close();
Thread.sleep(3000);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
index f0e0bb2..2e3a5c7 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
@@ -41,12 +41,7 @@ import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.display.Input;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.helium.HeliumPackage;
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterContextRunner;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.InterpreterResultMessage;
-import org.apache.zeppelin.interpreter.InterpreterSetting;
+import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
@@ -59,7 +54,6 @@ import org.apache.zeppelin.notebook.NotebookEventListener;
import org.apache.zeppelin.notebook.NotebookImportDeserializer;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.notebook.ParagraphJobListener;
-import org.apache.zeppelin.notebook.ParagraphRuntimeInfo;
import org.apache.zeppelin.notebook.repo.NotebookRepo.Revision;
import org.apache.zeppelin.notebook.socket.Message;
import org.apache.zeppelin.notebook.socket.Message.OP;
@@ -86,8 +80,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.Queues;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
import com.google.gson.reflect.TypeToken;
/**
@@ -463,7 +455,8 @@ public class NotebookServer extends WebSocketServlet
Notebook notebook = notebook();
List<Note> notes = notebook.getAllNotes();
for (Note note : notes) {
- List<String> ids = notebook.getInterpreterSettingManager().getInterpreters(note.getId());
+ List<String> ids = notebook.getInterpreterSettingManager()
+ .getInterpreterBinding(note.getId());
for (String id : ids) {
if (id.equals(interpreterGroupId)) {
broadcast(note.getId(), m);
@@ -1022,7 +1015,7 @@ public class NotebookServer extends WebSocketServlet
List<String> interpreterSettingIds = new LinkedList<>();
interpreterSettingIds.add(defaultInterpreterId);
for (String interpreterSettingId : notebook.getInterpreterSettingManager().
- getDefaultInterpreterSettingList()) {
+ getInterpreterSettingIds()) {
if (!interpreterSettingId.equals(defaultInterpreterId)) {
interpreterSettingIds.add(interpreterSettingId);
}
@@ -1382,12 +1375,13 @@ public class NotebookServer extends WebSocketServlet
List<InterpreterSetting> settings =
notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
- if (setting.getInterpreterGroup(user, note.getId()) == null) {
+ if (setting.getOrCreateInterpreterGroup(user, note.getId()) == null) {
continue;
}
- if (interpreterGroupId.equals(setting.getInterpreterGroup(user, note.getId()).getId())) {
+ if (interpreterGroupId.equals(setting.getOrCreateInterpreterGroup(user, note.getId())
+ .getId())) {
AngularObjectRegistry angularObjectRegistry =
- setting.getInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
+ setting.getOrCreateInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
// first trying to get local registry
ao = angularObjectRegistry.get(varName, noteId, paragraphId);
@@ -1424,12 +1418,13 @@ public class NotebookServer extends WebSocketServlet
List<InterpreterSetting> settings =
notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
- if (setting.getInterpreterGroup(user, n.getId()) == null) {
+ if (setting.getOrCreateInterpreterGroup(user, n.getId()) == null) {
continue;
}
- if (interpreterGroupId.equals(setting.getInterpreterGroup(user, n.getId()).getId())) {
+ if (interpreterGroupId.equals(setting.getOrCreateInterpreterGroup(user, n.getId())
+ .getId())) {
AngularObjectRegistry angularObjectRegistry =
- setting.getInterpreterGroup(user, n.getId()).getAngularObjectRegistry();
+ setting.getOrCreateInterpreterGroup(user, n.getId()).getAngularObjectRegistry();
this.broadcastExcept(n.getId(),
new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao)
.put("interpreterGroupId", interpreterGroupId).put("noteId", n.getId())
@@ -2302,13 +2297,13 @@ public class NotebookServer extends WebSocketServlet
for (InterpreterSetting intpSetting : settings) {
AngularObjectRegistry registry =
- intpSetting.getInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
+ intpSetting.getOrCreateInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
List<AngularObject> objects = registry.getAllWithGlobal(note.getId());
for (AngularObject object : objects) {
conn.send(serializeMessage(
new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", object)
.put("interpreterGroupId",
- intpSetting.getInterpreterGroup(user, note.getId()).getId())
+ intpSetting.getOrCreateInterpreterGroup(user, note.getId()).getId())
.put("noteId", note.getId()).put("paragraphId", object.getParagraphId())));
}
}
@@ -2354,7 +2349,7 @@ public class NotebookServer extends WebSocketServlet
}
List<String> settingIds =
- notebook.getInterpreterSettingManager().getInterpreters(note.getId());
+ notebook.getInterpreterSettingManager().getInterpreterBinding(note.getId());
for (String id : settingIds) {
if (interpreterGroupId.contains(id)) {
broadcast(note.getId(),
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-server/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java b/zeppelin-server/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
deleted file mode 100644
index 1b1306a..0000000
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.zeppelin.interpreter.mock;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
-import org.apache.zeppelin.scheduler.Scheduler;
-import org.apache.zeppelin.scheduler.SchedulerFactory;
-
-public class MockInterpreter1 extends Interpreter{
- Map<String, Object> vars = new HashMap<>();
-
- public MockInterpreter1(Properties property) {
- super(property);
- }
-
- @Override
- public void open() {
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public InterpreterResult interpret(String st, InterpreterContext context) {
- return new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl1: "+st);
- }
-
- @Override
- public void cancel(InterpreterContext context) {
- }
-
- @Override
- public FormType getFormType() {
- return FormType.SIMPLE;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-
- @Override
- public Scheduler getScheduler() {
- return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_"+this.hashCode());
- }
-
- @Override
- public List<InterpreterCompletion> completion(String buf, int cursor,
- InterpreterContext interpreterContext) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
index a7907db..e2f171f 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
@@ -307,10 +307,9 @@ public abstract class AbstractTestRestApi {
protected static void shutDown() throws Exception {
if (!wasRunning) {
// restart interpreter to stop all interpreter processes
- List<String> settingList = ZeppelinServer.notebook.getInterpreterSettingManager()
- .getDefaultInterpreterSettingList();
- for (String setting : settingList) {
- ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting);
+ List<InterpreterSetting> settingList = ZeppelinServer.notebook.getInterpreterSettingManager().get();
+ for (InterpreterSetting setting : settingList) {
+ ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting.getId());
}
if (shiroIni != null) {
FileUtils.deleteQuietly(shiroIni);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
index 28541bd..72dd8a7 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
@@ -80,7 +80,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
// then
assertThat(get, isAllowed());
- assertEquals(ZeppelinServer.notebook.getInterpreterSettingManager().getAvailableInterpreterSettings().size(),
+ assertEquals(ZeppelinServer.notebook.getInterpreterSettingManager().getInterpreterSettingTemplates().size(),
body.entrySet().size());
get.releaseConnection();
}
@@ -110,7 +110,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
@Test
public void testSettingsCRUD() throws IOException {
// when: call create setting API
- String rawRequest = "{\"name\":\"md2\",\"group\":\"md\"," +
+ String rawRequest = "{\"name\":\"md3\",\"group\":\"md\"," +
"\"properties\":{\"propname\": {\"value\": \"propvalue\", \"name\": \"propname\", \"type\": \"textarea\"}}," +
"\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
"\"dependencies\":[]," +
@@ -367,7 +367,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
@Test
public void testGetMetadataInfo() throws IOException {
- String jsonRequest = "{\"name\":\"spark\",\"group\":\"spark\"," +
+ String jsonRequest = "{\"name\":\"spark_new\",\"group\":\"spark\"," +
"\"properties\":{\"propname\": {\"value\": \"propvalue\", \"name\": \"propname\", \"type\": \"textarea\"}}," +
"\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
"\"dependencies\":[]," +
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
index 8da36a6..10d77b2 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
@@ -30,6 +30,7 @@ import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.notebook.socket.Message;
import org.apache.zeppelin.notebook.socket.Message.OP;
import org.apache.zeppelin.rest.AbstractTestRestApi;
+import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.server.ZeppelinServer;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.AfterClass;
@@ -95,7 +96,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
}
@Test
- public void testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent() throws IOException {
+ public void testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent() throws IOException, InterruptedException {
// create a notebook
Note note1 = notebook.createNote(anonymous);
@@ -104,7 +105,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
List<InterpreterSetting> settings = notebook.getInterpreterSettingManager().getInterpreterSettings(note1.getId());
for (InterpreterSetting setting : settings) {
if (setting.getName().equals("md")) {
- interpreterGroup = setting.getInterpreterGroup("anonymous", "sharedProcess");
+ interpreterGroup = setting.getOrCreateInterpreterGroup("anonymous", "sharedProcess");
break;
}
}
@@ -115,6 +116,14 @@ public class NotebookServerTest extends AbstractTestRestApi {
p1.setAuthenticationInfo(anonymous);
note1.run(p1.getId());
+ // wait for paragraph finished
+ while(true) {
+ if (p1.getStatus() == Job.Status.FINISHED) {
+ break;
+ }
+ Thread.sleep(100);
+ }
+
// add angularObject
interpreterGroup.getAngularObjectRegistry().add("object1", "value1", note1.getId(), null);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-server/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/resources/log4j.properties b/zeppelin-server/src/test/resources/log4j.properties
index b0d1067..8368993 100644
--- a/zeppelin-server/src/test/resources/log4j.properties
+++ b/zeppelin-server/src/test/resources/log4j.properties
@@ -33,7 +33,6 @@ log4j.logger.org.apache.hadoop.mapred=WARN
log4j.logger.org.apache.hadoop.hive.ql=WARN
log4j.logger.org.apache.hadoop.hive.metastore=WARN
log4j.logger.org.apache.haadoop.hive.service.HiveServer=WARN
-log4j.logger.org.apache.zeppelin.scheduler=WARN
log4j.logger.org.quartz=WARN
log4j.logger.DataNucleus=WARN
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index f00fe93..03cc069 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -17,22 +17,21 @@
package org.apache.zeppelin.conf;
-import java.io.File;
-import java.net.URL;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.XMLConfiguration;
import org.apache.commons.configuration.tree.ConfigurationNode;
import org.apache.commons.lang.StringUtils;
-import org.apache.zeppelin.notebook.repo.GitNotebookRepo;
import org.apache.zeppelin.util.Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
/**
* Zeppelin configuration.
*
@@ -528,7 +527,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ConfigurationKeyPredicate predicate) {
Map<String, String> configurations = new HashMap<>();
- for (ZeppelinConfiguration.ConfVars v : ZeppelinConfiguration.ConfVars.values()) {
+ for (ConfVars v : ConfVars.values()) {
String key = v.getVarName();
if (!predicate.apply(key)) {
@@ -653,7 +652,8 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_NOTEBOOK_MONGO_COLLECTION("zeppelin.notebook.mongo.collection", "notes"),
ZEPPELIN_NOTEBOOK_MONGO_URI("zeppelin.notebook.mongo.uri", "mongodb://localhost"),
ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT("zeppelin.notebook.mongo.autoimport", false),
- ZEPPELIN_NOTEBOOK_STORAGE("zeppelin.notebook.storage", GitNotebookRepo.class.getName()),
+ ZEPPELIN_NOTEBOOK_STORAGE("zeppelin.notebook.storage",
+ "org.apache.zeppelin.notebook.repo.GitNotebookRepo"),
ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC("zeppelin.notebook.one.way.sync", false),
// whether by default note is public or private
ZEPPELIN_NOTEBOOK_PUBLIC("zeppelin.notebook.public", true),
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
index 17a3529..e5f2e3b 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
@@ -16,14 +16,17 @@
*/
package org.apache.zeppelin.helium;
+import com.google.gson.Gson;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterSettingManager;
+import org.apache.zeppelin.interpreter.ManagedInterpreterGroup;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
import org.apache.zeppelin.notebook.Paragraph;
-import org.apache.zeppelin.resource.DistributedResourcePool;
-import org.apache.zeppelin.resource.ResourcePool;
-import org.apache.zeppelin.resource.ResourcePoolUtils;
-import org.apache.zeppelin.resource.ResourceSet;
+import org.apache.zeppelin.resource.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,19 +50,22 @@ public class Helium {
private final HeliumBundleFactory bundleFactory;
private final HeliumApplicationFactory applicationFactory;
+ private final InterpreterSettingManager interpreterSettingManager;
public Helium(
String heliumConfPath,
String registryPaths,
File registryCacheDir,
HeliumBundleFactory bundleFactory,
- HeliumApplicationFactory applicationFactory)
+ HeliumApplicationFactory applicationFactory,
+ InterpreterSettingManager interpreterSettingManager)
throws IOException {
this.heliumConfPath = heliumConfPath;
this.registryPaths = registryPaths;
this.registryCacheDir = registryCacheDir;
this.bundleFactory = bundleFactory;
this.applicationFactory = applicationFactory;
+ this.interpreterSettingManager = interpreterSettingManager;
heliumConf = loadConf(heliumConfPath);
allPackages = getAllPackageInfo();
}
@@ -350,7 +356,7 @@ public class Helium {
allResources = resourcePool.getAll();
}
} else {
- allResources = ResourcePoolUtils.getAllResources();
+ allResources = interpreterSettingManager.getAllResources();
}
for (List<HeliumPackageSearchResult> pkgs : allPackages.values()) {
@@ -478,4 +484,40 @@ public class Helium {
return mixed;
}
+
+ public ResourceSet getAllResources() {
+ return getAllResourcesExcept(null);
+ }
+
+ private ResourceSet getAllResourcesExcept(String interpreterGroupExcludsion) {
+ ResourceSet resourceSet = new ResourceSet();
+ for (ManagedInterpreterGroup intpGroup : interpreterSettingManager.getAllInterpreterGroup()) {
+ if (interpreterGroupExcludsion != null &&
+ intpGroup.getId().equals(interpreterGroupExcludsion)) {
+ continue;
+ }
+
+ RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
+ if (remoteInterpreterProcess == null) {
+ ResourcePool localPool = intpGroup.getResourcePool();
+ if (localPool != null) {
+ resourceSet.addAll(localPool.getAll());
+ }
+ } else if (remoteInterpreterProcess.isRunning()) {
+ List<String> resourceList = remoteInterpreterProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<List<String>>() {
+ @Override
+ public List<String> call(RemoteInterpreterService.Client client) throws Exception {
+ return client.resourcePoolGetAll();
+ }
+ }
+ );
+ Gson gson = new Gson();
+ for (String res : resourceList) {
+ resourceSet.add(gson.fromJson(res, Resource.class));
+ }
+ }
+ }
+ return resourceSet;
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java
index 84368a7..f4dfae3 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java
@@ -16,7 +16,6 @@
*/
package org.apache.zeppelin.helium;
-import org.apache.thrift.TException;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
@@ -80,7 +79,7 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
try {
// get interpreter process
Interpreter intp = paragraph.getRepl(paragraph.getRequiredReplName());
- InterpreterGroup intpGroup = intp.getInterpreterGroup();
+ ManagedInterpreterGroup intpGroup = (ManagedInterpreterGroup) intp.getInterpreterGroup();
RemoteInterpreterProcess intpProcess = intpGroup.getRemoteInterpreterProcess();
if (intpProcess == null) {
throw new ApplicationException("Target interpreter process is not running");
@@ -105,38 +104,33 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
private void load(RemoteInterpreterProcess intpProcess, ApplicationState appState)
throws Exception {
- RemoteInterpreterService.Client client = null;
-
synchronized (appState) {
if (appState.getStatus() == ApplicationState.Status.LOADED) {
// already loaded
return;
}
- try {
- appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADING);
- String pkgInfo = pkg.toJson();
- String appId = appState.getId();
-
- client = intpProcess.getClient();
- RemoteApplicationResult ret = client.loadApplication(
- appId,
- pkgInfo,
- paragraph.getNote().getId(),
- paragraph.getId());
-
- if (ret.isSuccess()) {
- appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADED);
- } else {
- throw new ApplicationException(ret.getMsg());
- }
- } catch (TException e) {
- intpProcess.releaseBrokenClient(client);
- throw e;
- } finally {
- if (client != null) {
- intpProcess.releaseClient(client);
- }
+ appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADING);
+ final String pkgInfo = pkg.toJson();
+ final String appId = appState.getId();
+
+ RemoteApplicationResult ret = intpProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<RemoteApplicationResult>() {
+ @Override
+ public RemoteApplicationResult call(RemoteInterpreterService.Client client)
+ throws Exception {
+ return client.loadApplication(
+ appId,
+ pkgInfo,
+ paragraph.getNote().getId(),
+ paragraph.getId());
+ }
+ }
+ );
+ if (ret.isSuccess()) {
+ appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADED);
+ } else {
+ throw new ApplicationException(ret.getMsg());
}
}
}
@@ -199,7 +193,7 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
}
}
- private void unload(ApplicationState appsToUnload) throws ApplicationException {
+ private void unload(final ApplicationState appsToUnload) throws ApplicationException {
synchronized (appsToUnload) {
if (appsToUnload.getStatus() != ApplicationState.Status.LOADED) {
throw new ApplicationException(
@@ -212,31 +206,24 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
}
RemoteInterpreterProcess intpProcess =
- intp.getInterpreterGroup().getRemoteInterpreterProcess();
+ ((ManagedInterpreterGroup) intp.getInterpreterGroup()).getRemoteInterpreterProcess();
if (intpProcess == null) {
throw new ApplicationException("Target interpreter process is not running");
}
- RemoteInterpreterService.Client client;
- try {
- client = intpProcess.getClient();
- } catch (Exception e) {
- throw new ApplicationException(e);
- }
-
- try {
- RemoteApplicationResult ret = client.unloadApplication(appsToUnload.getId());
-
- if (ret.isSuccess()) {
- appStatusChange(paragraph, appsToUnload.getId(), ApplicationState.Status.UNLOADED);
- } else {
- throw new ApplicationException(ret.getMsg());
- }
- } catch (TException e) {
- intpProcess.releaseBrokenClient(client);
- throw new ApplicationException(e);
- } finally {
- intpProcess.releaseClient(client);
+ RemoteApplicationResult ret = intpProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<RemoteApplicationResult>() {
+ @Override
+ public RemoteApplicationResult call(RemoteInterpreterService.Client client)
+ throws Exception {
+ return client.unloadApplication(appsToUnload.getId());
+ }
+ }
+ );
+ if (ret.isSuccess()) {
+ appStatusChange(paragraph, appsToUnload.getId(), ApplicationState.Status.UNLOADED);
+ } else {
+ throw new ApplicationException(ret.getMsg());
}
}
}
@@ -286,7 +273,7 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
}
}
- private void run(ApplicationState app) throws ApplicationException {
+ private void run(final ApplicationState app) throws ApplicationException {
synchronized (app) {
if (app.getStatus() != ApplicationState.Status.LOADED) {
throw new ApplicationException(
@@ -299,33 +286,23 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
}
RemoteInterpreterProcess intpProcess =
- intp.getInterpreterGroup().getRemoteInterpreterProcess();
+ ((ManagedInterpreterGroup) intp.getInterpreterGroup()).getRemoteInterpreterProcess();
if (intpProcess == null) {
throw new ApplicationException("Target interpreter process is not running");
}
- RemoteInterpreterService.Client client = null;
- try {
- client = intpProcess.getClient();
- } catch (Exception e) {
- throw new ApplicationException(e);
- }
-
- try {
- RemoteApplicationResult ret = client.runApplication(app.getId());
-
- if (ret.isSuccess()) {
- // success
- } else {
- throw new ApplicationException(ret.getMsg());
- }
- } catch (TException e) {
- intpProcess.releaseBrokenClient(client);
- client = null;
- throw new ApplicationException(e);
- } finally {
- if (client != null) {
- intpProcess.releaseClient(client);
- }
+ RemoteApplicationResult ret = intpProcess.callRemoteFunction(
+ new RemoteInterpreterProcess.RemoteFunction<RemoteApplicationResult>() {
+ @Override
+ public RemoteApplicationResult call(RemoteInterpreterService.Client client)
+ throws Exception {
+ return client.runApplication(app.getId());
+ }
+ }
+ );
+ if (ret.isSuccess()) {
+ // success
+ } else {
+ throw new ApplicationException(ret.getMsg());
}
}
}
[3/9] zeppelin git commit: [ZEPPELIN-2627] Interpreter Component
Refactoring
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java
index 1aab757..3c061a9 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java
@@ -1,327 +1,411 @@
-package org.apache.zeppelin.interpreter;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
+package org.apache.zeppelin.interpreter;
import org.junit.Test;
-import org.apache.zeppelin.dep.Dependency;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.mock;
+import static org.junit.Assert.assertNull;
public class InterpreterSettingTest {
@Test
- public void sharedModeCloseandRemoveInterpreterGroupTest() {
+ public void testCreateInterpreters() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.SHARED);
- InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
-
- interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
- @Override
- public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
- InterpreterOption option) {
- return new InterpreterGroup(interpreterGroupId);
- }
- });
-
- Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList1 = new ArrayList<>();
- interpreterList1.add(mockInterpreter1);
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
-
- // This won't effect anything
- Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList2 = new ArrayList<>();
- interpreterList2.add(mockInterpreter2);
- interpreterGroup = interpreterSetting.getInterpreterGroup("user2", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user2", "note1"), interpreterList2);
-
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
-
- interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user2");
- assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
+ interpreterOption.setRemote(false);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+
+ // create default interpreter for user1 and note1
+ assertEquals(EchoInterpreter.class.getName(), interpreterSetting.getDefaultInterpreter("user1", "note1").getClassName());
+
+ // create interpreter echo for user1 and note1
+ assertEquals(EchoInterpreter.class.getName(), interpreterSetting.getInterpreter("user1", "note1", "echo").getClassName());
+ assertEquals(interpreterSetting.getDefaultInterpreter("user1", "note1"), interpreterSetting.getInterpreter("user1", "note1", "echo"));
+
+ // create interpreter double_echo for user1 and note1
+ assertEquals(DoubleEchoInterpreter.class.getName(), interpreterSetting.getInterpreter("user1", "note1", "double_echo").getClassName());
+
+ // create non-existed interpreter
+ assertNull(interpreterSetting.getInterpreter("user1", "note1", "invalid_echo"));
}
@Test
- public void perUserScopedModeCloseAndRemoveInterpreterGroupTest() {
+ public void testSharedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
- interpreterOption.setPerUser(InterpreterOption.SCOPED);
- InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
-
- interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
- @Override
- public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
- InterpreterOption option) {
- return new InterpreterGroup(interpreterGroupId);
- }
- });
-
- Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList1 = new ArrayList<>();
- interpreterList1.add(mockInterpreter1);
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
-
- Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList2 = new ArrayList<>();
- interpreterList2.add(mockInterpreter2);
- interpreterGroup = interpreterSetting.getInterpreterGroup("user2", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user2", "note1"), interpreterList2);
+ interpreterOption.setPerUser(InterpreterOption.SHARED);
+ interpreterOption.setRemote(false);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+
+ // create default interpreter for user1 and note1
+ interpreterSetting.getDefaultInterpreter("user1", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ // create default interpreter for user2 and note1
+ interpreterSetting.getDefaultInterpreter("user2", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
- assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note1").size());
- assertEquals(2, interpreterSetting.getInterpreterGroup("user2", "note1").size());
- interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
- assertEquals(1, interpreterSetting.getInterpreterGroup("user2","note1").size());
+ // create default interpreter user1 and note2
+ interpreterSetting.getDefaultInterpreter("user1", "note2");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
- // Check if non-existed key works or not
- interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
- assertEquals(1, interpreterSetting.getInterpreterGroup("user2","note1").size());
+ // only 1 session is created, this session is shared across users and notes
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
- interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user2");
+ interpreterSetting.closeInterpreters("note1", "user1");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
- public void perUserIsolatedModeCloseAndRemoveInterpreterGroupTest() {
+ public void testPerUserScopedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
- interpreterOption.setPerUser(InterpreterOption.ISOLATED);
- InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
-
- interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
- @Override
- public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
- InterpreterOption option) {
- return new InterpreterGroup(interpreterGroupId);
- }
- });
-
- Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList1 = new ArrayList<>();
- interpreterList1.add(mockInterpreter1);
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
-
- Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList2 = new ArrayList<>();
- interpreterList2.add(mockInterpreter2);
- interpreterGroup = interpreterSetting.getInterpreterGroup("user2", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user2", "note1"), interpreterList2);
+ interpreterOption.setPerUser(InterpreterOption.SCOPED);
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+
+ // create interpreter for user1 and note1
+ interpreterSetting.getDefaultInterpreter("user1", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
- assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
- assertEquals(1, interpreterSetting.getInterpreterGroup("user2", "note1").size());
+ // create interpreter for user2 and note1
+ interpreterSetting.getDefaultInterpreter("user2", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
- interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
- assertEquals(1, interpreterSetting.getInterpreterGroup("user2","note1").size());
+ interpreterSetting.closeInterpreters("user1", "note1");
+ // InterpreterGroup is still there, but one session is removed
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
- interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user2");
+ interpreterSetting.closeInterpreters("user2", "note1");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
- public void perNoteScopedModeCloseAndRemoveInterpreterGroupTest() {
+ public void testPerNoteScopedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerNote(InterpreterOption.SCOPED);
- InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
-
- interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
- @Override
- public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
- InterpreterOption option) {
- return new InterpreterGroup(interpreterGroupId);
- }
- });
-
- Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList1 = new ArrayList<>();
- interpreterList1.add(mockInterpreter1);
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
-
- Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList2 = new ArrayList<>();
- interpreterList2.add(mockInterpreter2);
- interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note2");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note2"), interpreterList2);
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+
+ // create interpreter for user1 and note1
+ interpreterSetting.getDefaultInterpreter("user1", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+ // create interpreter for user1 and note2
+ interpreterSetting.getDefaultInterpreter("user1", "note2");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
- assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note1").size());
- assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note2").size());
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
- interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note2").size());
+ interpreterSetting.closeInterpreters("user1", "note1");
+ // InterpreterGroup is still there, but one session is removed
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
- // Check if non-existed key works or not
- interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note2").size());
+ interpreterSetting.closeInterpreters("user1", "note2");
+ assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
+ }
- interpreterSetting.closeAndRemoveInterpreterGroup("note2", "user1");
+ @Test
+ public void testPerUserIsolatedMode() {
+ InterpreterOption interpreterOption = new InterpreterOption();
+ interpreterOption.setPerUser(InterpreterOption.ISOLATED);
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+
+ // create interpreter for user1 and note1
+ interpreterSetting.getDefaultInterpreter("user1", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // create interpreter for user2 and note1
+ interpreterSetting.getDefaultInterpreter("user2", "note1");
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
+
+ // Each user own one InterpreterGroup and one session per InterpreterGroup
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(1).getSessionNum());
+
+ interpreterSetting.closeInterpreters("user1", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ interpreterSetting.closeInterpreters("user2", "note1");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
- public void perNoteIsolatedModeCloseAndRemoveInterpreterGroupTest() {
+ public void testPerNoteIsolatedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerNote(InterpreterOption.ISOLATED);
- InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
-
- interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
- @Override
- public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
- InterpreterOption option) {
- return new InterpreterGroup(interpreterGroupId);
- }
- });
-
- Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList1 = new ArrayList<>();
- interpreterList1.add(mockInterpreter1);
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
-
- Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList2 = new ArrayList<>();
- interpreterList2.add(mockInterpreter2);
- interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note2");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note2"), interpreterList2);
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+
+ // create interpreter for user1 and note1
+ interpreterSetting.getDefaultInterpreter("user1", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+ // create interpreter for user2 and note2
+ interpreterSetting.getDefaultInterpreter("user1", "note2");
assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note2").size());
+ // Each user own one InterpreterGroup and one session per InterpreterGroup
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(1).getSessionNum());
- interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note2").size());
+ interpreterSetting.closeInterpreters("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
-
- interpreterSetting.closeAndRemoveInterpreterGroup("note2", "user1");
+ interpreterSetting.closeInterpreters("user1", "note2");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
- public void perNoteScopedModeRemoveInterpreterGroupWhenNoteIsRemoved() {
+ public void testPerUserIsolatedPerNoteScopedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
+ interpreterOption.setPerUser(InterpreterOption.ISOLATED);
interpreterOption.setPerNote(InterpreterOption.SCOPED);
- InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
-
- interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
- @Override
- public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
- InterpreterOption option) {
- return new InterpreterGroup(interpreterGroupId);
- }
- });
-
- Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList1 = new ArrayList<>();
- interpreterList1.add(mockInterpreter1);
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+
+ // create interpreter for user1 and note1
+ interpreterSetting.getDefaultInterpreter("user1", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ interpreterSetting.getDefaultInterpreter("user1", "note2");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // create interpreter for user2 and note1
+ interpreterSetting.getDefaultInterpreter("user2", "note1");
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
+
+ // group1 for user1 has 2 sessions, and group2 for user2 has 1 session
+ assertEquals(interpreterSetting.getInterpreterGroup("user1", "note1"), interpreterSetting.getInterpreterGroup("user1", "note2"));
+ assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note1").getSessionNum());
+ assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note2").getSessionNum());
+ assertEquals(1, interpreterSetting.getInterpreterGroup("user2", "note1").getSessionNum());
+
+ // close one session for user1
+ interpreterSetting.closeInterpreters("user1", "note1");
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").getSessionNum());
+ // close another session for user1
+ interpreterSetting.closeInterpreters("user1", "note2");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
- // This method will be called when remove note
- interpreterSetting.closeAndRemoveInterpreterGroup("note1","");
+ // close session for user2
+ interpreterSetting.closeInterpreters("user2", "note1");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
- // Be careful that getInterpreterGroup makes interpreterGroup if it doesn't exist
- assertEquals(0, interpreterSetting.getInterpreterGroup("user1","note1").size());
}
@Test
- public void perNoteIsolatedModeRemoveInterpreterGroupWhenNoteIsRemoved() {
+ public void testPerUserIsolatedPerNoteIsolatedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
+ interpreterOption.setPerUser(InterpreterOption.ISOLATED);
interpreterOption.setPerNote(InterpreterOption.ISOLATED);
- InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
-
- interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
- @Override
- public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
- InterpreterOption option) {
- return new InterpreterGroup(interpreterGroupId);
- }
- });
-
- Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList1 = new ArrayList<>();
- interpreterList1.add(mockInterpreter1);
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+
+ // create interpreter for user1 and note1
+ interpreterSetting.getDefaultInterpreter("user1", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
+
+ // create interpreter for user1 and note2
+ interpreterSetting.getDefaultInterpreter("user1", "note2");
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
+
+ // create interpreter for user2 and note1
+ interpreterSetting.getDefaultInterpreter("user2", "note1");
+ assertEquals(3, interpreterSetting.getAllInterpreterGroups().size());
+
+ // create interpreter for user2 and note2
+ interpreterSetting.getDefaultInterpreter("user2", "note2");
+ assertEquals(4, interpreterSetting.getAllInterpreterGroups().size());
+
+ for (InterpreterGroup interpreterGroup : interpreterSetting.getAllInterpreterGroups()) {
+ // each InterpreterGroup has one session
+ assertEquals(1, interpreterGroup.getSessionNum());
+ }
+ // close one session for user1 and note1
+ interpreterSetting.closeInterpreters("user1", "note1");
+ assertEquals(3, interpreterSetting.getAllInterpreterGroups().size());
+
+ // close one session for user1 and note2
+ interpreterSetting.closeInterpreters("user1", "note2");
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
+
+ // close one session for user2 and note1
+ interpreterSetting.closeInterpreters("user2", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
- // This method will be called when remove note
- interpreterSetting.closeAndRemoveInterpreterGroup("note1","");
+ // close one session for user2 and note2
+ interpreterSetting.closeInterpreters("user2", "note2");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
- // Be careful that getInterpreterGroup makes interpreterGroup if it doesn't exist
- assertEquals(0, interpreterSetting.getInterpreterGroup("user1","note1").size());
}
@Test
- public void perUserScopedModeNeverRemoveInterpreterGroupWhenNoteIsRemoved() {
+ public void testPerUserScopedPerNoteScopedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.SCOPED);
- InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
-
- interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
- @Override
- public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
- InterpreterOption option) {
- return new InterpreterGroup(interpreterGroupId);
- }
- });
-
- Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList1 = new ArrayList<>();
- interpreterList1.add(mockInterpreter1);
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
-
+ interpreterOption.setPerNote(InterpreterOption.SCOPED);
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+
+ // create interpreter for user1 and note1
+ interpreterSetting.getDefaultInterpreter("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
- // This method will be called when remove note
- interpreterSetting.closeAndRemoveInterpreterGroup("note1","");
+ // create interpreter for user1 and note2
+ interpreterSetting.getDefaultInterpreter("user1", "note2");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
- // Be careful that getInterpreterGroup makes interpreterGroup if it doesn't exist
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note1").size());
- }
-
- @Test
- public void perUserIsolatedModeNeverRemoveInterpreterGroupWhenNoteIsRemoved() {
- InterpreterOption interpreterOption = new InterpreterOption();
- interpreterOption.setPerUser(InterpreterOption.ISOLATED);
- InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
-
- interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
- @Override
- public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
- InterpreterOption option) {
- return new InterpreterGroup(interpreterGroupId);
- }
- });
-
- Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
- List<Interpreter> interpreterList1 = new ArrayList<>();
- interpreterList1.add(mockInterpreter1);
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
- interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+ // create interpreter for user2 and note1
+ interpreterSetting.getDefaultInterpreter("user2", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
+ assertEquals(3, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
- // This method will be called when remove note
- interpreterSetting.closeAndRemoveInterpreterGroup("note1","");
+ // create interpreter for user2 and note2
+ interpreterSetting.getDefaultInterpreter("user2", "note2");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
- // Be careful that getInterpreterGroup makes interpreterGroup if it doesn't exist
- assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note1").size());
+ assertEquals(4, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // close one session for user1 and note1
+ interpreterSetting.closeInterpreters("user1", "note1");
+ assertEquals(3, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // close one session for user1 and note2
+ interpreterSetting.closeInterpreters("user1", "note2");
+ assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // close one session for user2 and note1
+ interpreterSetting.closeInterpreters("user2", "note1");
+ assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // close one session for user2 and note2
+ interpreterSetting.closeInterpreters("user2", "note2");
+ assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroupTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroupTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroupTest.java
new file mode 100644
index 0000000..a5d7bd4
--- /dev/null
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroupTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonatype.aether.RepositoryException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+
+public class ManagedInterpreterGroupTest {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ManagedInterpreterGroupTest.class);
+
+ private InterpreterSetting interpreterSetting;
+
+ @Before
+ public void setUp() throws IOException, RepositoryException {
+ InterpreterOption interpreterOption = new InterpreterOption();
+ interpreterOption.setPerUser(InterpreterOption.SCOPED);
+ interpreterOption.setRemote(false);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ interpreterSetting = new InterpreterSetting.Builder()
+ .setId("id")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .create();
+ }
+
+ @Test
+ public void testInterpreterGroup() {
+ ManagedInterpreterGroup interpreterGroup = new ManagedInterpreterGroup("group_1", interpreterSetting);
+ assertEquals(0, interpreterGroup.getSessionNum());
+
+ // create session_1
+ List<Interpreter> interpreters = interpreterGroup.getOrCreateSession("user1", "session_1");
+ assertEquals(2, interpreters.size());
+ assertEquals(EchoInterpreter.class.getName(), interpreters.get(0).getClassName());
+ assertEquals(DoubleEchoInterpreter.class.getName(), interpreters.get(1).getClassName());
+ assertEquals(1, interpreterGroup.getSessionNum());
+
+ // get the same interpreters when interpreterGroup.getOrCreateSession is invoked again
+ assertEquals(interpreters, interpreterGroup.getOrCreateSession("user1", "session_1"));
+ assertEquals(1, interpreterGroup.getSessionNum());
+
+ // create session_2
+ List<Interpreter> interpreters2 = interpreterGroup.getOrCreateSession("user1", "session_2");
+ assertEquals(2, interpreters2.size());
+ assertEquals(EchoInterpreter.class.getName(), interpreters2.get(0).getClassName());
+ assertEquals(DoubleEchoInterpreter.class.getName(), interpreters2.get(1).getClassName());
+ assertEquals(2, interpreterGroup.getSessionNum());
+
+ // close session_1
+ interpreterGroup.close("session_1");
+ assertEquals(1, interpreterGroup.getSessionNum());
+
+ // close InterpreterGroup
+ interpreterGroup.close();
+ assertEquals(0, interpreterGroup.getSessionNum());
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java
new file mode 100644
index 0000000..9deafcf
--- /dev/null
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java
@@ -0,0 +1,60 @@
+package org.apache.zeppelin.interpreter;
+
+import org.apache.zeppelin.scheduler.Scheduler;
+import org.apache.zeppelin.scheduler.SchedulerFactory;
+
+import java.util.Properties;
+
+/**
+ * Interpreter that only accept long value and sleep for such period
+ */
+public class SleepInterpreter extends Interpreter {
+
+ public SleepInterpreter(Properties property) {
+ super(property);
+ }
+
+ @Override
+ public void open() {
+
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ try {
+ Thread.sleep(Long.parseLong(st));
+ return new InterpreterResult(InterpreterResult.Code.SUCCESS);
+ } catch (Exception e) {
+ return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
+ }
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+
+ }
+
+ @Override
+ public FormType getFormType() {
+ return FormType.NATIVE;
+ }
+
+ @Override
+ public Scheduler getScheduler() {
+ if (Boolean.parseBoolean(property.getProperty("zeppelin.SleepInterpreter.parallel", "false"))) {
+ return SchedulerFactory.singleton().createOrGetParallelScheduler(
+ "Parallel-" + SleepInterpreter.class.getName(), 10);
+ }
+ return super.getScheduler();
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
index b16e937..a533c12 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
@@ -17,11 +17,6 @@
package org.apache.zeppelin.interpreter.mock;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
@@ -29,8 +24,14 @@ import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
-public class MockInterpreter1 extends Interpreter{
-Map<String, Object> vars = new HashMap<>();
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class MockInterpreter1 extends Interpreter {
+
+ Map<String, Object> vars = new HashMap<>();
public MockInterpreter1(Properties property) {
super(property);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter11.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter11.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter11.java
deleted file mode 100644
index 5b9e802..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter11.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.mock;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
-import org.apache.zeppelin.scheduler.Scheduler;
-import org.apache.zeppelin.scheduler.SchedulerFactory;
-
-public class MockInterpreter11 extends Interpreter{
- Map<String, Object> vars = new HashMap<>();
-
- public MockInterpreter11(Properties property) {
- super(property);
- }
- boolean open;
-
- @Override
- public void open() {
- open = true;
- }
-
- @Override
- public void close() {
- open = false;
- }
-
- public boolean isOpen() {
- return open;
- }
-
- @Override
- public InterpreterResult interpret(String st, InterpreterContext context) {
- return new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl11: "+st);
- }
-
- @Override
- public void cancel(InterpreterContext context) {
- }
-
- @Override
- public FormType getFormType() {
- return FormType.SIMPLE;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-
- @Override
- public Scheduler getScheduler() {
- return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_"+this.hashCode());
- }
-
- @Override
- public List<InterpreterCompletion> completion(String buf, int cursor,
- InterpreterContext interpreterContext) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java
index 7a52f7d..f36df56 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java
@@ -17,11 +17,6 @@
package org.apache.zeppelin.interpreter.mock;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
@@ -29,6 +24,11 @@ import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
public class MockInterpreter2 extends Interpreter{
Map<String, Object> vars = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java
index c8c64ea..c9dc5c0 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java
@@ -17,23 +17,6 @@
package org.apache.zeppelin.interpreter.remote;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyInt;
-import static org.mockito.Mockito.atMost;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-
import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -43,6 +26,19 @@ import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Mockito.*;
+
public class AppendOutputRunnerTest {
private static final int NUM_EVENTS = 10000;
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
index f7404e3..61e4ef0 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
@@ -17,15 +17,10 @@
package org.apache.zeppelin.interpreter.remote;
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.Properties;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.zeppelin.display.*;
+import org.apache.zeppelin.display.AngularObject;
+import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.display.AngularObjectRegistryListener;
+import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterAngular;
import org.apache.zeppelin.resource.LocalResourcePool;
@@ -34,17 +29,25 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+
public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
private static final String INTERPRETER_SCRIPT =
- System.getProperty("os.name").startsWith("Windows") ?
- "../bin/interpreter.cmd" :
- "../bin/interpreter.sh";
+ System.getProperty("os.name").startsWith("Windows") ?
+ "../bin/interpreter.cmd" :
+ "../bin/interpreter.sh";
- private InterpreterGroup intpGroup;
- private HashMap<String, String> env;
private RemoteInterpreter intp;
private InterpreterContext context;
private RemoteAngularObjectRegistry localRegistry;
+ private InterpreterSetting interpreterSetting;
private AtomicInteger onAdd;
private AtomicInteger onUpdate;
@@ -56,32 +59,24 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
onUpdate = new AtomicInteger(0);
onRemove = new AtomicInteger(0);
- intpGroup = new InterpreterGroup("intpId");
- localRegistry = new RemoteAngularObjectRegistry("intpId", this, intpGroup);
- intpGroup.setAngularObjectRegistry(localRegistry);
- env = new HashMap<>();
- env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
-
- Properties p = new Properties();
-
- intp = new RemoteInterpreter(
- p,
- "note",
- MockInterpreterAngular.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- null,
- null,
- "anonymous",
- false
- );
-
- intpGroup.put("note", new LinkedList<Interpreter>());
- intpGroup.get("note").add(intp);
- intp.setInterpreterGroup(intpGroup);
+ InterpreterOption interpreterOption = new InterpreterOption();
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(MockInterpreterAngular.class.getName(), "mock", true, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT);
+ interpreterSetting = new InterpreterSetting.Builder()
+ .setId("test")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .setRunner(runner)
+ .setInterpreterDir("../interpeters/test")
+ .create();
+
+ intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
+ localRegistry = (RemoteAngularObjectRegistry) intp.getInterpreterGroup().getAngularObjectRegistry();
context = new InterpreterContext(
"note",
@@ -92,17 +87,17 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
+ new AngularObjectRegistry(intp.getInterpreterGroup().getId(), null),
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null);
intp.open();
+
}
@After
public void tearDown() throws Exception {
- intp.close();
- intpGroup.close();
+ interpreterSetting.close();
}
@Test
@@ -147,7 +142,7 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
Thread.sleep(500); // waitFor eventpoller pool event
String[] result = ret.message().get(0).getData().split(" ");
assertEquals("0", result[0]); // size of registry
-
+
// create object
ret = intp.interpret("add n1 v1", context);
Thread.sleep(500);
@@ -172,11 +167,11 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
Thread.sleep(500); // waitFor eventpoller pool event
String[] result = ret.message().get(0).getData().split(" ");
assertEquals("0", result[0]); // size of registry
-
+
// create object
localRegistry.addAndNotifyRemoteProcess("n1", "v1", "note", null);
-
- // get from remote registry
+
+ // get from remote registry
ret = intp.interpret("get", context);
Thread.sleep(500); // waitFor eventpoller pool event
result = ret.message().get(0).getData().split(" ");
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
index 3f865cb..1687060 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
@@ -17,22 +17,18 @@
package org.apache.zeppelin.interpreter.remote;
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterOutputStream;
+import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import java.io.File;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.Properties;
+import java.util.*;
import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
/**
@@ -43,41 +39,32 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
- private InterpreterGroup intpGroup;
- private HashMap<String, String> env;
+
+ private InterpreterSetting interpreterSetting;
@Before
public void setUp() throws Exception {
- intpGroup = new InterpreterGroup();
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- env = new HashMap<>();
- env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
+ InterpreterOption interpreterOption = new InterpreterOption();
+
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(MockInterpreterOutputStream.class.getName(), "mock", true, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT);
+ interpreterSetting = new InterpreterSetting.Builder()
+ .setId("test")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .setRunner(runner)
+ .setInterpreterDir("../interpeters/test")
+ .create();
}
@After
public void tearDown() throws Exception {
- intpGroup.close();
- }
-
- private RemoteInterpreter createMockInterpreter() {
- RemoteInterpreter intp = new RemoteInterpreter(
- new Properties(),
- "note",
- MockInterpreterOutputStream.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- this,
- null,
- "anonymous",
- false);
-
- intpGroup.get("note").add(intp);
- intp.setInterpreterGroup(intpGroup);
- return intp;
+ interpreterSetting.close();
}
private InterpreterContext createInterpreterContext() {
@@ -90,14 +77,14 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
+ null,
null,
new LinkedList<InterpreterContextRunner>(), null);
}
@Test
public void testInterpreterResultOnly() {
- RemoteInterpreter intp = createMockInterpreter();
+ RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
InterpreterResult ret = intp.interpret("SUCCESS::staticresult", createInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
assertEquals("staticresult", ret.message().get(0).getData());
@@ -113,7 +100,7 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
@Test
public void testInterpreterOutputStreamOnly() {
- RemoteInterpreter intp = createMockInterpreter();
+ RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
InterpreterResult ret = intp.interpret("SUCCESS:streamresult:", createInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
assertEquals("streamresult", ret.message().get(0).getData());
@@ -125,7 +112,7 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
@Test
public void testInterpreterResultOutputStreamMixed() {
- RemoteInterpreter intp = createMockInterpreter();
+ RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
InterpreterResult ret = intp.interpret("SUCCESS:stream:static", createInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
assertEquals("stream", ret.message().get(0).getData());
@@ -134,7 +121,7 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
@Test
public void testOutputType() {
- RemoteInterpreter intp = createMockInterpreter();
+ RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
InterpreterResult ret = intp.interpret("SUCCESS:%html hello:", createInterpreterContext());
assertEquals(InterpreterResult.Type.HTML, ret.message().get(0).getType());
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
deleted file mode 100644
index b85d7ef..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.mockito.Mockito.*;
-
-import java.util.HashMap;
-import java.util.Properties;
-
-import org.apache.thrift.TException;
-import org.apache.thrift.transport.TTransportException;
-import org.apache.zeppelin.interpreter.Constants;
-import org.apache.zeppelin.interpreter.InterpreterException;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
-import org.junit.Test;
-
-public class RemoteInterpreterProcessTest {
- private static final String INTERPRETER_SCRIPT =
- System.getProperty("os.name").startsWith("Windows") ?
- "../bin/interpreter.cmd" :
- "../bin/interpreter.sh";
- private static final int DUMMY_PORT=3678;
-
- @Test
- public void testStartStop() {
- InterpreterGroup intpGroup = new InterpreterGroup();
- RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
- INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap<String, String>(),
- 10 * 1000, null, null,"fakeName");
- assertFalse(rip.isRunning());
- assertEquals(0, rip.referenceCount());
- assertEquals(1, rip.reference(intpGroup, "anonymous", false));
- assertEquals(2, rip.reference(intpGroup, "anonymous", false));
- assertEquals(true, rip.isRunning());
- assertEquals(1, rip.dereference());
- assertEquals(true, rip.isRunning());
- assertEquals(0, rip.dereference());
- assertEquals(false, rip.isRunning());
- }
-
- @Test
- public void testClientFactory() throws Exception {
- InterpreterGroup intpGroup = new InterpreterGroup();
- RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
- INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap<String, String>(),
- mock(RemoteInterpreterEventPoller.class), 10 * 1000, "fakeName");
- rip.reference(intpGroup, "anonymous", false);
- assertEquals(0, rip.getNumActiveClient());
- assertEquals(0, rip.getNumIdleClient());
-
- Client client = rip.getClient();
- assertEquals(1, rip.getNumActiveClient());
- assertEquals(0, rip.getNumIdleClient());
-
- rip.releaseClient(client);
- assertEquals(0, rip.getNumActiveClient());
- assertEquals(1, rip.getNumIdleClient());
-
- rip.dereference();
- }
-
- @Test
- public void testStartStopRemoteInterpreter() throws TException, InterruptedException {
- RemoteInterpreterServer server = new RemoteInterpreterServer(3678);
- server.start();
- boolean running = false;
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() - startTime < 10 * 1000) {
- if (server.isRunning()) {
- running = true;
- break;
- } else {
- Thread.sleep(200);
- }
- }
- Properties properties = new Properties();
- properties.setProperty(Constants.ZEPPELIN_INTERPRETER_PORT, "3678");
- properties.setProperty(Constants.ZEPPELIN_INTERPRETER_HOST, "localhost");
- InterpreterGroup intpGroup = mock(InterpreterGroup.class);
- when(intpGroup.getProperty()).thenReturn(properties);
- when(intpGroup.containsKey(Constants.EXISTING_PROCESS)).thenReturn(true);
-
- RemoteInterpreterProcess rip = new RemoteInterpreterManagedProcess(
- INTERPRETER_SCRIPT,
- "nonexists",
- "fakeRepo",
- new HashMap<String, String>(),
- mock(RemoteInterpreterEventPoller.class)
- , 10 * 1000,
- "fakeName");
- assertFalse(rip.isRunning());
- assertEquals(0, rip.referenceCount());
- assertEquals(1, rip.reference(intpGroup, "anonymous", false));
- assertEquals(true, rip.isRunning());
- }
-
-
- @Test
- public void testPropagateError() throws TException, InterruptedException {
- InterpreterGroup intpGroup = new InterpreterGroup();
- RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
- "echo hello_world", "nonexists", "fakeRepo", new HashMap<String, String>(),
- 10 * 1000, null, null, "fakeName");
- assertFalse(rip.isRunning());
- assertEquals(0, rip.referenceCount());
- try {
- assertEquals(1, rip.reference(intpGroup, "anonymous", false));
- } catch (InterpreterException e) {
- e.getMessage().contains("hello_world");
- }
- assertEquals(0, rip.referenceCount());
- }
-}
[9/9] zeppelin git commit: [ZEPPELIN-2627] Interpreter Component
Refactoring
Posted by zj...@apache.org.
[ZEPPELIN-2627] Interpreter Component Refactoring
What is this PR for?
I didn't intended to make such large change at the beginning, but found many things are coupled together that I have to make such large change. Several suggestions for you how to review and read it.
I move the interpreter package from zeppelin-zengine to zeppelin-interpreter, this is needed for this refactoring.
The overall change is the same as I described in the design doc. I would suggest you to read the unit test first. These unit test is very readable and easy to understand what the code is doing now. `InterpreterFactoryTest`, `InterpreterGroupTest`, `InterpreterSettingTest`, `InterpreterSettingManagerTest`, `RemoteInterpreterTest`.
Remove the referent counting logic. Now I will kill the interpreter process as long as all the sessions in the same interpreter group is closed. (I plan to add another kind of policy for the interpreter process lifecycle control, ZEPPELIN-2197)
The RemoteFunction I introduced is for reducing code duplicates when we use RPC.
The changes in `Job.java` and `RemoteScheduler` is for fixing the race issue bug. This bug cause the flaky test we see often in `ZeppelinSparkClusterTest.pySparkTest`
What type of PR is it?
[Bug Fix | Refactoring]
Todos
- Task
What is the Jira issue?
https://issues.apache.org/jira/browse/ZEPPELIN-2627
How should this be tested?
Unit test is added
Screenshots (if appropriate)
Questions:
Does the licenses files need update? No
Is there breaking changes for older versions? No
Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #2554 from zjffdu/ZEPPELIN-2627-2 and squashes the following commits:
fa0d435 [Jeff Zhang] minor update
74bcb91 [Jeff Zhang] [ZEPPELIN-2627] Interpreter Component Refactoring
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/d6203c51
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/d6203c51
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/d6203c51
Branch: refs/heads/master
Commit: d6203c51ed9eef5e616090326d3dd6dddf21216a
Parents: 69d58f3
Author: Jeff Zhang <zj...@apache.org>
Authored: Fri Sep 1 12:50:46 2017 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Sun Sep 3 10:41:10 2017 +0800
----------------------------------------------------------------------
.travis.yml | 1 +
.../zeppelin/helium/ZeppelinDevServer.java | 8 +-
.../zeppelin/interpreter/Interpreter.java | 1 -
.../zeppelin/interpreter/InterpreterGroup.java | 237 +---
.../zeppelin/interpreter/InterpreterRunner.java | 9 +
.../interpreter/remote/AppendOutputBuffer.java | 54 -
.../interpreter/remote/AppendOutputRunner.java | 116 --
.../interpreter/remote/ClientFactory.java | 84 --
.../remote/InterpreterContextRunnerPool.java | 88 --
.../interpreter/remote/RemoteAngularObject.java | 53 -
.../remote/RemoteInterpreterEventPoller.java | 571 ---------
.../remote/RemoteInterpreterProcess.java | 242 ----
.../RemoteInterpreterProcessListener.java | 45 -
.../remote/RemoteInterpreterServer.java | 64 +-
.../zeppelin/resource/ResourcePoolUtils.java | 138 ---
.../java/org/apache/zeppelin/scheduler/Job.java | 11 +-
.../zeppelin/scheduler/RemoteScheduler.java | 426 -------
.../zeppelin/scheduler/SchedulerFactory.java | 63 +-
.../zeppelin/tabledata/TableDataProxy.java | 1 -
.../java/org/apache/zeppelin/util/IdHashes.java | 76 ++
.../java/org/apache/zeppelin/util/Util.java | 76 ++
.../zeppelin/interpreter/DummyInterpreter.java | 43 -
.../InterpreterOutputChangeWatcherTest.java | 11 +-
.../zeppelin/interpreter/InterpreterTest.java | 38 +
.../remote/RemoteInterpreterUtilsTest.java | 33 +
.../src/test/resources/conf/interpreter.json | 115 ++
.../interpreter/test/interpreter-setting.json | 42 +
.../src/test/resources/log4j.properties | 4 +-
.../zeppelin/rest/InterpreterRestApi.java | 4 +-
.../apache/zeppelin/server/ZeppelinServer.java | 37 +-
.../apache/zeppelin/socket/NotebookServer.java | 35 +-
.../interpreter/mock/MockInterpreter1.java | 75 --
.../zeppelin/rest/AbstractTestRestApi.java | 7 +-
.../zeppelin/rest/InterpreterRestApiTest.java | 6 +-
.../zeppelin/socket/NotebookServerTest.java | 13 +-
.../src/test/resources/log4j.properties | 1 -
.../zeppelin/conf/ZeppelinConfiguration.java | 20 +-
.../java/org/apache/zeppelin/helium/Helium.java | 54 +-
.../helium/HeliumApplicationFactory.java | 127 +-
.../interpreter/InterpreterFactory.java | 352 +-----
.../interpreter/InterpreterGroupFactory.java | 26 -
.../interpreter/InterpreterInfoSaving.java | 68 +-
.../interpreter/InterpreterSetting.java | 801 ++++++++++---
.../interpreter/InterpreterSettingManager.java | 1131 +++++++----------
.../interpreter/ManagedInterpreterGroup.java | 136 +++
.../interpreter/install/InstallInterpreter.java | 4 +-
.../interpreter/remote/AppendOutputBuffer.java | 54 +
.../interpreter/remote/AppendOutputRunner.java | 116 ++
.../interpreter/remote/ClientFactory.java | 84 ++
.../remote/InterpreterContextRunnerPool.java | 88 ++
.../interpreter/remote/RemoteAngularObject.java | 54 +
.../remote/RemoteAngularObjectRegistry.java | 83 +-
.../interpreter/remote/RemoteInterpreter.java | 667 ++++-------
.../remote/RemoteInterpreterEventPoller.java | 525 ++++++++
.../remote/RemoteInterpreterManagedProcess.java | 13 +
.../remote/RemoteInterpreterProcess.java | 168 +++
.../RemoteInterpreterProcessListener.java | 45 +
.../zeppelin/notebook/ApplicationState.java | 1 -
.../java/org/apache/zeppelin/notebook/Note.java | 15 +-
.../org/apache/zeppelin/notebook/Notebook.java | 29 +-
.../org/apache/zeppelin/notebook/Paragraph.java | 23 +-
.../zeppelin/scheduler/RemoteScheduler.java | 390 ++++++
.../java/org/apache/zeppelin/util/Util.java | 76 --
.../helium/HeliumApplicationFactoryTest.java | 97 +-
.../org/apache/zeppelin/helium/HeliumTest.java | 8 +-
.../interpreter/AbstractInterpreterTest.java | 74 ++
.../interpreter/DoubleEchoInterpreter.java | 59 +
.../zeppelin/interpreter/EchoInterpreter.java | 65 +
.../interpreter/InterpreterFactoryTest.java | 483 +-------
.../InterpreterSettingManagerTest.java | 286 +++++
.../interpreter/InterpreterSettingTest.java | 570 +++++----
.../ManagedInterpreterGroupTest.java | 90 ++
.../zeppelin/interpreter/SleepInterpreter.java | 60 +
.../interpreter/mock/MockInterpreter1.java | 15 +-
.../interpreter/mock/MockInterpreter11.java | 83 --
.../interpreter/mock/MockInterpreter2.java | 10 +-
.../remote/AppendOutputRunnerTest.java | 30 +-
.../remote/RemoteAngularObjectTest.java | 89 +-
.../RemoteInterpreterOutputTestStream.java | 67 +-
.../remote/RemoteInterpreterProcessTest.java | 131 --
.../remote/RemoteInterpreterTest.java | 1132 +++++-------------
.../remote/RemoteInterpreterUtilsTest.java | 34 -
.../mock/GetAngularObjectSizeInterpreter.java | 63 +
.../remote/mock/GetEnvPropertyInterpreter.java | 82 ++
.../remote/mock/MockInterpreterA.java | 7 +-
.../remote/mock/MockInterpreterAngular.java | 9 +-
.../remote/mock/MockInterpreterB.java | 126 --
.../remote/mock/MockInterpreterEnv.java | 80 --
.../mock/MockInterpreterOutputStream.java | 5 +-
.../mock/MockInterpreterResourcePool.java | 11 +-
.../notebook/NoteInterpreterLoaderTest.java | 243 ----
.../apache/zeppelin/notebook/NotebookTest.java | 174 ++-
.../apache/zeppelin/notebook/ParagraphTest.java | 10 +-
.../notebook/repo/NotebookRepoSyncTest.java | 75 +-
.../notebook/repo/VFSNotebookRepoTest.java | 64 +-
.../resource/DistributedResourcePoolTest.java | 93 +-
.../zeppelin/scheduler/RemoteSchedulerTest.java | 131 +-
.../src/test/resources/conf/interpreter.json | 115 ++
.../interpreter/mock/interpreter-setting.json | 12 -
.../interpreter/mock1/interpreter-setting.json | 19 +
.../interpreter/mock2/interpreter-setting.json | 19 +
.../mock_resource_pool/interpreter-setting.json | 19 +
.../interpreter/test/interpreter-setting.json | 53 +
.../src/test/resources/log4j.properties | 3 +-
104 files changed, 5774 insertions(+), 7000 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 97ca60a..64ea559 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -162,6 +162,7 @@ after_success:
after_failure:
- echo "Travis exited with ${TRAVIS_TEST_RESULT}"
- find . -name rat.txt | xargs cat
+ - cat logs/*
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.log
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
- cat zeppelin-web/npm-debug.log
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java
----------------------------------------------------------------------
diff --git a/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java b/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java
index 21ce283..2484469 100644
--- a/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java
+++ b/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java
@@ -43,13 +43,13 @@ public class ZeppelinDevServer extends
}
@Override
- protected Interpreter getInterpreter(String sessionKey, String className) throws TException {
+ protected Interpreter getInterpreter(String sessionId, String className) throws TException {
synchronized (this) {
InterpreterGroup interpreterGroup = getInterpreterGroup();
if (interpreterGroup == null || interpreterGroup.isEmpty()) {
createInterpreter(
"dev",
- sessionKey,
+ sessionId,
DevInterpreter.class.getName(),
new HashMap<String, String>(),
"anonymous");
@@ -57,11 +57,11 @@ public class ZeppelinDevServer extends
}
}
- Interpreter intp = super.getInterpreter(sessionKey, className);
+ Interpreter intp = super.getInterpreter(sessionId, className);
interpreter = (DevInterpreter) (
((LazyOpenInterpreter) intp).getInnerInterpreter());
interpreter.setInterpreterEvent(this);
- return super.getInterpreter(sessionKey, className);
+ return super.getInterpreter(sessionId, className);
}
@Override
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
index 74506dd..05599a0 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
@@ -149,7 +149,6 @@ public abstract class Interpreter {
@ZeppelinApi
public Interpreter(Properties property) {
- logger.debug("Properties: {}", property);
this.property = property;
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
index 5cbab6b..5428cdb 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
@@ -17,107 +17,90 @@
package org.apache.zeppelin.interpreter;
+import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.resource.ResourcePool;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
import java.util.Collection;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Properties;
import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
-import org.apache.zeppelin.resource.ResourcePool;
-import org.apache.zeppelin.scheduler.Scheduler;
-import org.apache.zeppelin.scheduler.SchedulerFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
/**
- * InterpreterGroup is list of interpreters in the same interpreter group.
- * For example spark, pyspark, sql interpreters are in the same 'spark' group
- * and InterpreterGroup will have reference to these all interpreters.
+ * InterpreterGroup is collections of interpreter sessions.
+ * One session could include multiple interpreters.
+ * For example spark, pyspark, sql interpreters are in the same 'spark' interpreter session.
*
* Remember, list of interpreters are dedicated to a session. Session could be shared across user
* or notes, so the sessionId could be user or noteId or their combination.
- * So InterpreterGroup internally manages map of [interpreterSessionKey(noteId, user, or
+ * So InterpreterGroup internally manages map of [sessionId(noteId, user, or
* their combination), list of interpreters]
*
- * A InterpreterGroup runs on interpreter process.
- * And unit of interpreter instantiate, restart, bind, unbind.
+ * A InterpreterGroup runs interpreter process while its subclass ManagedInterpreterGroup runs
+ * in zeppelin server process.
*/
-public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter>> {
- String id;
+public class InterpreterGroup {
private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterGroup.class);
- AngularObjectRegistry angularObjectRegistry;
- InterpreterHookRegistry hookRegistry;
- RemoteInterpreterProcess remoteInterpreterProcess; // attached remote interpreter process
- ResourcePool resourcePool;
- boolean angularRegistryPushed = false;
-
- // map [notebook session, Interpreters in the group], to support per note session interpreters
- //Map<String, List<Interpreter>> interpreters = new ConcurrentHashMap<String,
- // List<Interpreter>>();
-
- private static final Map<String, InterpreterGroup> allInterpreterGroups =
- new ConcurrentHashMap<>();
-
- public static InterpreterGroup getByInterpreterGroupId(String id) {
- return allInterpreterGroups.get(id);
- }
-
- public static Collection<InterpreterGroup> getAll() {
- return new LinkedList(allInterpreterGroups.values());
- }
+ protected String id;
+ // sessionId --> interpreters
+ protected Map<String, List<Interpreter>> sessions = new ConcurrentHashMap();
+ private AngularObjectRegistry angularObjectRegistry;
+ private InterpreterHookRegistry hookRegistry;
+ private ResourcePool resourcePool;
+ private boolean angularRegistryPushed = false;
/**
- * Create InterpreterGroup with given id
+ * Create InterpreterGroup with given id, used in InterpreterProcess
* @param id
*/
public InterpreterGroup(String id) {
this.id = id;
- allInterpreterGroups.put(id, this);
}
/**
* Create InterpreterGroup with autogenerated id
*/
public InterpreterGroup() {
- getId();
- allInterpreterGroups.put(id, this);
+ this.id = generateId();
}
private static String generateId() {
- return "InterpreterGroup_" + System.currentTimeMillis() + "_"
- + new Random().nextInt();
+ return "InterpreterGroup_" + System.currentTimeMillis() + "_" + new Random().nextInt();
}
public String getId() {
- synchronized (this) {
- if (id == null) {
- id = generateId();
- }
- return id;
- }
+ return this.id;
}
- /**
- * Get combined property of all interpreters in this group
- * @return
- */
- public Properties getProperty() {
- Properties p = new Properties();
+ //TODO(zjffdu) change it to getSession. For now just keep this method to reduce code change
+ public synchronized List<Interpreter> get(String sessionId) {
+ return sessions.get(sessionId);
+ }
- for (List<Interpreter> intpGroupForASession : this.values()) {
- for (Interpreter intp : intpGroupForASession) {
- p.putAll(intp.getProperty());
- }
- // it's okay to break here while every List<Interpreters> will have the same property set
- break;
+ //TODO(zjffdu) change it to addSession. For now just keep this method to reduce code change
+ public synchronized void put(String sessionId, List<Interpreter> interpreters) {
+ this.sessions.put(sessionId, interpreters);
+ }
+
+ public synchronized void addInterpreterToSession(Interpreter interpreter, String sessionId) {
+ LOGGER.debug("Add Interpreter {} to session {}", interpreter.getClassName(), sessionId);
+ List<Interpreter> interpreters = get(sessionId);
+ if (interpreters == null) {
+ interpreters = new ArrayList<>();
}
- return p;
+ interpreters.add(interpreter);
+ put(sessionId, interpreters);
+ }
+
+ //TODO(zjffdu) rename it to a more proper name.
+ //For now just keep this method to reduce code change
+ public Collection<List<Interpreter>> values() {
+ return sessions.values();
}
public AngularObjectRegistry getAngularObjectRegistry() {
@@ -136,128 +119,8 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
this.hookRegistry = hookRegistry;
}
- public RemoteInterpreterProcess getRemoteInterpreterProcess() {
- return remoteInterpreterProcess;
- }
-
- public void setRemoteInterpreterProcess(RemoteInterpreterProcess remoteInterpreterProcess) {
- this.remoteInterpreterProcess = remoteInterpreterProcess;
- }
-
- /**
- * Close all interpreter instances in this group
- */
- public void close() {
- LOGGER.info("Close interpreter group " + getId());
- List<Interpreter> intpToClose = new LinkedList<>();
- for (List<Interpreter> intpGroupForSession : this.values()) {
- intpToClose.addAll(intpGroupForSession);
- }
- close(intpToClose);
-
- // make sure remote interpreter process terminates
- if (remoteInterpreterProcess != null) {
- while (remoteInterpreterProcess.referenceCount() > 0) {
- remoteInterpreterProcess.dereference();
- }
- remoteInterpreterProcess = null;
- }
- allInterpreterGroups.remove(id);
- }
-
- /**
- * Close all interpreter instances in this group for the session
- * @param sessionId
- */
- public void close(String sessionId) {
- LOGGER.info("Close interpreter group " + getId() + " for session: " + sessionId);
- final List<Interpreter> intpForSession = this.get(sessionId);
-
- close(intpForSession);
- }
-
- private void close(final Collection<Interpreter> intpToClose) {
- close(null, null, null, intpToClose);
- }
-
- public void close(final Map<String, InterpreterGroup> interpreterGroupRef,
- final String processKey, final String sessionKey) {
- LOGGER.info("Close interpreter group " + getId() + " for session: " + sessionKey);
- close(interpreterGroupRef, processKey, sessionKey, this.get(sessionKey));
- }
-
- private void close(final Map<String, InterpreterGroup> interpreterGroupRef,
- final String processKey, final String sessionKey, final Collection<Interpreter> intpToClose) {
- if (intpToClose == null) {
- return;
- }
- Thread t = new Thread() {
- public void run() {
- for (Interpreter interpreter : intpToClose) {
- Scheduler scheduler = interpreter.getScheduler();
- interpreter.close();
-
- if (null != scheduler) {
- SchedulerFactory.singleton().removeScheduler(scheduler.getName());
- }
- }
-
- if (remoteInterpreterProcess != null) {
- //TODO(jl): Because interpreter.close() runs as a seprate thread, we cannot guarantee
- // refernceCount is a proper value. And as the same reason, we must not call
- // remoteInterpreterProcess.dereference twice - this method also be called by
- // interpreter.close().
-
- // remoteInterpreterProcess.dereference();
- if (remoteInterpreterProcess.referenceCount() <= 0) {
- remoteInterpreterProcess = null;
- allInterpreterGroups.remove(id);
- }
- }
-
- // TODO(jl): While closing interpreters in a same session, we should remove after all
- // interpreters are removed. OMG. It's too dirty!!
- if (null != interpreterGroupRef && null != processKey && null != sessionKey) {
- InterpreterGroup interpreterGroup = interpreterGroupRef.get(processKey);
- if (1 == interpreterGroup.size() && interpreterGroup.containsKey(sessionKey)) {
- interpreterGroupRef.remove(processKey);
- } else {
- interpreterGroup.remove(sessionKey);
- }
- }
- }
- };
-
- t.start();
- try {
- t.join();
- } catch (InterruptedException e) {
- LOGGER.error("Can't close interpreter: {}", getId(), e);
- }
-
-
- }
-
- /**
- * Close all interpreter instances in this group
- */
- public void shutdown() {
- LOGGER.info("Close interpreter group " + getId());
-
- // make sure remote interpreter process terminates
- if (remoteInterpreterProcess != null) {
- while (remoteInterpreterProcess.referenceCount() > 0) {
- remoteInterpreterProcess.dereference();
- }
- remoteInterpreterProcess = null;
- }
- allInterpreterGroups.remove(id);
-
- List<Interpreter> intpToClose = new LinkedList<>();
- for (List<Interpreter> intpGroupForSession : this.values()) {
- intpToClose.addAll(intpGroupForSession);
- }
- close(intpToClose);
+ public int getSessionNum() {
+ return sessions.size();
}
public void setResourcePool(ResourcePool resourcePool) {
@@ -275,4 +138,8 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
public void setAngularRegistryPushed(boolean angularRegistryPushed) {
this.angularRegistryPushed = angularRegistryPushed;
}
+
+ public boolean isEmpty() {
+ return sessions.isEmpty();
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java
index 020564b..e60ada7 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java
@@ -12,6 +12,15 @@ public class InterpreterRunner {
@SerializedName("win")
private String winPath;
+ public InterpreterRunner() {
+
+ }
+
+ public InterpreterRunner(String linuxPath, String winPath) {
+ this.linuxPath = linuxPath;
+ this.winPath = winPath;
+ }
+
public String getPath() {
return System.getProperty("os.name").startsWith("Windows") ? winPath : linuxPath;
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputBuffer.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputBuffer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputBuffer.java
deleted file mode 100644
index b139404..0000000
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputBuffer.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote;
-
-/**
- * This element stores the buffered
- * append-data of paragraph's output.
- */
-public class AppendOutputBuffer {
-
- private String noteId;
- private String paragraphId;
- private int index;
- private String data;
-
- public AppendOutputBuffer(String noteId, String paragraphId, int index, String data) {
- this.noteId = noteId;
- this.paragraphId = paragraphId;
- this.index = index;
- this.data = data;
- }
-
- public String getNoteId() {
- return noteId;
- }
-
- public String getParagraphId() {
- return paragraphId;
- }
-
- public int getIndex() {
- return index;
- }
-
- public String getData() {
- return data;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunner.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunner.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunner.java
deleted file mode 100644
index 03d9191..0000000
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunner.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote;
-
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This thread sends paragraph's append-data
- * periodically, rather than continously, with
- * a period of BUFFER_TIME_MS. It handles append-data
- * for all paragraphs across all notebooks.
- */
-public class AppendOutputRunner implements Runnable {
-
- private static final Logger logger =
- LoggerFactory.getLogger(AppendOutputRunner.class);
- public static final Long BUFFER_TIME_MS = new Long(100);
- private static final Long SAFE_PROCESSING_TIME = new Long(10);
- private static final Long SAFE_PROCESSING_STRING_SIZE = new Long(100000);
-
- private final BlockingQueue<AppendOutputBuffer> queue = new LinkedBlockingQueue<>();
- private final RemoteInterpreterProcessListener listener;
-
- public AppendOutputRunner(RemoteInterpreterProcessListener listener) {
- this.listener = listener;
- }
-
- @Override
- public void run() {
-
- Map<String, StringBuilder> stringBufferMap = new HashMap<>();
- List<AppendOutputBuffer> list = new LinkedList<>();
-
- /* "drainTo" method does not wait for any element
- * to be present in the queue, and thus this loop would
- * continuosly run (with period of BUFFER_TIME_MS). "take()" method
- * waits for the queue to become non-empty and then removes
- * one element from it. Rest elements from queue (if present) are
- * removed using "drainTo" method. Thus we save on some un-necessary
- * cpu-cycles.
- */
- try {
- list.add(queue.take());
- } catch (InterruptedException e) {
- logger.error("Wait for OutputBuffer queue interrupted: " + e.getMessage());
- }
- Long processingStartTime = System.currentTimeMillis();
- queue.drainTo(list);
-
- for (AppendOutputBuffer buffer: list) {
- String noteId = buffer.getNoteId();
- String paragraphId = buffer.getParagraphId();
- int index = buffer.getIndex();
- String stringBufferKey = noteId + ":" + paragraphId + ":" + index;
-
- StringBuilder builder = stringBufferMap.containsKey(stringBufferKey) ?
- stringBufferMap.get(stringBufferKey) : new StringBuilder();
-
- builder.append(buffer.getData());
- stringBufferMap.put(stringBufferKey, builder);
- }
- Long processingTime = System.currentTimeMillis() - processingStartTime;
-
- if (processingTime > SAFE_PROCESSING_TIME) {
- logger.warn("Processing time for buffered append-output is high: " +
- processingTime + " milliseconds.");
- } else {
- logger.debug("Processing time for append-output took "
- + processingTime + " milliseconds");
- }
-
- Long sizeProcessed = new Long(0);
- for (String stringBufferKey : stringBufferMap.keySet()) {
- StringBuilder buffer = stringBufferMap.get(stringBufferKey);
- sizeProcessed += buffer.length();
- String[] keys = stringBufferKey.split(":");
- listener.onOutputAppend(keys[0], keys[1], Integer.parseInt(keys[2]), buffer.toString());
- }
-
- if (sizeProcessed > SAFE_PROCESSING_STRING_SIZE) {
- logger.warn("Processing size for buffered append-output is high: " +
- sizeProcessed + " characters.");
- } else {
- logger.debug("Processing size for append-output is " +
- sizeProcessed + " characters");
- }
- }
-
- public void appendBuffer(String noteId, String paragraphId, int index, String outputToAppend) {
- queue.offer(new AppendOutputBuffer(noteId, paragraphId, index, outputToAppend));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java
deleted file mode 100644
index b2cb78f..0000000
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.commons.pool2.BasePooledObjectFactory;
-import org.apache.commons.pool2.PooledObject;
-import org.apache.commons.pool2.impl.DefaultPooledObject;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.protocol.TProtocol;
-import org.apache.thrift.transport.TSocket;
-import org.apache.thrift.transport.TTransportException;
-import org.apache.zeppelin.interpreter.InterpreterException;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
-
-/**
- *
- */
-public class ClientFactory extends BasePooledObjectFactory<Client>{
- private String host;
- private int port;
- Map<Client, TSocket> clientSocketMap = new HashMap<>();
-
- public ClientFactory(String host, int port) {
- this.host = host;
- this.port = port;
- }
-
- @Override
- public Client create() throws Exception {
- TSocket transport = new TSocket(host, port);
- try {
- transport.open();
- } catch (TTransportException e) {
- throw new InterpreterException(e);
- }
-
- TProtocol protocol = new TBinaryProtocol(transport);
- Client client = new RemoteInterpreterService.Client(protocol);
-
- synchronized (clientSocketMap) {
- clientSocketMap.put(client, transport);
- }
- return client;
- }
-
- @Override
- public PooledObject<Client> wrap(Client client) {
- return new DefaultPooledObject<>(client);
- }
-
- @Override
- public void destroyObject(PooledObject<Client> p) {
- synchronized (clientSocketMap) {
- if (clientSocketMap.containsKey(p.getObject())) {
- clientSocketMap.get(p.getObject()).close();
- clientSocketMap.remove(p.getObject());
- }
- }
- }
-
- @Override
- public boolean validateObject(PooledObject<Client> p) {
- return p.getObject().getOutputProtocol().getTransport().isOpen();
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java
deleted file mode 100644
index 064abd5..0000000
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote;
-
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.zeppelin.interpreter.InterpreterContextRunner;
-import org.apache.zeppelin.interpreter.InterpreterException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- */
-public class InterpreterContextRunnerPool {
- Logger logger = LoggerFactory.getLogger(InterpreterContextRunnerPool.class);
- private Map<String, List<InterpreterContextRunner>> interpreterContextRunners;
-
- public InterpreterContextRunnerPool() {
- interpreterContextRunners = new HashMap<>();
-
- }
-
- // add runner
- public void add(String noteId, InterpreterContextRunner runner) {
- synchronized (interpreterContextRunners) {
- if (!interpreterContextRunners.containsKey(noteId)) {
- interpreterContextRunners.put(noteId, new LinkedList<InterpreterContextRunner>());
- }
-
- interpreterContextRunners.get(noteId).add(runner);
- }
- }
-
- // replace all runners to noteId
- public void addAll(String noteId, List<InterpreterContextRunner> runners) {
- synchronized (interpreterContextRunners) {
- if (!interpreterContextRunners.containsKey(noteId)) {
- interpreterContextRunners.put(noteId, new LinkedList<InterpreterContextRunner>());
- }
-
- interpreterContextRunners.get(noteId).addAll(runners);
- }
- }
-
- public void clear(String noteId) {
- synchronized (interpreterContextRunners) {
- interpreterContextRunners.remove(noteId);
- }
- }
-
-
- public void run(String noteId, String paragraphId) {
- synchronized (interpreterContextRunners) {
- List<InterpreterContextRunner> list = interpreterContextRunners.get(noteId);
- if (list != null) {
- for (InterpreterContextRunner r : list) {
- if (noteId.equals(r.getNoteId()) && paragraphId.equals(r.getParagraphId())) {
- logger.info("run paragraph {} on note {} from InterpreterContext",
- r.getParagraphId(), r.getNoteId());
- r.run();
- return;
- }
- }
- }
-
- throw new InterpreterException("Can not run paragraph " + paragraphId + " on " + noteId);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObject.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObject.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObject.java
deleted file mode 100644
index c1f9b94..0000000
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObject.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote;
-
-import org.apache.zeppelin.display.AngularObject;
-import org.apache.zeppelin.display.AngularObjectListener;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
-
-/**
- * Proxy for AngularObject that exists in remote interpreter process
- */
-public class RemoteAngularObject extends AngularObject {
-
- private transient InterpreterGroup interpreterGroup;
-
- RemoteAngularObject(String name, Object o, String noteId, String paragraphId,
- InterpreterGroup interpreterGroup,
- AngularObjectListener listener) {
- super(name, o, noteId, paragraphId, listener);
- this.interpreterGroup = interpreterGroup;
- }
-
- @Override
- public void set(Object o, boolean emit) {
- set(o, emit, true);
- }
-
- public void set(Object o, boolean emitWeb, boolean emitRemoteProcess) {
- super.set(o, emitWeb);
-
- if (emitRemoteProcess) {
- // send updated value to remote interpreter
- interpreterGroup.getRemoteInterpreterProcess().
- updateRemoteAngularObject(
- getName(), getNoteId(), getParagraphId(), o);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
deleted file mode 100644
index 26c9d79..0000000
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
+++ /dev/null
@@ -1,571 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import org.apache.thrift.TException;
-import org.apache.zeppelin.display.AngularObject;
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.helium.ApplicationEventListener;
-import org.apache.zeppelin.interpreter.InterpreterContextRunner;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.RemoteZeppelinServerResource;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
-import org.apache.zeppelin.interpreter.thrift.ZeppelinServerResourceParagraphRunner;
-import org.apache.zeppelin.resource.Resource;
-import org.apache.zeppelin.resource.ResourceId;
-import org.apache.zeppelin.resource.ResourcePool;
-import org.apache.zeppelin.resource.ResourceSet;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.nio.ByteBuffer;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Processes message from RemoteInterpreter process
- */
-public class RemoteInterpreterEventPoller extends Thread {
- private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterEventPoller.class);
- private final ScheduledExecutorService appendService =
- Executors.newSingleThreadScheduledExecutor();
- private final RemoteInterpreterProcessListener listener;
- private final ApplicationEventListener appListener;
-
- private volatile boolean shutdown;
-
- private RemoteInterpreterProcess interpreterProcess;
- private InterpreterGroup interpreterGroup;
-
- Gson gson = new Gson();
-
- public RemoteInterpreterEventPoller(
- RemoteInterpreterProcessListener listener,
- ApplicationEventListener appListener) {
- this.listener = listener;
- this.appListener = appListener;
- shutdown = false;
- }
-
- public void setInterpreterProcess(RemoteInterpreterProcess interpreterProcess) {
- this.interpreterProcess = interpreterProcess;
- }
-
- public void setInterpreterGroup(InterpreterGroup interpreterGroup) {
- this.interpreterGroup = interpreterGroup;
- }
-
- @Override
- public void run() {
- Client client = null;
- AppendOutputRunner runner = new AppendOutputRunner(listener);
- ScheduledFuture<?> appendFuture = appendService.scheduleWithFixedDelay(
- runner, 0, AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
-
- while (!shutdown) {
- // wait and retry
- if (!interpreterProcess.isRunning()) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- // nothing to do
- }
- continue;
- }
-
- try {
- client = interpreterProcess.getClient();
- } catch (Exception e1) {
- logger.error("Can't get RemoteInterpreterEvent", e1);
- waitQuietly();
- continue;
- }
-
- RemoteInterpreterEvent event = null;
- boolean broken = false;
- try {
- event = client.getEvent();
- } catch (TException e) {
- broken = true;
- logger.error("Can't get RemoteInterpreterEvent", e);
- waitQuietly();
- continue;
- } finally {
- interpreterProcess.releaseClient(client, broken);
- }
-
- AngularObjectRegistry angularObjectRegistry = interpreterGroup.getAngularObjectRegistry();
-
- try {
- if (event.getType() != RemoteInterpreterEventType.NO_OP) {
- logger.debug("Receive message from RemoteInterpreter Process: " + event.toString());
- }
- if (event.getType() == RemoteInterpreterEventType.NO_OP) {
- continue;
- } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_ADD) {
- AngularObject angularObject = AngularObject.fromJson(event.getData());
- angularObjectRegistry.add(angularObject.getName(),
- angularObject.get(), angularObject.getNoteId(), angularObject.getParagraphId());
- } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_UPDATE) {
- AngularObject angularObject = AngularObject.fromJson(event.getData());
- AngularObject localAngularObject = angularObjectRegistry.get(
- angularObject.getName(), angularObject.getNoteId(), angularObject.getParagraphId());
- if (localAngularObject instanceof RemoteAngularObject) {
- // to avoid ping-pong loop
- ((RemoteAngularObject) localAngularObject).set(
- angularObject.get(), true, false);
- } else {
- localAngularObject.set(angularObject.get());
- }
- } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_REMOVE) {
- AngularObject angularObject = AngularObject.fromJson(event.getData());
- angularObjectRegistry.remove(angularObject.getName(), angularObject.getNoteId(),
- angularObject.getParagraphId());
- } else if (event.getType() == RemoteInterpreterEventType.RUN_INTERPRETER_CONTEXT_RUNNER) {
- InterpreterContextRunner runnerFromRemote = gson.fromJson(
- event.getData(), RemoteInterpreterContextRunner.class);
-
- listener.onRemoteRunParagraph(
- runnerFromRemote.getNoteId(), runnerFromRemote.getParagraphId());
-
- } else if (event.getType() == RemoteInterpreterEventType.RESOURCE_POOL_GET_ALL) {
- ResourceSet resourceSet = getAllResourcePoolExcept();
- sendResourcePoolResponseGetAll(resourceSet);
- } else if (event.getType() == RemoteInterpreterEventType.RESOURCE_GET) {
- String resourceIdString = event.getData();
- ResourceId resourceId = ResourceId.fromJson(resourceIdString);
- logger.debug("RESOURCE_GET {} {}", resourceId.getResourcePoolId(), resourceId.getName());
- Object o = getResource(resourceId);
- sendResourceResponseGet(resourceId, o);
- } else if (event.getType() == RemoteInterpreterEventType.RESOURCE_INVOKE_METHOD) {
- String message = event.getData();
- InvokeResourceMethodEventMessage invokeMethodMessage =
- InvokeResourceMethodEventMessage.fromJson(message);
- Object ret = invokeResourceMethod(invokeMethodMessage);
- sendInvokeMethodResult(invokeMethodMessage, ret);
- } else if (event.getType() == RemoteInterpreterEventType.OUTPUT_APPEND) {
- // on output append
- Map<String, String> outputAppend = gson.fromJson(
- event.getData(), new TypeToken<Map<String, Object>>() {}.getType());
- String noteId = (String) outputAppend.get("noteId");
- String paragraphId = (String) outputAppend.get("paragraphId");
- int index = Integer.parseInt(outputAppend.get("index"));
- String outputToAppend = (String) outputAppend.get("data");
-
- String appId = (String) outputAppend.get("appId");
-
- if (appId == null) {
- runner.appendBuffer(noteId, paragraphId, index, outputToAppend);
- } else {
- appListener.onOutputAppend(noteId, paragraphId, index, appId, outputToAppend);
- }
- } else if (event.getType() == RemoteInterpreterEventType.OUTPUT_UPDATE_ALL) {
- Map<String, Object> outputUpdate = gson.fromJson(
- event.getData(), new TypeToken<Map<String, Object>>() {}.getType());
- String noteId = (String) outputUpdate.get("noteId");
- String paragraphId = (String) outputUpdate.get("paragraphId");
-
- // clear the output
- List<Map<String, String>> messages =
- (List<Map<String, String>>) outputUpdate.get("messages");
-
- if (messages != null) {
- listener.onOutputClear(noteId, paragraphId);
- for (int i = 0; i < messages.size(); i++) {
- Map<String, String> m = messages.get(i);
- InterpreterResult.Type type =
- InterpreterResult.Type.valueOf((String) m.get("type"));
- String outputToUpdate = (String) m.get("data");
-
- listener.onOutputUpdated(noteId, paragraphId, i, type, outputToUpdate);
- }
- }
- } else if (event.getType() == RemoteInterpreterEventType.OUTPUT_UPDATE) {
- // on output update
- Map<String, String> outputAppend = gson.fromJson(
- event.getData(), new TypeToken<Map<String, Object>>() {}.getType());
- String noteId = (String) outputAppend.get("noteId");
- String paragraphId = (String) outputAppend.get("paragraphId");
- int index = Integer.parseInt(outputAppend.get("index"));
- InterpreterResult.Type type =
- InterpreterResult.Type.valueOf((String) outputAppend.get("type"));
- String outputToUpdate = (String) outputAppend.get("data");
- String appId = (String) outputAppend.get("appId");
-
- if (appId == null) {
- listener.onOutputUpdated(noteId, paragraphId, index, type, outputToUpdate);
- } else {
- appListener.onOutputUpdated(noteId, paragraphId, index, appId, type, outputToUpdate);
- }
- } else if (event.getType() == RemoteInterpreterEventType.APP_STATUS_UPDATE) {
- // on output update
- Map<String, String> appStatusUpdate = gson.fromJson(
- event.getData(), new TypeToken<Map<String, String>>() {}.getType());
-
- String noteId = appStatusUpdate.get("noteId");
- String paragraphId = appStatusUpdate.get("paragraphId");
- String appId = appStatusUpdate.get("appId");
- String status = appStatusUpdate.get("status");
-
- appListener.onStatusChange(noteId, paragraphId, appId, status);
- } else if (event.getType() == RemoteInterpreterEventType.REMOTE_ZEPPELIN_SERVER_RESOURCE) {
- RemoteZeppelinServerResource reqResourceBody = RemoteZeppelinServerResource.fromJson(
- event.getData());
- progressRemoteZeppelinControlEvent(
- reqResourceBody.getResourceType(), listener, reqResourceBody);
-
- } else if (event.getType() == RemoteInterpreterEventType.META_INFOS) {
- Map<String, String> metaInfos = gson.fromJson(event.getData(),
- new TypeToken<Map<String, String>>() {
- }.getType());
- String settingId = RemoteInterpreterUtils.
- getInterpreterSettingId(interpreterGroup.getId());
- listener.onMetaInfosReceived(settingId, metaInfos);
- } else if (event.getType() == RemoteInterpreterEventType.PARA_INFOS) {
- Map<String, String> paraInfos = gson.fromJson(event.getData(),
- new TypeToken<Map<String, String>>() {
- }.getType());
- String noteId = paraInfos.get("noteId");
- String paraId = paraInfos.get("paraId");
- String settingId = RemoteInterpreterUtils.
- getInterpreterSettingId(interpreterGroup.getId());
- if (noteId != null && paraId != null && settingId != null) {
- listener.onParaInfosReceived(noteId, paraId, settingId, paraInfos);
- }
- }
- logger.debug("Event from remote process {}", event.getType());
- } catch (Exception e) {
- logger.error("Can't handle event " + event, e);
- }
- }
- try {
- clearUnreadEvents(interpreterProcess.getClient());
- } catch (Exception e1) {
- logger.error("Can't get RemoteInterpreterEvent", e1);
- }
- if (appendFuture != null) {
- appendFuture.cancel(true);
- }
- }
-
- private void clearUnreadEvents(Client client) throws TException {
- while (client.getEvent().getType() != RemoteInterpreterEventType.NO_OP) {}
- }
-
- private void progressRemoteZeppelinControlEvent(
- RemoteZeppelinServerResource.Type resourceType,
- RemoteInterpreterProcessListener remoteWorksEventListener,
- RemoteZeppelinServerResource reqResourceBody) throws Exception {
- boolean broken = false;
- final Gson gson = new Gson();
- final String eventOwnerKey = reqResourceBody.getOwnerKey();
- Client interpreterServerMain = null;
- try {
- interpreterServerMain = interpreterProcess.getClient();
- final Client eventClient = interpreterServerMain;
- if (resourceType == RemoteZeppelinServerResource.Type.PARAGRAPH_RUNNERS) {
- final List<ZeppelinServerResourceParagraphRunner> remoteRunners = new LinkedList<>();
-
- ZeppelinServerResourceParagraphRunner reqRunnerContext =
- new ZeppelinServerResourceParagraphRunner();
-
- Map<String, Object> reqResourceMap = (Map<String, Object>) reqResourceBody.getData();
- String noteId = (String) reqResourceMap.get("noteId");
- String paragraphId = (String) reqResourceMap.get("paragraphId");
-
- reqRunnerContext.setNoteId(noteId);
- reqRunnerContext.setParagraphId(paragraphId);
-
- RemoteInterpreterProcessListener.RemoteWorksEventListener callBackEvent =
- new RemoteInterpreterProcessListener.RemoteWorksEventListener() {
-
- @Override
- public void onFinished(Object resultObject) {
- boolean clientBroken = false;
- if (resultObject != null && resultObject instanceof List) {
- List<InterpreterContextRunner> runnerList =
- (List<InterpreterContextRunner>) resultObject;
- for (InterpreterContextRunner r : runnerList) {
- remoteRunners.add(
- new ZeppelinServerResourceParagraphRunner(r.getNoteId(), r.getParagraphId())
- );
- }
-
- final RemoteZeppelinServerResource resResource =
- new RemoteZeppelinServerResource();
- resResource.setOwnerKey(eventOwnerKey);
- resResource.setResourceType(RemoteZeppelinServerResource.Type.PARAGRAPH_RUNNERS);
- resResource.setData(remoteRunners);
-
- try {
- eventClient.onReceivedZeppelinResource(resResource.toJson());
- } catch (Exception e) {
- clientBroken = true;
- logger.error("Can't get RemoteInterpreterEvent", e);
- waitQuietly();
- } finally {
- interpreterProcess.releaseClient(eventClient, clientBroken);
- }
- }
- }
-
- @Override
- public void onError() {
- logger.info("onGetParagraphRunners onError");
- }
- };
-
- remoteWorksEventListener.onGetParagraphRunners(
- reqRunnerContext.getNoteId(), reqRunnerContext.getParagraphId(), callBackEvent);
- }
- } catch (Exception e) {
- broken = true;
- logger.error("Can't get RemoteInterpreterEvent", e);
- waitQuietly();
-
- } finally {
- interpreterProcess.releaseClient(interpreterServerMain, broken);
- }
- }
-
- private void sendResourcePoolResponseGetAll(ResourceSet resourceSet) {
- Client client = null;
- boolean broken = false;
- try {
- client = interpreterProcess.getClient();
- List<String> resourceList = new LinkedList<>();
- Gson gson = new Gson();
- for (Resource r : resourceSet) {
- resourceList.add(gson.toJson(r));
- }
- client.resourcePoolResponseGetAll(resourceList);
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- broken = true;
- } finally {
- if (client != null) {
- interpreterProcess.releaseClient(client, broken);
- }
- }
- }
-
- private ResourceSet getAllResourcePoolExcept() {
- ResourceSet resourceSet = new ResourceSet();
- for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
- if (intpGroup.getId().equals(interpreterGroup.getId())) {
- continue;
- }
-
- RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
- if (remoteInterpreterProcess == null) {
- ResourcePool localPool = intpGroup.getResourcePool();
- if (localPool != null) {
- resourceSet.addAll(localPool.getAll());
- }
- } else if (interpreterProcess.isRunning()) {
- Client client = null;
- boolean broken = false;
- try {
- client = remoteInterpreterProcess.getClient();
- List<String> resourceList = client.resourcePoolGetAll();
- Gson gson = new Gson();
- for (String res : resourceList) {
- resourceSet.add(Resource.fromJson(res));
- }
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- broken = true;
- } finally {
- if (client != null) {
- intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
- }
- }
- }
- }
- return resourceSet;
- }
-
- private void sendResourceResponseGet(ResourceId resourceId, Object o) {
- Client client = null;
- boolean broken = false;
- try {
- client = interpreterProcess.getClient();
- Gson gson = new Gson();
- String rid = gson.toJson(resourceId);
- ByteBuffer obj;
- if (o == null) {
- obj = ByteBuffer.allocate(0);
- } else {
- obj = Resource.serializeObject(o);
- }
- client.resourceResponseGet(rid, obj);
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- broken = true;
- } finally {
- if (client != null) {
- interpreterProcess.releaseClient(client, broken);
- }
- }
- }
-
- private Object getResource(ResourceId resourceId) {
- InterpreterGroup intpGroup = InterpreterGroup.getByInterpreterGroupId(
- resourceId.getResourcePoolId());
- if (intpGroup == null) {
- return null;
- }
- RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
- if (remoteInterpreterProcess == null) {
- ResourcePool localPool = intpGroup.getResourcePool();
- if (localPool != null) {
- return localPool.get(resourceId.getName());
- }
- } else if (interpreterProcess.isRunning()) {
- Client client = null;
- boolean broken = false;
- try {
- client = remoteInterpreterProcess.getClient();
- ByteBuffer res = client.resourceGet(
- resourceId.getNoteId(),
- resourceId.getParagraphId(),
- resourceId.getName());
- Object o = Resource.deserializeObject(res);
- return o;
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- broken = true;
- } finally {
- if (client != null) {
- intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
- }
- }
- }
- return null;
- }
-
- public void sendInvokeMethodResult(InvokeResourceMethodEventMessage message, Object o) {
- Client client = null;
- boolean broken = false;
- try {
- client = interpreterProcess.getClient();
- Gson gson = new Gson();
- String invokeMessage = gson.toJson(message);
- ByteBuffer obj;
- if (o == null) {
- obj = ByteBuffer.allocate(0);
- } else {
- obj = Resource.serializeObject(o);
- }
- client.resourceResponseInvokeMethod(invokeMessage, obj);
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- broken = true;
- } finally {
- if (client != null) {
- interpreterProcess.releaseClient(client, broken);
- }
- }
- }
-
- private Object invokeResourceMethod(InvokeResourceMethodEventMessage message) {
- ResourceId resourceId = message.resourceId;
- InterpreterGroup intpGroup = InterpreterGroup.getByInterpreterGroupId(
- resourceId.getResourcePoolId());
- if (intpGroup == null) {
- return null;
- }
-
- RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
- if (remoteInterpreterProcess == null) {
- ResourcePool localPool = intpGroup.getResourcePool();
- if (localPool != null) {
- Resource res = localPool.get(resourceId.getName());
- if (res != null) {
- try {
- return res.invokeMethod(
- message.methodName,
- message.getParamTypes(),
- message.params,
- message.returnResourceName);
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- return null;
- }
- } else {
- // object is null. can't invoke any method
- logger.error("Can't invoke method {} on null object", message.methodName);
- return null;
- }
- } else {
- logger.error("no resource pool");
- return null;
- }
- } else if (interpreterProcess.isRunning()) {
- Client client = null;
- boolean broken = false;
- try {
- client = remoteInterpreterProcess.getClient();
- ByteBuffer res = client.resourceInvokeMethod(
- resourceId.getNoteId(),
- resourceId.getParagraphId(),
- resourceId.getName(),
- gson.toJson(message));
- Object o = Resource.deserializeObject(res);
- return o;
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- broken = true;
- } finally {
- if (client != null) {
- intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
- }
- }
- }
- return null;
- }
-
- private void waitQuietly() {
- try {
- synchronized (this) {
- wait(1000);
- }
- } catch (InterruptedException ignored) {
- logger.info("Error in RemoteInterpreterEventPoller while waitQuietly : ", ignored);
- }
- }
-
- public void shutdown() {
- shutdown = true;
- synchronized (this) {
- notify();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
deleted file mode 100644
index 1d48a1e..0000000
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.zeppelin.interpreter.remote;
-
-import com.google.gson.Gson;
-import org.apache.commons.pool2.impl.GenericObjectPool;
-import org.apache.thrift.TException;
-import org.apache.zeppelin.helium.ApplicationEventListener;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Abstract class for interpreter process
- */
-public abstract class RemoteInterpreterProcess {
- private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterProcess.class);
-
- // number of sessions that are attached to this process
- private final AtomicInteger referenceCount;
-
- private GenericObjectPool<Client> clientPool;
- private final RemoteInterpreterEventPoller remoteInterpreterEventPoller;
- private final InterpreterContextRunnerPool interpreterContextRunnerPool;
- private int connectTimeout;
-
- public RemoteInterpreterProcess(
- int connectTimeout,
- RemoteInterpreterProcessListener listener,
- ApplicationEventListener appListener) {
- this(new RemoteInterpreterEventPoller(listener, appListener),
- connectTimeout);
- }
-
- RemoteInterpreterProcess(RemoteInterpreterEventPoller remoteInterpreterEventPoller,
- int connectTimeout) {
- this.interpreterContextRunnerPool = new InterpreterContextRunnerPool();
- referenceCount = new AtomicInteger(0);
- this.remoteInterpreterEventPoller = remoteInterpreterEventPoller;
- this.connectTimeout = connectTimeout;
- }
-
- public abstract String getHost();
- public abstract int getPort();
- public abstract void start(String userName, Boolean isUserImpersonate);
- public abstract void stop();
- public abstract boolean isRunning();
-
- public int getConnectTimeout() {
- return connectTimeout;
- }
-
- public int reference(InterpreterGroup interpreterGroup, String userName,
- Boolean isUserImpersonate) {
- synchronized (referenceCount) {
- if (!isRunning()) {
- start(userName, isUserImpersonate);
- }
-
- if (clientPool == null) {
- clientPool = new GenericObjectPool<>(new ClientFactory(getHost(), getPort()));
- clientPool.setTestOnBorrow(true);
-
- remoteInterpreterEventPoller.setInterpreterGroup(interpreterGroup);
- remoteInterpreterEventPoller.setInterpreterProcess(this);
- remoteInterpreterEventPoller.start();
- }
- return referenceCount.incrementAndGet();
- }
- }
-
- public Client getClient() throws Exception {
- if (clientPool == null || clientPool.isClosed()) {
- return null;
- }
- return clientPool.borrowObject();
- }
-
- public void releaseClient(Client client) {
- releaseClient(client, false);
- }
-
- public void releaseClient(Client client, boolean broken) {
- if (broken) {
- releaseBrokenClient(client);
- } else {
- try {
- clientPool.returnObject(client);
- } catch (Exception e) {
- logger.warn("exception occurred during releasing thrift client", e);
- }
- }
- }
-
- public void releaseBrokenClient(Client client) {
- try {
- clientPool.invalidateObject(client);
- } catch (Exception e) {
- logger.warn("exception occurred during releasing thrift client", e);
- }
- }
-
- public int dereference() {
- synchronized (referenceCount) {
- int r = referenceCount.decrementAndGet();
- if (r == 0) {
- logger.info("shutdown interpreter process");
- remoteInterpreterEventPoller.shutdown();
-
- // first try shutdown
- Client client = null;
- try {
- client = getClient();
- client.shutdown();
- } catch (Exception e) {
- // safely ignore exception while client.shutdown() may terminates remote process
- logger.info("Exception in RemoteInterpreterProcess while synchronized dereference, can " +
- "safely ignore exception while client.shutdown() may terminates remote process");
- logger.debug(e.getMessage(), e);
- } finally {
- if (client != null) {
- // no longer used
- releaseBrokenClient(client);
- }
- }
-
- clientPool.clear();
- clientPool.close();
-
- // wait for some time (connectTimeout) and force kill
- // remote process server.serve() loop is not always finishing gracefully
- long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() - startTime < connectTimeout) {
- if (this.isRunning()) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- logger.error("Exception in RemoteInterpreterProcess while synchronized dereference " +
- "Thread.sleep", e);
- }
- } else {
- break;
- }
- }
- }
- return r;
- }
- }
-
- public int referenceCount() {
- synchronized (referenceCount) {
- return referenceCount.get();
- }
- }
-
- public int getNumActiveClient() {
- if (clientPool == null) {
- return 0;
- } else {
- return clientPool.getNumActive();
- }
- }
-
- public int getNumIdleClient() {
- if (clientPool == null) {
- return 0;
- } else {
- return clientPool.getNumIdle();
- }
- }
-
- public void setMaxPoolSize(int size) {
- if (clientPool != null) {
- //Size + 2 for progress poller , cancel operation
- clientPool.setMaxTotal(size + 2);
- }
- }
-
- public int getMaxPoolSize() {
- if (clientPool != null) {
- return clientPool.getMaxTotal();
- } else {
- return 0;
- }
- }
-
- /**
- * Called when angular object is updated in client side to propagate
- * change to the remote process
- * @param name
- * @param o
- */
- public void updateRemoteAngularObject(String name, String noteId, String paragraphId, Object o) {
- Client client = null;
- try {
- client = getClient();
- } catch (NullPointerException e) {
- // remote process not started
- logger.info("NullPointerException in RemoteInterpreterProcess while " +
- "updateRemoteAngularObject getClient, remote process not started", e);
- return;
- } catch (Exception e) {
- logger.error("Can't update angular object", e);
- }
-
- boolean broken = false;
- try {
- Gson gson = new Gson();
- client.angularObjectUpdate(name, noteId, paragraphId, gson.toJson(o));
- } catch (TException e) {
- broken = true;
- logger.error("Can't update angular object", e);
- } catch (NullPointerException e) {
- logger.error("Remote interpreter process not started", e);
- return;
- } finally {
- if (client != null) {
- releaseClient(client, broken);
- }
- }
- }
-
- public InterpreterContextRunnerPool getInterpreterContextRunnerPool() {
- return interpreterContextRunnerPool;
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessListener.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessListener.java
deleted file mode 100644
index 0e9dc51..0000000
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessListener.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.zeppelin.interpreter.remote;
-
-import org.apache.zeppelin.interpreter.InterpreterResult;
-
-import java.util.Map;
-
-/**
- * Event from remoteInterpreterProcess
- */
-public interface RemoteInterpreterProcessListener {
- public void onOutputAppend(String noteId, String paragraphId, int index, String output);
- public void onOutputUpdated(
- String noteId, String paragraphId, int index, InterpreterResult.Type type, String output);
- public void onOutputClear(String noteId, String paragraphId);
- public void onMetaInfosReceived(String settingId, Map<String, String> metaInfos);
- public void onRemoteRunParagraph(String noteId, String ParagraphID) throws Exception;
- public void onGetParagraphRunners(
- String noteId, String paragraphId, RemoteWorksEventListener callback);
-
- /**
- * Remote works for Interpreter callback listener
- */
- public interface RemoteWorksEventListener {
- public void onFinished(Object resultObject);
- public void onError();
- }
- public void onParaInfosReceived(String noteId, String paragraphId,
- String interpreterSettingId, Map<String, String> metaInfos);
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
index 3853468..6925360 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
@@ -106,9 +106,14 @@ public class RemoteInterpreterServer
@Override
public void shutdown() throws TException {
+ logger.info("Shutting down...");
eventClient.waitForEventQueueBecomesEmpty(DEFAULT_SHUTDOWN_TIMEOUT);
if (interpreterGroup != null) {
- interpreterGroup.close();
+ for (List<Interpreter> session : interpreterGroup.values()) {
+ for (Interpreter interpreter : session) {
+ interpreter.close();
+ }
+ }
}
server.stop();
@@ -159,7 +164,7 @@ public class RemoteInterpreterServer
}
@Override
- public void createInterpreter(String interpreterGroupId, String sessionKey, String
+ public void createInterpreter(String interpreterGroupId, String sessionId, String
className, Map<String, String> properties, String userName) throws TException {
if (interpreterGroup == null) {
interpreterGroup = new InterpreterGroup(interpreterGroupId);
@@ -190,20 +195,11 @@ public class RemoteInterpreterServer
replClass.getConstructor(new Class[] {Properties.class});
Interpreter repl = constructor.newInstance(p);
repl.setClassloaderUrls(new URL[]{});
-
- synchronized (interpreterGroup) {
- List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
- if (interpreters == null) {
- interpreters = new LinkedList<>();
- interpreterGroup.put(sessionKey, interpreters);
- }
-
- interpreters.add(new LazyOpenInterpreter(repl));
- }
-
logger.info("Instantiate interpreter {}", className);
repl.setInterpreterGroup(interpreterGroup);
repl.setUserName(userName);
+
+ interpreterGroup.addInterpreterToSession(new LazyOpenInterpreter(repl), sessionId);
} catch (ClassNotFoundException | NoSuchMethodException | SecurityException
| InstantiationException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException e) {
@@ -237,13 +233,13 @@ public class RemoteInterpreterServer
}
}
- protected Interpreter getInterpreter(String sessionKey, String className) throws TException {
+ protected Interpreter getInterpreter(String sessionId, String className) throws TException {
if (interpreterGroup == null) {
throw new TException(
new InterpreterException("Interpreter instance " + className + " not created"));
}
synchronized (interpreterGroup) {
- List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
+ List<Interpreter> interpreters = interpreterGroup.get(sessionId);
if (interpreters == null) {
throw new TException(
new InterpreterException("Interpreter " + className + " not initialized"));
@@ -259,19 +255,20 @@ public class RemoteInterpreterServer
}
@Override
- public void open(String noteId, String className) throws TException {
- Interpreter intp = getInterpreter(noteId, className);
+ public void open(String sessionId, String className) throws TException {
+ logger.info(String.format("Open Interpreter %s for session %s ", className, sessionId));
+ Interpreter intp = getInterpreter(sessionId, className);
intp.open();
}
@Override
- public void close(String sessionKey, String className) throws TException {
+ public void close(String sessionId, String className) throws TException {
// unload all applications
for (String appId : runningApplications.keySet()) {
RunningApplication appInfo = runningApplications.get(appId);
// see NoteInterpreterLoader.SHARED_SESSION
- if (appInfo.noteId.equals(sessionKey) || sessionKey.equals("shared_session")) {
+ if (appInfo.noteId.equals(sessionId) || sessionId.equals("shared_session")) {
try {
logger.info("Unload App {} ", appInfo.pkg.getName());
appInfo.app.unload();
@@ -286,7 +283,7 @@ public class RemoteInterpreterServer
// close interpreters
List<Interpreter> interpreters;
synchronized (interpreterGroup) {
- interpreters = interpreterGroup.get(sessionKey);
+ interpreters = interpreterGroup.get(sessionId);
}
if (interpreters != null) {
Iterator<Interpreter> it = interpreters.iterator();
@@ -322,7 +319,6 @@ public class RemoteInterpreterServer
intp,
st,
context);
-
scheduler.submit(job);
while (!job.isTerminated()) {
@@ -566,30 +562,34 @@ public class RemoteInterpreterServer
}
@Override
- public int getProgress(String noteId, String className,
+ public int getProgress(String sessionId, String className,
RemoteInterpreterContext interpreterContext)
throws TException {
Integer manuallyProvidedProgress = progressMap.get(interpreterContext.getParagraphId());
if (manuallyProvidedProgress != null) {
return manuallyProvidedProgress;
} else {
- Interpreter intp = getInterpreter(noteId, className);
+ Interpreter intp = getInterpreter(sessionId, className);
+ if (intp == null) {
+ throw new TException("No interpreter {} existed for session {}".format(
+ className, sessionId));
+ }
return intp.getProgress(convert(interpreterContext, null));
}
}
@Override
- public String getFormType(String noteId, String className) throws TException {
- Interpreter intp = getInterpreter(noteId, className);
+ public String getFormType(String sessionId, String className) throws TException {
+ Interpreter intp = getInterpreter(sessionId, className);
return intp.getFormType().toString();
}
@Override
- public List<InterpreterCompletion> completion(String noteId,
+ public List<InterpreterCompletion> completion(String sessionId,
String className, String buf, int cursor, RemoteInterpreterContext remoteInterpreterContext)
throws TException {
- Interpreter intp = getInterpreter(noteId, className);
+ Interpreter intp = getInterpreter(sessionId, className);
List completion = intp.completion(buf, cursor, convert(remoteInterpreterContext, null));
return completion;
}
@@ -766,16 +766,16 @@ public class RemoteInterpreterServer
}
@Override
- public String getStatus(String sessionKey, String jobId)
+ public String getStatus(String sessionId, String jobId)
throws TException {
if (interpreterGroup == null) {
- return "Unknown";
+ return Status.UNKNOWN.name();
}
synchronized (interpreterGroup) {
- List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
+ List<Interpreter> interpreters = interpreterGroup.get(sessionId);
if (interpreters == null) {
- return "Unknown";
+ return Status.UNKNOWN.name();
}
for (Interpreter intp : interpreters) {
@@ -792,7 +792,7 @@ public class RemoteInterpreterServer
}
}
}
- return "Unknown";
+ return Status.UNKNOWN.name();
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java
deleted file mode 100644
index b26995a..0000000
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.resource;
-
-import org.apache.zeppelin.interpreter.InterpreterGroup;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
-import org.slf4j.Logger;
-
-import java.util.List;
-
-/**
- * Utilities for ResourcePool
- */
-public class ResourcePoolUtils {
- static Logger logger = org.slf4j.LoggerFactory.getLogger(ResourcePoolUtils.class);
-
- public static ResourceSet getAllResources() {
- return getAllResourcesExcept(null);
- }
-
- public static ResourceSet getAllResourcesExcept(String interpreterGroupExcludsion) {
- ResourceSet resourceSet = new ResourceSet();
- for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
- if (interpreterGroupExcludsion != null &&
- intpGroup.getId().equals(interpreterGroupExcludsion)) {
- continue;
- }
-
- RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
- if (remoteInterpreterProcess == null) {
- ResourcePool localPool = intpGroup.getResourcePool();
- if (localPool != null) {
- resourceSet.addAll(localPool.getAll());
- }
- } else if (remoteInterpreterProcess.isRunning()) {
- RemoteInterpreterService.Client client = null;
- boolean broken = false;
- try {
- client = remoteInterpreterProcess.getClient();
- if (client == null) {
- // remote interpreter may not started yet or terminated.
- continue;
- }
- List<String> resourceList = client.resourcePoolGetAll();
- for (String res : resourceList) {
- resourceSet.add(Resource.fromJson(res));
- }
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- broken = true;
- } finally {
- if (client != null) {
- intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
- }
- }
- }
- }
- return resourceSet;
- }
-
- public static void removeResourcesBelongsToNote(String noteId) {
- removeResourcesBelongsToParagraph(noteId, null);
- }
-
- public static void removeResourcesBelongsToParagraph(String noteId, String paragraphId) {
- for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
- ResourceSet resourceSet = new ResourceSet();
- RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
- if (remoteInterpreterProcess == null) {
- ResourcePool localPool = intpGroup.getResourcePool();
- if (localPool != null) {
- resourceSet.addAll(localPool.getAll());
- }
- if (noteId != null) {
- resourceSet = resourceSet.filterByNoteId(noteId);
- }
- if (paragraphId != null) {
- resourceSet = resourceSet.filterByParagraphId(paragraphId);
- }
-
- for (Resource r : resourceSet) {
- localPool.remove(
- r.getResourceId().getNoteId(),
- r.getResourceId().getParagraphId(),
- r.getResourceId().getName());
- }
- } else if (remoteInterpreterProcess.isRunning()) {
- RemoteInterpreterService.Client client = null;
- boolean broken = false;
- try {
- client = remoteInterpreterProcess.getClient();
- List<String> resourceList = client.resourcePoolGetAll();
- for (String res : resourceList) {
- resourceSet.add(Resource.fromJson(res));
- }
-
- if (noteId != null) {
- resourceSet = resourceSet.filterByNoteId(noteId);
- }
- if (paragraphId != null) {
- resourceSet = resourceSet.filterByParagraphId(paragraphId);
- }
-
- for (Resource r : resourceSet) {
- client.resourceRemove(
- r.getResourceId().getNoteId(),
- r.getResourceId().getParagraphId(),
- r.getResourceId().getName());
- }
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- broken = true;
- } finally {
- if (client != null) {
- intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
- }
- }
- }
- }
- }
-}
-
[2/9] zeppelin git commit: [ZEPPELIN-2627] Interpreter Component
Refactoring
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
index 95235e5..0a5e484 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
@@ -17,839 +17,352 @@
package org.apache.zeppelin.interpreter.remote;
-import static org.junit.Assert.*;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
import org.apache.thrift.transport.TTransportException;
-import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterEnv;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
-import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
-import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterA;
-import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterB;
-import org.apache.zeppelin.resource.LocalResourcePool;
-import org.apache.zeppelin.scheduler.Job;
-import org.apache.zeppelin.scheduler.Job.Status;
-import org.apache.zeppelin.scheduler.Scheduler;
+import org.apache.zeppelin.interpreter.remote.mock.GetAngularObjectSizeInterpreter;
+import org.apache.zeppelin.interpreter.remote.mock.GetEnvPropertyInterpreter;
+import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import org.mockito.Mockito;
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Properties;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
public class RemoteInterpreterTest {
private static final String INTERPRETER_SCRIPT =
- System.getProperty("os.name").startsWith("Windows") ?
- "../bin/interpreter.cmd" :
- "../bin/interpreter.sh";
+ System.getProperty("os.name").startsWith("Windows") ?
+ "../bin/interpreter.cmd" :
+ "../bin/interpreter.sh";
- private InterpreterGroup intpGroup;
- private HashMap<String, String> env;
+ private InterpreterSetting interpreterSetting;
@Before
public void setUp() throws Exception {
- intpGroup = new InterpreterGroup();
- env = new HashMap<>();
- env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
+ InterpreterOption interpreterOption = new InterpreterOption();
+
+ interpreterOption.setRemote(true);
+ InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo3 = new InterpreterInfo(SleepInterpreter.class.getName(), "sleep", false, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo4 = new InterpreterInfo(GetEnvPropertyInterpreter.class.getName(), "get", false, new HashMap<String, Object>());
+ InterpreterInfo interpreterInfo5 = new InterpreterInfo(GetAngularObjectSizeInterpreter.class.getName(), "angular_obj",false, new HashMap<String, Object>());
+ List<InterpreterInfo> interpreterInfos = new ArrayList<>();
+ interpreterInfos.add(interpreterInfo1);
+ interpreterInfos.add(interpreterInfo2);
+ interpreterInfos.add(interpreterInfo3);
+ interpreterInfos.add(interpreterInfo4);
+ interpreterInfos.add(interpreterInfo5);
+ InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT);
+ interpreterSetting = new InterpreterSetting.Builder()
+ .setId("test")
+ .setName("test")
+ .setGroup("test")
+ .setInterpreterInfos(interpreterInfos)
+ .setOption(interpreterOption)
+ .setRunner(runner)
+ .setInterpreterDir("../interpeters/test")
+ .create();
}
@After
public void tearDown() throws Exception {
- intpGroup.close();
+ interpreterSetting.close();
}
- private RemoteInterpreter createMockInterpreterA(Properties p) {
- return createMockInterpreterA(p, "note");
- }
+ @Test
+ public void testSharedMode() {
+ interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
+
+ Interpreter interpreter1 = interpreterSetting.getDefaultInterpreter("user1", "note1");
+ Interpreter interpreter2 = interpreterSetting.getDefaultInterpreter("user2", "note1");
+ assertTrue(interpreter1 instanceof RemoteInterpreter);
+ RemoteInterpreter remoteInterpreter1 = (RemoteInterpreter) interpreter1;
+ assertTrue(interpreter2 instanceof RemoteInterpreter);
+ RemoteInterpreter remoteInterpreter2 = (RemoteInterpreter) interpreter2;
+
+ InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
+ "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+ null, null, new ArrayList<InterpreterContextRunner>(), null);
+ assertEquals("hello", remoteInterpreter1.interpret("hello", context1).message().get(0).getData());
+ assertEquals(Interpreter.FormType.NATIVE, interpreter1.getFormType());
+ assertEquals(0, remoteInterpreter1.getProgress(context1));
+ assertNotNull(remoteInterpreter1.getOrCreateInterpreterProcess());
+ assertTrue(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess().isRunning());
+
+ assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
+ assertEquals(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess(),
+ remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess());
+
+ // Call InterpreterGroup.close instead of Interpreter.close, otherwise we will have the
+ // RemoteInterpreterProcess leakage.
+ remoteInterpreter1.getInterpreterGroup().close(remoteInterpreter1.getSessionId());
+ assertNull(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess());
+ try {
+ assertEquals("hello", remoteInterpreter1.interpret("hello", context1).message().get(0).getData());
+ fail("Should not be able to call interpret after interpreter is closed");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
- private RemoteInterpreter createMockInterpreterA(Properties p, String noteId) {
- return new RemoteInterpreter(
- p,
- noteId,
- MockInterpreterA.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- null,
- null,
- "anonymous",
- false);
+ try {
+ assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
+ fail("Should not be able to call getProgress after RemoterInterpreterProcess is stoped");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
}
- private RemoteInterpreter createMockInterpreterB(Properties p) {
- return createMockInterpreterB(p, "note");
- }
+ @Test
+ public void testScopedMode() {
+ interpreterSetting.getOption().setPerUser(InterpreterOption.SCOPED);
+
+ Interpreter interpreter1 = interpreterSetting.getDefaultInterpreter("user1", "note1");
+ Interpreter interpreter2 = interpreterSetting.getDefaultInterpreter("user2", "note1");
+ assertTrue(interpreter1 instanceof RemoteInterpreter);
+ RemoteInterpreter remoteInterpreter1 = (RemoteInterpreter) interpreter1;
+ assertTrue(interpreter2 instanceof RemoteInterpreter);
+ RemoteInterpreter remoteInterpreter2 = (RemoteInterpreter) interpreter2;
+
+ InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
+ "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+ null, null, new ArrayList<InterpreterContextRunner>(), null);
+ assertEquals("hello", remoteInterpreter1.interpret("hello", context1).message().get(0).getData());
+ assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
+ assertEquals(Interpreter.FormType.NATIVE, interpreter1.getFormType());
+ assertEquals(0, remoteInterpreter1.getProgress(context1));
+
+ assertNotNull(remoteInterpreter1.getOrCreateInterpreterProcess());
+ assertTrue(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess().isRunning());
+
+ assertEquals(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess(),
+ remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess());
+ // Call InterpreterGroup.close instead of Interpreter.close, otherwise we will have the
+ // RemoteInterpreterProcess leakage.
+ remoteInterpreter1.getInterpreterGroup().close(remoteInterpreter1.getSessionId());
+ try {
+ assertEquals("hello", remoteInterpreter1.interpret("hello", context1).message().get(0).getData());
+ fail("Should not be able to call interpret after interpreter is closed");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
- private RemoteInterpreter createMockInterpreterB(Properties p, String noteId) {
- return new RemoteInterpreter(
- p,
- noteId,
- MockInterpreterB.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- null,
- null,
- "anonymous",
- false);
+ assertTrue(remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess().isRunning());
+ assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
+ remoteInterpreter2.getInterpreterGroup().close(remoteInterpreter2.getSessionId());
+ try {
+ assertEquals("hello", remoteInterpreter2.interpret("hello", context1));
+ fail("Should not be able to call interpret after interpreter is closed");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ assertNull(remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess());
}
@Test
- public void testRemoteInterperterCall() throws TTransportException, IOException {
- Properties p = new Properties();
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.get("note").add(intpA);
-
- intpA.setInterpreterGroup(intpGroup);
-
- RemoteInterpreter intpB = createMockInterpreterB(p);
-
- intpGroup.get("note").add(intpB);
- intpB.setInterpreterGroup(intpGroup);
-
-
- RemoteInterpreterProcess process = intpA.getInterpreterProcess();
- process.equals(intpB.getInterpreterProcess());
-
- assertFalse(process.isRunning());
- assertEquals(0, process.getNumIdleClient());
- assertEquals(0, process.referenceCount());
-
- intpA.open(); // initializa all interpreters in the same group
- assertTrue(process.isRunning());
- assertEquals(1, process.getNumIdleClient());
- assertEquals(1, process.referenceCount());
-
- intpA.interpret("1",
- new InterpreterContext(
- "note",
- "id",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
-
- intpB.open();
- assertEquals(1, process.referenceCount());
-
- intpA.close();
- assertEquals(0, process.referenceCount());
- intpB.close();
- assertEquals(0, process.referenceCount());
+ public void testIsolatedMode() {
+ interpreterSetting.getOption().setPerUser(InterpreterOption.ISOLATED);
+
+ Interpreter interpreter1 = interpreterSetting.getDefaultInterpreter("user1", "note1");
+ Interpreter interpreter2 = interpreterSetting.getDefaultInterpreter("user2", "note1");
+ assertTrue(interpreter1 instanceof RemoteInterpreter);
+ RemoteInterpreter remoteInterpreter1 = (RemoteInterpreter) interpreter1;
+ assertTrue(interpreter2 instanceof RemoteInterpreter);
+ RemoteInterpreter remoteInterpreter2 = (RemoteInterpreter) interpreter2;
+
+ InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
+ "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+ null, null, new ArrayList<InterpreterContextRunner>(), null);
+ assertEquals("hello", remoteInterpreter1.interpret("hello", context1).message().get(0).getData());
+ assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
+ assertEquals(Interpreter.FormType.NATIVE, interpreter1.getFormType());
+ assertEquals(0, remoteInterpreter1.getProgress(context1));
+ assertNotNull(remoteInterpreter1.getOrCreateInterpreterProcess());
+ assertTrue(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess().isRunning());
+
+ assertNotEquals(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess(),
+ remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess());
+ // Call InterpreterGroup.close instead of Interpreter.close, otherwise we will have the
+ // RemoteInterpreterProcess leakage.
+ remoteInterpreter1.getInterpreterGroup().close(remoteInterpreter1.getSessionId());
+ assertNull(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess());
+ assertTrue(remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess().isRunning());
+ try {
+ remoteInterpreter1.interpret("hello", context1);
+ fail("Should not be able to call getProgress after interpreter is closed");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
- assertFalse(process.isRunning());
+ assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
+ remoteInterpreter2.getInterpreterGroup().close(remoteInterpreter2.getSessionId());
+ try {
+ assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
+ fail("Should not be able to call interpret after interpreter is closed");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ assertNull(remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess());
}
@Test
public void testExecuteIncorrectPrecode() throws TTransportException, IOException {
- Properties p = new Properties();
- p.put("zeppelin.MockInterpreterA.precode", "fail test");
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.get("note").add(intpA);
-
- intpA.setInterpreterGroup(intpGroup);
-
- RemoteInterpreterProcess process = intpA.getInterpreterProcess();
-
- intpA.open();
-
- InterpreterResult result = intpA.interpret("1",
- new InterpreterContext(
- "note",
- "id",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
-
-
-
- intpA.close();
- assertEquals(Code.ERROR, result.code());
+ interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
+ interpreterSetting.setProperty("zeppelin.SleepInterpreter.precode", "fail test");
+ Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "sleep");
+ InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
+ "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+ null, null, new ArrayList<InterpreterContextRunner>(), null);
+ assertEquals(Code.ERROR, interpreter1.interpret("10", context1).code());
}
@Test
public void testExecuteCorrectPrecode() throws TTransportException, IOException {
- Properties p = new Properties();
- p.put("zeppelin.MockInterpreterA.precode", "2");
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.get("note").add(intpA);
-
- intpA.setInterpreterGroup(intpGroup);
-
- RemoteInterpreterProcess process = intpA.getInterpreterProcess();
-
- intpA.open();
-
- InterpreterResult result = intpA.interpret("1",
- new InterpreterContext(
- "note",
- "id",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
-
-
-
- intpA.close();
- assertEquals(Code.SUCCESS, result.code());
- assertEquals("1", result.message().get(0).getData());
+ interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
+ interpreterSetting.setProperty("zeppelin.SleepInterpreter.precode", "1");
+ Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "sleep");
+ InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
+ "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+ null, null, new ArrayList<InterpreterContextRunner>(), null);
+ assertEquals(Code.SUCCESS, interpreter1.interpret("10", context1).code());
}
@Test
public void testRemoteInterperterErrorStatus() throws TTransportException, IOException {
- Properties p = new Properties();
-
- RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.put("note", new LinkedList<Interpreter>());
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
- intpA.open();
- InterpreterResult ret = intpA.interpret("non numeric value",
- new InterpreterContext(
- "noteId",
- "id",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
-
- assertEquals(Code.ERROR, ret.code());
- }
+ interpreterSetting.setProperty("zeppelin.interpreter.echo.fail", "true");
+ interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
- @Test
- public void testRemoteSchedulerSharing() throws TTransportException, IOException {
- Properties p = new Properties();
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- RemoteInterpreter intpA = new RemoteInterpreter(
- p,
- "note",
- MockInterpreterA.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- null,
- null,
- "anonymous",
- false);
-
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
- RemoteInterpreter intpB = new RemoteInterpreter(
- p,
- "note",
- MockInterpreterB.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- null,
- null,
- "anonymous",
- false);
-
- intpGroup.get("note").add(intpB);
- intpB.setInterpreterGroup(intpGroup);
-
- intpA.open();
- intpB.open();
+ Interpreter interpreter1 = interpreterSetting.getDefaultInterpreter("user1", "note1");
+ assertTrue(interpreter1 instanceof RemoteInterpreter);
+ RemoteInterpreter remoteInterpreter1 = (RemoteInterpreter) interpreter1;
- long start = System.currentTimeMillis();
- InterpreterResult ret = intpA.interpret("500",
- new InterpreterContext(
- "note",
- "id",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
- assertEquals("500", ret.message().get(0).getData());
-
- ret = intpB.interpret("500",
- new InterpreterContext(
- "note",
- "id",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
- assertEquals("1000", ret.message().get(0).getData());
- long end = System.currentTimeMillis();
- assertTrue(end - start >= 1000);
-
-
- intpA.close();
- intpB.close();
+ InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
+ "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+ null, null, new ArrayList<InterpreterContextRunner>(), null);
+ assertEquals(Code.ERROR, remoteInterpreter1.interpret("hello", context1).code());
}
@Test
- public void testRemoteSchedulerSharingSubmit() throws TTransportException, IOException, InterruptedException {
- Properties p = new Properties();
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- final RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
- final RemoteInterpreter intpB = createMockInterpreterB(p);
-
- intpGroup.get("note").add(intpB);
- intpB.setInterpreterGroup(intpGroup);
-
- intpA.open();
- intpB.open();
-
- long start = System.currentTimeMillis();
- Job jobA = new Job("jobA", null) {
- private Object r;
-
+ public void testFIFOScheduler() throws InterruptedException {
+ interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
+ // by default SleepInterpreter would use FIFOScheduler
+
+ final Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "sleep");
+ final InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
+ "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+ null, null, new ArrayList<InterpreterContextRunner>(), null);
+ // run this dummy interpret method first to launch the RemoteInterpreterProcess to avoid the
+ // time overhead of launching the process.
+ interpreter1.interpret("1", context1);
+ Thread thread1 = new Thread() {
@Override
- public Object getReturn() {
- return r;
- }
-
- @Override
- public void setResult(Object results) {
- this.r = results;
- }
-
- @Override
- public int progress() {
- return 0;
- }
-
- @Override
- public Map<String, Object> info() {
- return null;
- }
-
- @Override
- protected Object jobRun() throws Throwable {
- return intpA.interpret("500",
- new InterpreterContext(
- "note",
- "jobA",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
- }
-
- @Override
- protected boolean jobAbort() {
- return false;
+ public void run() {
+ assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code());
}
-
};
- intpA.getScheduler().submit(jobA);
-
- Job jobB = new Job("jobB", null) {
-
- private Object r;
-
- @Override
- public Object getReturn() {
- return r;
- }
-
- @Override
- public void setResult(Object results) {
- this.r = results;
- }
-
- @Override
- public int progress() {
- return 0;
- }
-
- @Override
- public Map<String, Object> info() {
- return null;
- }
-
- @Override
- protected Object jobRun() throws Throwable {
- return intpB.interpret("500",
- new InterpreterContext(
- "note",
- "jobB",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
- }
-
+ Thread thread2 = new Thread() {
@Override
- protected boolean jobAbort() {
- return false;
+ public void run() {
+ assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code());
}
-
};
- intpB.getScheduler().submit(jobB);
- // wait until both job finished
- while (jobA.getStatus() != Status.FINISHED ||
- jobB.getStatus() != Status.FINISHED) {
- Thread.sleep(100);
- }
- long end = System.currentTimeMillis();
- assertTrue(end - start >= 1000);
-
- assertEquals("1000", ((InterpreterResult) jobB.getReturn()).message().get(0).getData());
-
- intpA.close();
- intpB.close();
- }
-
- @Test
- public void testRunOrderPreserved() throws InterruptedException {
- Properties p = new Properties();
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- final RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
- intpA.open();
-
- int concurrency = 3;
- final List<InterpreterResultMessage> results = new LinkedList<>();
-
- Scheduler scheduler = intpA.getScheduler();
- for (int i = 0; i < concurrency; i++) {
- final String jobId = Integer.toString(i);
- scheduler.submit(new Job(jobId, Integer.toString(i), null, 200) {
- private Object r;
-
- @Override
- public Object getReturn() {
- return r;
- }
-
- @Override
- public void setResult(Object results) {
- this.r = results;
- }
-
- @Override
- public int progress() {
- return 0;
- }
-
- @Override
- public Map<String, Object> info() {
- return null;
- }
-
- @Override
- protected Object jobRun() throws Throwable {
- InterpreterResult ret = intpA.interpret(getJobName(), new InterpreterContext(
- "note",
- jobId,
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
-
- synchronized (results) {
- results.addAll(ret.message());
- results.notify();
- }
- return null;
- }
-
- @Override
- protected boolean jobAbort() {
- return false;
- }
-
- });
- }
-
- // wait for job finished
- synchronized (results) {
- while (results.size() != concurrency) {
- results.wait(300);
- }
- }
-
- int i = 0;
- for (InterpreterResultMessage result : results) {
- assertEquals(Integer.toString(i++), result.getData());
- }
- assertEquals(concurrency, i);
-
- intpA.close();
- }
-
-
- @Test
- public void testRunParallel() throws InterruptedException {
- Properties p = new Properties();
- p.put("parallel", "true");
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- final RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
- intpA.open();
-
- int concurrency = 4;
- final int timeToSleep = 1000;
- final List<InterpreterResultMessage> results = new LinkedList<>();
long start = System.currentTimeMillis();
-
- Scheduler scheduler = intpA.getScheduler();
- for (int i = 0; i < concurrency; i++) {
- final String jobId = Integer.toString(i);
- scheduler.submit(new Job(jobId, Integer.toString(i), null, 300) {
- private Object r;
-
- @Override
- public Object getReturn() {
- return r;
- }
-
- @Override
- public void setResult(Object results) {
- this.r = results;
- }
-
- @Override
- public int progress() {
- return 0;
- }
-
- @Override
- public Map<String, Object> info() {
- return null;
- }
-
- @Override
- protected Object jobRun() throws Throwable {
- String stmt = Integer.toString(timeToSleep);
- InterpreterResult ret = intpA.interpret(stmt, new InterpreterContext(
- "note",
- jobId,
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
-
- synchronized (results) {
- results.addAll(ret.message());
- results.notify();
- }
- return stmt;
- }
-
- @Override
- protected boolean jobAbort() {
- return false;
- }
-
- });
- }
-
- // wait for job finished
- synchronized (results) {
- while (results.size() != concurrency) {
- results.wait(300);
- }
- }
-
+ thread1.start();
+ thread2.start();
+ thread1.join();
+ thread2.join();
long end = System.currentTimeMillis();
-
- assertTrue(end - start < timeToSleep * concurrency);
-
- intpA.close();
+ assertTrue((end - start) >= 200);
}
@Test
- public void testInterpreterGroupResetBeforeProcessStarts() {
- Properties p = new Properties();
-
- RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpA.setInterpreterGroup(intpGroup);
- RemoteInterpreterProcess processA = intpA.getInterpreterProcess();
-
- intpA.setInterpreterGroup(new InterpreterGroup(intpA.getInterpreterGroup().getId()));
- RemoteInterpreterProcess processB = intpA.getInterpreterProcess();
-
- assertNotSame(processA.hashCode(), processB.hashCode());
- }
-
- @Test
- public void testInterpreterGroupResetAfterProcessFinished() {
- Properties p = new Properties();
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpA.setInterpreterGroup(intpGroup);
- RemoteInterpreterProcess processA = intpA.getInterpreterProcess();
- intpA.open();
-
- processA.dereference(); // intpA.close();
-
- intpA.setInterpreterGroup(new InterpreterGroup(intpA.getInterpreterGroup().getId()));
- RemoteInterpreterProcess processB = intpA.getInterpreterProcess();
-
- assertNotSame(processA.hashCode(), processB.hashCode());
- }
-
- @Test
- public void testInterpreterGroupResetDuringProcessRunning() throws InterruptedException {
- Properties p = new Properties();
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- final RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
- intpA.open();
-
- Job jobA = new Job("jobA", null) {
- private Object r;
-
- @Override
- public Object getReturn() {
- return r;
- }
-
- @Override
- public void setResult(Object results) {
- this.r = results;
- }
-
- @Override
- public int progress() {
- return 0;
- }
-
- @Override
- public Map<String, Object> info() {
- return null;
- }
-
+ public void testParallelScheduler() throws InterruptedException {
+ interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
+ interpreterSetting.setProperty("zeppelin.SleepInterpreter.parallel", "true");
+
+ final Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "sleep");
+ final InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
+ "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+ null, null, new ArrayList<InterpreterContextRunner>(), null);
+
+ // run this dummy interpret method first to launch the RemoteInterpreterProcess to avoid the
+ // time overhead of launching the process.
+ interpreter1.interpret("1", context1);
+ Thread thread1 = new Thread() {
@Override
- protected Object jobRun() throws Throwable {
- return intpA.interpret("2000",
- new InterpreterContext(
- "note",
- "jobA",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null));
+ public void run() {
+ assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code());
}
-
+ };
+ Thread thread2 = new Thread() {
@Override
- protected boolean jobAbort() {
- return false;
+ public void run() {
+ assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code());
}
-
};
- intpA.getScheduler().submit(jobA);
-
- // wait for job started
- while (intpA.getScheduler().getJobsRunning().size() == 0) {
- Thread.sleep(100);
- }
-
- // restart interpreter
- RemoteInterpreterProcess processA = intpA.getInterpreterProcess();
- intpA.close();
-
- InterpreterGroup newInterpreterGroup =
- new InterpreterGroup(intpA.getInterpreterGroup().getId());
- newInterpreterGroup.put("note", new LinkedList<Interpreter>());
-
- intpA.setInterpreterGroup(newInterpreterGroup);
- intpA.open();
- RemoteInterpreterProcess processB = intpA.getInterpreterProcess();
-
- assertNotSame(processA.hashCode(), processB.hashCode());
-
+ long start = System.currentTimeMillis();
+ thread1.start();
+ thread2.start();
+ thread1.join();
+ thread2.join();
+ long end = System.currentTimeMillis();
+ assertTrue((end - start) <= 200);
}
@Test
public void testRemoteInterpreterSharesTheSameSchedulerInstanceInTheSameGroup() {
- Properties p = new Properties();
- intpGroup.put("note", new LinkedList<Interpreter>());
-
- RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
- RemoteInterpreter intpB = createMockInterpreterB(p);
-
- intpGroup.get("note").add(intpB);
- intpB.setInterpreterGroup(intpGroup);
-
- intpA.open();
- intpB.open();
-
- assertEquals(intpA.getScheduler(), intpB.getScheduler());
+ interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
+ Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "sleep");
+ Interpreter interpreter2 = interpreterSetting.getInterpreter("user1", "note1", "echo");
+ assertEquals(interpreter1.getInterpreterGroup(), interpreter2.getInterpreterGroup());
+ assertEquals(interpreter1.getScheduler(), interpreter2.getScheduler());
}
@Test
public void testMultiInterpreterSession() {
- Properties p = new Properties();
- intpGroup.put("sessionA", new LinkedList<Interpreter>());
- intpGroup.put("sessionB", new LinkedList<Interpreter>());
-
- RemoteInterpreter intpAsessionA = createMockInterpreterA(p, "sessionA");
- intpGroup.get("sessionA").add(intpAsessionA);
- intpAsessionA.setInterpreterGroup(intpGroup);
-
- RemoteInterpreter intpBsessionA = createMockInterpreterB(p, "sessionA");
- intpGroup.get("sessionA").add(intpBsessionA);
- intpBsessionA.setInterpreterGroup(intpGroup);
-
- intpAsessionA.open();
- intpBsessionA.open();
-
- assertEquals(intpAsessionA.getScheduler(), intpBsessionA.getScheduler());
-
- RemoteInterpreter intpAsessionB = createMockInterpreterA(p, "sessionB");
- intpGroup.get("sessionB").add(intpAsessionB);
- intpAsessionB.setInterpreterGroup(intpGroup);
-
- RemoteInterpreter intpBsessionB = createMockInterpreterB(p, "sessionB");
- intpGroup.get("sessionB").add(intpBsessionB);
- intpBsessionB.setInterpreterGroup(intpGroup);
-
- intpAsessionB.open();
- intpBsessionB.open();
-
- assertEquals(intpAsessionB.getScheduler(), intpBsessionB.getScheduler());
- assertNotEquals(intpAsessionA.getScheduler(), intpAsessionB.getScheduler());
+ interpreterSetting.getOption().setPerUser(InterpreterOption.SCOPED);
+ Interpreter interpreter1_user1 = interpreterSetting.getInterpreter("user1", "note1", "sleep");
+ Interpreter interpreter2_user1 = interpreterSetting.getInterpreter("user1", "note1", "echo");
+ assertEquals(interpreter1_user1.getInterpreterGroup(), interpreter2_user1.getInterpreterGroup());
+ assertEquals(interpreter1_user1.getScheduler(), interpreter2_user1.getScheduler());
+
+ Interpreter interpreter1_user2 = interpreterSetting.getInterpreter("user2", "note1", "sleep");
+ Interpreter interpreter2_user2 = interpreterSetting.getInterpreter("user2", "note1", "echo");
+ assertEquals(interpreter1_user2.getInterpreterGroup(), interpreter2_user2.getInterpreterGroup());
+ assertEquals(interpreter1_user2.getScheduler(), interpreter2_user2.getScheduler());
+
+ // scheduler is shared in session but not across session
+ assertNotEquals(interpreter1_user1.getScheduler(), interpreter1_user2.getScheduler());
}
@Test
public void should_push_local_angular_repo_to_remote() throws Exception {
- //Given
- final Client client = Mockito.mock(Client.class);
- final RemoteInterpreter intr = new RemoteInterpreter(new Properties(), "noteId",
- MockInterpreterA.class.getName(), "runner", "path", "localRepo", env, 10 * 1000, null,
- null, "anonymous", false);
- final AngularObjectRegistry registry = new AngularObjectRegistry("spark", null);
- registry.add("name", "DuyHai DOAN", "nodeId", "paragraphId");
- final InterpreterGroup interpreterGroup = new InterpreterGroup("groupId");
- interpreterGroup.setAngularObjectRegistry(registry);
- intr.setInterpreterGroup(interpreterGroup);
- final java.lang.reflect.Type registryType = new TypeToken<Map<String,
- Map<String, AngularObject>>>() {}.getType();
- final Gson gson = new Gson();
- final String expected = gson.toJson(registry.getRegistry(), registryType);
+ final AngularObjectRegistry registry = new AngularObjectRegistry("spark", null);
+ registry.add("name_1", "value_1", "note_1", "paragraphId_1");
+ registry.add("name_2", "value_2", "node_2", "paragraphId_2");
+ Interpreter interpreter = interpreterSetting.getInterpreter("user1", "note1", "angular_obj");
+ interpreter.getInterpreterGroup().setAngularObjectRegistry(registry);
- //When
- intr.pushAngularObjectRegistryToRemote(client);
+ final InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "repl",
+ "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+ null, null, new ArrayList<InterpreterContextRunner>(), null);
- //Then
- Mockito.verify(client).angularRegistryPush(expected);
+ InterpreterResult result = interpreter.interpret("dummy", context);
+ assertEquals(Code.SUCCESS, result.code());
+ assertEquals("2", result.message().get(0).getData());
}
@Test
@@ -864,112 +377,21 @@ public class RemoteInterpreterTest {
}
@Test
- public void testEnvronmentAndPropertySet() {
- Properties p = new Properties();
- p.setProperty("MY_ENV1", "env value 1");
- p.setProperty("my.property.1", "property value 1");
-
- RemoteInterpreter intp = new RemoteInterpreter(
- p,
- "note",
- MockInterpreterEnv.class.getName(),
- new File(INTERPRETER_SCRIPT).getAbsolutePath(),
- "fake",
- "fakeRepo",
- env,
- 10 * 1000,
- null,
- null,
- "anonymous",
- false);
-
- intpGroup.put("note", new LinkedList<Interpreter>());
- intpGroup.get("note").add(intp);
- intp.setInterpreterGroup(intpGroup);
-
- intp.open();
-
- InterpreterContext context = new InterpreterContext(
- "noteId",
- "id",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null);
-
-
- assertEquals("env value 1", intp.interpret("getEnv MY_ENV1", context).message().get(0).getData());
- assertEquals(Code.ERROR, intp.interpret("getProperty MY_ENV1", context).code());
- assertEquals(Code.ERROR, intp.interpret("getEnv my.property.1", context).code());
- assertEquals("property value 1", intp.interpret("getProperty my.property.1", context).message().get(0).getData());
-
- intp.close();
- }
+ public void testEnvironmentAndProperty() {
+ interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
+ interpreterSetting.setProperty("ENV_1", "VALUE_1");
+ interpreterSetting.setProperty("property_1", "value_1");
- @Test
- public void testSetProgress() throws InterruptedException {
- // given MockInterpreterA set progress through InterpreterContext
- Properties p = new Properties();
- p.setProperty("progress", "50");
- final RemoteInterpreter intpA = createMockInterpreterA(p);
-
- intpGroup.put("note", new LinkedList<Interpreter>());
- intpGroup.get("note").add(intpA);
- intpA.setInterpreterGroup(intpGroup);
-
- intpA.open();
-
- final InterpreterContext context1 = new InterpreterContext(
- "noteId",
- "id1",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null);
-
- InterpreterContext context2 = new InterpreterContext(
- "noteId",
- "id2",
- null,
- "title",
- "text",
- new AuthenticationInfo(),
- new HashMap<String, Object>(),
- new GUI(),
- new AngularObjectRegistry(intpGroup.getId(), null),
- new LocalResourcePool("pool1"),
- new LinkedList<InterpreterContextRunner>(), null);
-
-
- assertEquals(0, intpA.getProgress(context1));
- assertEquals(0, intpA.getProgress(context2));
-
- // when interpreter update progress through InterpreterContext
- Thread t = new Thread() {
- public void run() {
- InterpreterResult ret = intpA.interpret("1000", context1);
- }
- };
- t.start();
+ final Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "get");
+ final InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
+ "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
+ null, null, new ArrayList<InterpreterContextRunner>(), null);
- // then progress need to be updated in given context
- while(intpA.getProgress(context1) == 0) Thread.yield();
- assertEquals(50, intpA.getProgress(context1));
- assertEquals(0, intpA.getProgress(context2));
+ assertEquals("VALUE_1", interpreter1.interpret("getEnv ENV_1", context1).message().get(0).getData());
+ assertEquals("null", interpreter1.interpret("getEnv ENV_2", context1).message().get(0).getData());
- t.join();
- assertEquals(0, intpA.getProgress(context1));
- assertEquals(0, intpA.getProgress(context2));
+ assertEquals("value_1", interpreter1.interpret("getProperty property_1", context1).message().get(0).getData());
+ assertEquals("null", interpreter1.interpret("getProperty property_2", context1).message().get(0).getData());
}
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java
deleted file mode 100644
index 975d6ea..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils;
-import org.junit.Test;
-
-public class RemoteInterpreterUtilsTest {
-
- @Test
- public void testFindRandomAvailablePortOnAllLocalInterfaces() throws IOException {
- assertTrue(RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces() > 0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/GetAngularObjectSizeInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/GetAngularObjectSizeInterpreter.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/GetAngularObjectSizeInterpreter.java
new file mode 100644
index 0000000..6d6495f
--- /dev/null
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/GetAngularObjectSizeInterpreter.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter.remote.mock;
+
+
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+
+import java.util.Properties;
+
+public class GetAngularObjectSizeInterpreter extends Interpreter {
+
+ public GetAngularObjectSizeInterpreter(Properties property) {
+ super(property);
+ }
+
+ @Override
+ public void open() {
+
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ return new InterpreterResult(InterpreterResult.Code.SUCCESS,
+ "" + context.getAngularObjectRegistry().getRegistry().size());
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+
+ }
+
+ @Override
+ public FormType getFormType() {
+ return FormType.NATIVE;
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/GetEnvPropertyInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/GetEnvPropertyInterpreter.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/GetEnvPropertyInterpreter.java
new file mode 100644
index 0000000..a039a59
--- /dev/null
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/GetEnvPropertyInterpreter.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.interpreter.remote.mock;
+
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.apache.zeppelin.scheduler.Scheduler;
+import org.apache.zeppelin.scheduler.SchedulerFactory;
+
+import java.util.List;
+import java.util.Properties;
+
+
+public class GetEnvPropertyInterpreter extends Interpreter {
+
+ public GetEnvPropertyInterpreter(Properties property) {
+ super(property);
+ }
+
+ @Override
+ public void open() {
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ String[] cmd = st.split(" ");
+ if (cmd[0].equals("getEnv")) {
+ return new InterpreterResult(InterpreterResult.Code.SUCCESS, System.getenv(cmd[1]) == null ? "null" : System.getenv(cmd[1]));
+ } else if (cmd[0].equals("getProperty")){
+ return new InterpreterResult(InterpreterResult.Code.SUCCESS, System.getProperty(cmd[1]) == null ? "null" : System.getProperty(cmd[1]));
+ } else {
+ return new InterpreterResult(InterpreterResult.Code.ERROR, cmd[0]);
+ }
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+
+ }
+
+ @Override
+ public FormType getFormType() {
+ return FormType.NATIVE;
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+
+ @Override
+ public List<InterpreterCompletion> completion(String buf, int cursor,
+ InterpreterContext interpreterContext) {
+ return null;
+ }
+
+ @Override
+ public Scheduler getScheduler() {
+ return SchedulerFactory.singleton().createOrGetFIFOScheduler("interpreter_" + this.hashCode());
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java
index 50d9888..5a3e57c 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java
@@ -17,19 +17,18 @@
package org.apache.zeppelin.interpreter.remote.mock;
-import java.util.List;
-import java.util.Properties;
-
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
-import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
+import java.util.List;
+import java.util.Properties;
+
public class MockInterpreterA extends Interpreter {
private String lastSt;
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java
index d4b26ad..ec89241 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java
@@ -17,19 +17,18 @@
package org.apache.zeppelin.interpreter.remote.mock;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.atomic.AtomicInteger;
-
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectWatcher;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicInteger;
+
public class MockInterpreterAngular extends Interpreter {
AtomicInteger numWatch = new AtomicInteger(0);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterB.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterB.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterB.java
deleted file mode 100644
index 7103335..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterB.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.interpreter.remote.mock;
-
-import java.util.List;
-import java.util.Properties;
-
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterException;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
-import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.InterpreterResult.Code;
-import org.apache.zeppelin.interpreter.WrappedInterpreter;
-import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
-import org.apache.zeppelin.scheduler.Scheduler;
-
-public class MockInterpreterB extends Interpreter {
-
- public MockInterpreterB(Properties property) {
- super(property);
- }
-
- @Override
- public void open() {
- //new RuntimeException().printStackTrace();
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public InterpreterResult interpret(String st, InterpreterContext context) {
- MockInterpreterA intpA = getInterpreterA();
- String intpASt = intpA.getLastStatement();
- long timeToSleep = Long.parseLong(st);
- if (intpASt != null) {
- timeToSleep += Long.parseLong(intpASt);
- }
- try {
- Thread.sleep(timeToSleep);
- } catch (NumberFormatException | InterruptedException e) {
- throw new InterpreterException(e);
- }
- return new InterpreterResult(Code.SUCCESS, Long.toString(timeToSleep));
- }
-
- @Override
- public void cancel(InterpreterContext context) {
-
- }
-
- @Override
- public FormType getFormType() {
- return FormType.NATIVE;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-
- @Override
- public List<InterpreterCompletion> completion(String buf, int cursor,
- InterpreterContext interpreterContext) {
- return null;
- }
-
- public MockInterpreterA getInterpreterA() {
- InterpreterGroup interpreterGroup = getInterpreterGroup();
- synchronized (interpreterGroup) {
- for (List<Interpreter> interpreters : interpreterGroup.values()) {
- boolean belongsToSameNoteGroup = false;
- MockInterpreterA a = null;
- for (Interpreter intp : interpreters) {
- if (intp.getClassName().equals(MockInterpreterA.class.getName())) {
- Interpreter p = intp;
- while (p instanceof WrappedInterpreter) {
- p = ((WrappedInterpreter) p).getInnerInterpreter();
- }
- a = (MockInterpreterA) p;
- }
-
- Interpreter p = intp;
- while (p instanceof WrappedInterpreter) {
- p = ((WrappedInterpreter) p).getInnerInterpreter();
- }
- if (this == p) {
- belongsToSameNoteGroup = true;
- }
- }
- if (belongsToSameNoteGroup) {
- return a;
- }
- }
- }
- return null;
- }
-
- @Override
- public Scheduler getScheduler() {
- MockInterpreterA intpA = getInterpreterA();
- if (intpA != null) {
- return intpA.getScheduler();
- }
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterEnv.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterEnv.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterEnv.java
deleted file mode 100644
index 12e11f7..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterEnv.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.zeppelin.interpreter.remote.mock;
-
-import org.apache.zeppelin.interpreter.*;
-import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
-import org.apache.zeppelin.scheduler.Scheduler;
-import org.apache.zeppelin.scheduler.SchedulerFactory;
-
-import java.util.List;
-import java.util.Properties;
-
-
-public class MockInterpreterEnv extends Interpreter {
-
- public MockInterpreterEnv(Properties property) {
- super(property);
- }
-
- @Override
- public void open() {
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public InterpreterResult interpret(String st, InterpreterContext context) {
- String[] cmd = st.split(" ");
- if (cmd[0].equals("getEnv")) {
- return new InterpreterResult(InterpreterResult.Code.SUCCESS, System.getenv(cmd[1]));
- } else if (cmd[0].equals("getProperty")){
- return new InterpreterResult(InterpreterResult.Code.SUCCESS, System.getProperty(cmd[1]));
- } else {
- return new InterpreterResult(InterpreterResult.Code.ERROR, cmd[0]);
- }
- }
-
- @Override
- public void cancel(InterpreterContext context) {
-
- }
-
- @Override
- public FormType getFormType() {
- return FormType.NATIVE;
- }
-
- @Override
- public int getProgress(InterpreterContext context) {
- return 0;
- }
-
- @Override
- public List<InterpreterCompletion> completion(String buf, int cursor,
- InterpreterContext interpreterContext) {
- return null;
- }
-
- @Override
- public Scheduler getScheduler() {
- return SchedulerFactory.singleton().createOrGetFIFOScheduler("interpreter_" + this.hashCode());
- }
-}
-
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java
index 349315c..1890cbc 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java
@@ -16,7 +16,10 @@
*/
package org.apache.zeppelin.interpreter.remote.mock;
-import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterResourcePool.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterResourcePool.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterResourcePool.java
index c4ff6ab..ee9f15c 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterResourcePool.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterResourcePool.java
@@ -17,22 +17,19 @@
package org.apache.zeppelin.interpreter.remote.mock;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.atomic.AtomicInteger;
-
import com.google.gson.Gson;
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.display.AngularObjectWatcher;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.resource.Resource;
import org.apache.zeppelin.resource.ResourcePool;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicInteger;
+
public class MockInterpreterResourcePool extends Interpreter {
AtomicInteger numWatch = new AtomicInteger(0);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java
deleted file mode 100644
index 5632513..0000000
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.zeppelin.notebook;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
-import org.apache.zeppelin.dep.Dependency;
-import org.apache.zeppelin.dep.DependencyResolver;
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterFactory;
-import org.apache.zeppelin.interpreter.InterpreterInfo;
-import org.apache.zeppelin.interpreter.InterpreterOption;
-import org.apache.zeppelin.interpreter.DefaultInterpreterProperty;
-import org.apache.zeppelin.interpreter.InterpreterProperty;
-import org.apache.zeppelin.interpreter.InterpreterSettingManager;
-import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
-import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
-import org.apache.zeppelin.interpreter.mock.MockInterpreter11;
-import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-public class NoteInterpreterLoaderTest {
-
- private File tmpDir;
- private ZeppelinConfiguration conf;
- private InterpreterFactory factory;
- private InterpreterSettingManager interpreterSettingManager;
- private DependencyResolver depResolver;
-
- @Before
- public void setUp() throws Exception {
- tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
- tmpDir.mkdirs();
- new File(tmpDir, "conf").mkdirs();
-
- System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
-
- conf = ZeppelinConfiguration.create();
-
- depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
-
- ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
- interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, Maps.<String, Object>newHashMap()));
- interpreterInfos.add(new InterpreterInfo(MockInterpreter11.class.getName(), "mock11", false, Maps.<String, Object>newHashMap()));
- ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
- interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, Maps.<String, Object>newHashMap()));
-
- interpreterSettingManager.add("group1", interpreterInfos, Lists.<Dependency>newArrayList(), new InterpreterOption(), Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock", null);
- interpreterSettingManager.add("group2", interpreterInfos2, Lists.<Dependency>newArrayList(), new InterpreterOption(), Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock", null);
-
- interpreterSettingManager.createNewSetting("group1", "group1", Lists.<Dependency>newArrayList(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
- interpreterSettingManager.createNewSetting("group2", "group2", Lists.<Dependency>newArrayList(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
-
-
- }
-
- @After
- public void tearDown() throws Exception {
- delete(tmpDir);
- Interpreter.registeredInterpreters.clear();
- }
-
- @Test
- public void testGetInterpreter() throws IOException {
- interpreterSettingManager.setInterpreters("user", "note", interpreterSettingManager.getDefaultInterpreterSettingList());
-
- // when there're no interpreter selection directive
- assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", null).getClassName());
- assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", "").getClassName());
- assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", " ").getClassName());
-
- // when group name is omitted
- assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("user", "note", "mock11").getClassName());
-
- // when 'name' is ommitted
- assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", "group1").getClassName());
- assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("user", "note", "group2").getClassName());
-
- // when nothing is ommitted
- assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", "group1.mock1").getClassName());
- assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("user", "note", "group1.mock11").getClassName());
- assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("user", "note", "group2.mock2").getClassName());
-
- interpreterSettingManager.closeNote("user", "note");
- }
-
- @Test
- public void testNoteSession() throws IOException {
- interpreterSettingManager.setInterpreters("user", "noteA", interpreterSettingManager.getDefaultInterpreterSettingList());
- interpreterSettingManager.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
-
- interpreterSettingManager.setInterpreters("user", "noteB", interpreterSettingManager.getDefaultInterpreterSettingList());
- interpreterSettingManager.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
-
- // interpreters are not created before accessing it
- assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA"));
- assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB"));
-
- factory.getInterpreter("user", "noteA", null).open();
- factory.getInterpreter("user", "noteB", null).open();
-
- assertTrue(
- factory.getInterpreter("user", "noteA", null).getInterpreterGroup().getId().equals(
- factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId()));
-
- // interpreters are created after accessing it
- assertNotNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA"));
- assertNotNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB"));
-
- // invalid close
- interpreterSettingManager.closeNote("user", "note");
- assertNotNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA"));
- assertNotNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB"));
-
- // when
- interpreterSettingManager.closeNote("user", "noteA");
- interpreterSettingManager.closeNote("user", "noteB");
-
- // interpreters are destroyed after close
- assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA"));
- assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB"));
-
- }
-
- @Test
- public void testNotePerInterpreterProcess() throws IOException {
- interpreterSettingManager.setInterpreters("user", "noteA", interpreterSettingManager.getDefaultInterpreterSettingList());
- interpreterSettingManager.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
-
- interpreterSettingManager.setInterpreters("user", "noteB", interpreterSettingManager.getDefaultInterpreterSettingList());
- interpreterSettingManager.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
-
- // interpreters are not created before accessing it
- assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session"));
- assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session"));
-
- factory.getInterpreter("user", "noteA", null).open();
- factory.getInterpreter("user", "noteB", null).open();
-
- // per note interpreter process
- assertFalse(
- factory.getInterpreter("user", "noteA", null).getInterpreterGroup().getId().equals(
- factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId()));
-
- // interpreters are created after accessing it
- assertNotNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session"));
- assertNotNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session"));
-
- // when
- interpreterSettingManager.closeNote("user", "noteA");
- interpreterSettingManager.closeNote("user", "noteB");
-
- // interpreters are destroyed after close
- assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session"));
- assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session"));
- }
-
- @Test
- public void testNoteInterpreterCloseForAll() throws IOException {
- interpreterSettingManager.setInterpreters("user", "FitstNote", interpreterSettingManager.getDefaultInterpreterSettingList());
- interpreterSettingManager.getInterpreterSettings("FitstNote").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
-
- interpreterSettingManager.setInterpreters("user", "yourFirstNote", interpreterSettingManager.getDefaultInterpreterSettingList());
- interpreterSettingManager.getInterpreterSettings("yourFirstNote").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
-
- // interpreters are not created before accessing it
- assertNull(interpreterSettingManager.getInterpreterSettings("FitstNote").get(0).getInterpreterGroup("user", "FitstNote").get("FitstNote"));
- assertNull(interpreterSettingManager.getInterpreterSettings("yourFirstNote").get(0).getInterpreterGroup("user", "yourFirstNote").get("yourFirstNote"));
-
- Interpreter firstNoteIntp = factory.getInterpreter("user", "FitstNote", "group1.mock1");
- Interpreter yourFirstNoteIntp = factory.getInterpreter("user", "yourFirstNote", "group1.mock1");
-
- firstNoteIntp.open();
- yourFirstNoteIntp.open();
-
- assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
- assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
-
- interpreterSettingManager.closeNote("user", "FitstNote");
-
- assertFalse(((LazyOpenInterpreter)firstNoteIntp).isOpen());
- assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
-
- //reopen
- firstNoteIntp.open();
-
- assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
- assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
-
- // invalid check
- interpreterSettingManager.closeNote("invalid", "Note");
-
- assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
- assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
-
- // invalid contains value check
- interpreterSettingManager.closeNote("u", "Note");
-
- assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
- assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
- }
-
-
- private void delete(File file){
- if(file.isFile()) file.delete();
- else if(file.isDirectory()){
- File [] files = file.listFiles();
- if(files!=null && files.length>0){
- for(File f : files){
- delete(f);
- }
- }
- file.delete();
- }
- }
-}
[4/9] zeppelin git commit: [ZEPPELIN-2627] Interpreter Component
Refactoring
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java
new file mode 100644
index 0000000..ac9d536
--- /dev/null
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java
@@ -0,0 +1,390 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.scheduler;
+
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
+import org.apache.zeppelin.scheduler.Job.Status;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+
+/**
+ * RemoteScheduler runs in ZeppelinServer and proxies Scheduler running on RemoteInterpreter
+ *
+ */
+public class RemoteScheduler implements Scheduler {
+ Logger logger = LoggerFactory.getLogger(RemoteScheduler.class);
+
+ List<Job> queue = new LinkedList<>();
+ List<Job> running = new LinkedList<>();
+ private ExecutorService executor;
+ private SchedulerListener listener;
+ boolean terminate = false;
+ private String name;
+ private int maxConcurrency;
+ private final String sessionId;
+ private RemoteInterpreter remoteInterpreter;
+
+ public RemoteScheduler(String name, ExecutorService executor, String sessionId,
+ RemoteInterpreter remoteInterpreter, SchedulerListener listener,
+ int maxConcurrency) {
+ this.name = name;
+ this.executor = executor;
+ this.listener = listener;
+ this.sessionId = sessionId;
+ this.remoteInterpreter = remoteInterpreter;
+ this.maxConcurrency = maxConcurrency;
+ }
+
+ @Override
+ public void run() {
+ while (terminate == false) {
+ Job job = null;
+
+ synchronized (queue) {
+ if (running.size() >= maxConcurrency || queue.isEmpty() == true) {
+ try {
+ queue.wait(500);
+ } catch (InterruptedException e) {
+ logger.error("Exception in RemoteScheduler while run queue.wait", e);
+ }
+ continue;
+ }
+
+ job = queue.remove(0);
+ running.add(job);
+ }
+
+ // run
+ Scheduler scheduler = this;
+ JobRunner jobRunner = new JobRunner(scheduler, job);
+ executor.execute(jobRunner);
+
+ // wait until it is submitted to the remote
+ while (!jobRunner.isJobSubmittedInRemote()) {
+ synchronized (queue) {
+ try {
+ queue.wait(500);
+ } catch (InterruptedException e) {
+ logger.error("Exception in RemoteScheduler while jobRunner.isJobSubmittedInRemote " +
+ "queue.wait", e);
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Collection<Job> getJobsWaiting() {
+ List<Job> ret = new LinkedList<>();
+ synchronized (queue) {
+ for (Job job : queue) {
+ ret.add(job);
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public Job removeFromWaitingQueue(String jobId) {
+ synchronized (queue) {
+ Iterator<Job> it = queue.iterator();
+ while (it.hasNext()) {
+ Job job = it.next();
+ if (job.getId().equals(jobId)) {
+ it.remove();
+ return job;
+ }
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Collection<Job> getJobsRunning() {
+ List<Job> ret = new LinkedList<>();
+ synchronized (queue) {
+ for (Job job : running) {
+ ret.add(job);
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public void submit(Job job) {
+ if (terminate) {
+ throw new RuntimeException("Scheduler already terminated");
+ }
+ job.setStatus(Status.PENDING);
+
+ synchronized (queue) {
+ queue.add(job);
+ queue.notify();
+ }
+ }
+
+ public void setMaxConcurrency(int maxConcurrency) {
+ this.maxConcurrency = maxConcurrency;
+ synchronized (queue) {
+ queue.notify();
+ }
+ }
+
+ /**
+ * Role of the class is get status info from remote process from PENDING to
+ * RUNNING status.
+ */
+ private class JobStatusPoller extends Thread {
+ private long initialPeriodMsec;
+ private long initialPeriodCheckIntervalMsec;
+ private long checkIntervalMsec;
+ private volatile boolean terminate;
+ private JobListener listener;
+ private Job job;
+ volatile Status lastStatus;
+
+ public JobStatusPoller(long initialPeriodMsec,
+ long initialPeriodCheckIntervalMsec, long checkIntervalMsec, Job job,
+ JobListener listener) {
+ setName("JobStatusPoller-" + job.getId());
+ this.initialPeriodMsec = initialPeriodMsec;
+ this.initialPeriodCheckIntervalMsec = initialPeriodCheckIntervalMsec;
+ this.checkIntervalMsec = checkIntervalMsec;
+ this.job = job;
+ this.listener = listener;
+ this.terminate = false;
+ }
+
+ @Override
+ public void run() {
+ long started = System.currentTimeMillis();
+ while (terminate == false) {
+ long current = System.currentTimeMillis();
+ long interval;
+ if (current - started < initialPeriodMsec) {
+ interval = initialPeriodCheckIntervalMsec;
+ } else {
+ interval = checkIntervalMsec;
+ }
+
+ synchronized (this) {
+ try {
+ this.wait(interval);
+ } catch (InterruptedException e) {
+ logger.error("Exception in RemoteScheduler while run this.wait", e);
+ }
+ }
+
+ if (terminate) {
+ // terminated by shutdown
+ break;
+ }
+
+ Status newStatus = getStatus();
+ if (newStatus == Status.UNKNOWN) { // unknown
+ continue;
+ }
+
+ if (newStatus != Status.READY && newStatus != Status.PENDING) {
+ // we don't need more
+ break;
+ }
+ }
+ terminate = true;
+ }
+
+ public void shutdown() {
+ terminate = true;
+ synchronized (this) {
+ this.notify();
+ }
+ }
+
+
+ private Status getLastStatus() {
+ if (terminate == true) {
+ if (job.getErrorMessage() != null) {
+ return Status.ERROR;
+ } else if (lastStatus != Status.FINISHED &&
+ lastStatus != Status.ERROR &&
+ lastStatus != Status.ABORT) {
+ return Status.FINISHED;
+ } else {
+ return (lastStatus == null) ? Status.FINISHED : lastStatus;
+ }
+ } else {
+ return (lastStatus == null) ? Status.UNKNOWN : lastStatus;
+ }
+ }
+
+ public synchronized Status getStatus() {
+ if (!remoteInterpreter.isOpened()) {
+ return getLastStatus();
+ }
+ Status status = Status.valueOf(remoteInterpreter.getStatus(job.getId()));
+ if (status == Status.UNKNOWN) {
+ // not found this job in the remote schedulers.
+ // maybe not submitted, maybe already finished
+ //Status status = getLastStatus();
+ listener.afterStatusChange(job, null, null);
+ return job.getStatus();
+ }
+ lastStatus = status;
+ listener.afterStatusChange(job, null, status);
+ return status;
+ }
+ }
+
+ //TODO(zjffdu) need to refactor the schdule module which is too complicated
+ private class JobRunner implements Runnable, JobListener {
+ private final Logger logger = LoggerFactory.getLogger(JobRunner.class);
+ private Scheduler scheduler;
+ private Job job;
+ private volatile boolean jobExecuted;
+ volatile boolean jobSubmittedRemotely;
+
+ public JobRunner(Scheduler scheduler, Job job) {
+ this.scheduler = scheduler;
+ this.job = job;
+ jobExecuted = false;
+ jobSubmittedRemotely = false;
+ }
+
+ public boolean isJobSubmittedInRemote() {
+ return jobSubmittedRemotely;
+ }
+
+ @Override
+ public void run() {
+ if (job.isAborted()) {
+ synchronized (queue) {
+ job.setStatus(Status.ABORT);
+ job.aborted = false;
+
+ running.remove(job);
+ queue.notify();
+ }
+ jobSubmittedRemotely = true;
+
+ return;
+ }
+
+ JobStatusPoller jobStatusPoller = new JobStatusPoller(1500, 100, 500,
+ job, this);
+ jobStatusPoller.start();
+
+ if (listener != null) {
+ listener.jobStarted(scheduler, job);
+ }
+ job.run();
+
+ jobExecuted = true;
+ jobSubmittedRemotely = true;
+
+ jobStatusPoller.shutdown();
+ try {
+ jobStatusPoller.join();
+ } catch (InterruptedException e) {
+ logger.error("JobStatusPoller interrupted", e);
+ }
+
+ // set job status based on result.
+ Object jobResult = job.getReturn();
+ if (job.isAborted()) {
+ job.setStatus(Status.ABORT);
+ } else if (job.getException() != null) {
+ logger.debug("Job ABORT, " + job.getId());
+ job.setStatus(Status.ERROR);
+ } else if (jobResult != null && jobResult instanceof InterpreterResult
+ && ((InterpreterResult) jobResult).code() == Code.ERROR) {
+ logger.debug("Job Error, " + job.getId());
+ job.setStatus(Status.ERROR);
+ } else {
+ logger.debug("Job Finished, " + job.getId());
+ job.setStatus(Status.FINISHED);
+ }
+
+ synchronized (queue) {
+ if (listener != null) {
+ listener.jobFinished(scheduler, job);
+ }
+
+ // reset aborted flag to allow retry
+ job.aborted = false;
+
+ running.remove(job);
+ queue.notify();
+ }
+ }
+
+ @Override
+ public void onProgressUpdate(Job job, int progress) {
+ }
+
+ @Override
+ public void beforeStatusChange(Job job, Status before, Status after) {
+ }
+
+ @Override
+ public void afterStatusChange(Job job, Status before, Status after) {
+ // Update remoteStatus
+ if (jobExecuted == false) {
+ if (after == Status.FINISHED || after == Status.ABORT
+ || after == Status.ERROR) {
+ // it can be status of last run.
+ // so not updating the remoteStatus
+ return;
+ } else if (after == Status.RUNNING) {
+ jobSubmittedRemotely = true;
+ job.setStatus(Status.RUNNING);
+ }
+ } else {
+ jobSubmittedRemotely = true;
+ }
+
+ // only set status when it is RUNNING
+ // We would set other status based on the interpret result
+ if (after == Status.RUNNING) {
+ job.setStatus(Status.RUNNING);
+ }
+ }
+ }
+
+ @Override
+ public void stop() {
+ terminate = true;
+ synchronized (queue) {
+ queue.notify();
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java
deleted file mode 100644
index be45b9e..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.util;
-
-import org.apache.commons.lang3.StringUtils;
-
-import java.io.IOException;
-import java.util.Properties;
-
-/**
- * TODO(moon) : add description.
- */
-public class Util {
- private static final String PROJECT_PROPERTIES_VERSION_KEY = "version";
- private static final String GIT_PROPERTIES_COMMIT_ID_KEY = "git.commit.id.abbrev";
- private static final String GIT_PROPERTIES_COMMIT_TS_KEY = "git.commit.time";
-
- private static Properties projectProperties;
- private static Properties gitProperties;
-
- static {
- projectProperties = new Properties();
- gitProperties = new Properties();
- try {
- projectProperties.load(Util.class.getResourceAsStream("/project.properties"));
- gitProperties.load(Util.class.getResourceAsStream("/git.properties"));
- } catch (IOException e) {
- //Fail to read project.properties
- }
- }
-
- /**
- * Get Zeppelin version
- *
- * @return Current Zeppelin version
- */
- public static String getVersion() {
- return StringUtils.defaultIfEmpty(projectProperties.getProperty(PROJECT_PROPERTIES_VERSION_KEY),
- StringUtils.EMPTY);
- }
-
- /**
- * Get Zeppelin Git latest commit id
- *
- * @return Latest Zeppelin commit id
- */
- public static String getGitCommitId() {
- return StringUtils.defaultIfEmpty(gitProperties.getProperty(GIT_PROPERTIES_COMMIT_ID_KEY),
- StringUtils.EMPTY);
- }
-
- /**
- * Get Zeppelin Git latest commit timestamp
- *
- * @return Latest Zeppelin commit timestamp
- */
- public static String getGitTimestamp() {
- return StringUtils.defaultIfEmpty(gitProperties.getProperty(GIT_PROPERTIES_COMMIT_TS_KEY),
- StringUtils.EMPTY);
- }
-}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
index 305258a..bf49490 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
@@ -16,15 +16,18 @@
*/
package org.apache.zeppelin.helium;
-import com.google.common.collect.Maps;
-import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.dep.Dependency;
-import org.apache.zeppelin.dep.DependencyResolver;
-import org.apache.zeppelin.interpreter.*;
-import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
-import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
-import org.apache.zeppelin.notebook.*;
+import org.apache.zeppelin.interpreter.AbstractInterpreterTest;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterResultMessage;
+import org.apache.zeppelin.interpreter.InterpreterSetting;
+import org.apache.zeppelin.notebook.ApplicationState;
+import org.apache.zeppelin.notebook.JobListenerFactory;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.Notebook;
+import org.apache.zeppelin.notebook.NotebookAuthorization;
+import org.apache.zeppelin.notebook.Paragraph;
+import org.apache.zeppelin.notebook.ParagraphJobListener;
import org.apache.zeppelin.notebook.repo.VFSNotebookRepo;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.SchedulerFactory;
@@ -35,24 +38,16 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import java.io.File;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
-public class HeliumApplicationFactoryTest implements JobListenerFactory {
- private File tmpDir;
- private File notebookDir;
- private ZeppelinConfiguration conf;
+public class HeliumApplicationFactoryTest extends AbstractInterpreterTest implements JobListenerFactory {
+
private SchedulerFactory schedulerFactory;
- private DependencyResolver depResolver;
- private InterpreterFactory factory;
- private InterpreterSettingManager interpreterSettingManager;
private VFSNotebookRepo notebookRepo;
private Notebook notebook;
private HeliumApplicationFactory heliumAppFactory;
@@ -60,46 +55,15 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
@Before
public void setUp() throws Exception {
- tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZepelinLTest_"+System.currentTimeMillis());
- tmpDir.mkdirs();
- File confDir = new File(tmpDir, "conf");
- confDir.mkdirs();
- notebookDir = new File(tmpDir + "/notebook");
- notebookDir.mkdirs();
-
- File home = new File(getClass().getClassLoader().getResource("note").getFile()) // zeppelin/zeppelin-zengine/target/test-classes/note
- .getParentFile() // zeppelin/zeppelin-zengine/target/test-classes
- .getParentFile() // zeppelin/zeppelin-zengine/target
- .getParentFile() // zeppelin/zeppelin-zengine
- .getParentFile(); // zeppelin
-
- System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), home.getAbsolutePath());
- System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), tmpDir.getAbsolutePath() + "/conf");
- System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
-
- conf = new ZeppelinConfiguration();
-
- this.schedulerFactory = new SchedulerFactory();
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "mock1,mock2");
+ super.setUp();
+ this.schedulerFactory = SchedulerFactory.singleton();
heliumAppFactory = new HeliumApplicationFactory();
- depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- factory = new InterpreterFactory(conf, null, null, heliumAppFactory, depResolver, false, interpreterSettingManager);
- HashMap<String, String> env = new HashMap<>();
- env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
- factory.setEnv(env);
-
- ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
- interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
- interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
-
- ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
- interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock2", interpreterInfos2, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock2", null);
- interpreterSettingManager.createNewSetting("mock2", "mock2", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
+ // set AppEventListener properly
+ for (InterpreterSetting interpreterSetting : interpreterSettingManager.get()) {
+ interpreterSetting.setAppEventListener(heliumAppFactory);
+ }
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
@@ -108,7 +72,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
conf,
notebookRepo,
schedulerFactory,
- factory,
+ interpreterFactory,
interpreterSettingManager,
this,
search,
@@ -124,16 +88,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
@After
public void tearDown() throws Exception {
- List<InterpreterSetting> settings = interpreterSettingManager.get();
- for (InterpreterSetting setting : settings) {
- for (InterpreterGroup intpGroup : setting.getAllInterpreterGroups()) {
- intpGroup.close();
- }
- }
-
- FileUtils.deleteDirectory(tmpDir);
- System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(),
- ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getStringValue());
+ super.tearDown();
}
@@ -150,7 +105,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters("user", note1.getId(),interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding("user", note1.getId(),interpreterSettingManager.getInterpreterSettingIds());
Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@@ -196,7 +151,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
- interpreterSettingManager.setInterpreters("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreterBinding("user", note1.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@@ -236,7 +191,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
- notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@@ -297,7 +252,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
- notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
+ notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getInterpreterSettingIds());
String mock1IntpSettingId = null;
for (InterpreterSetting setting : notebook.getBindedInterpreterSettings(note1.getId())) {
if (setting.getName().equals("mock1")) {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java
index 6b4932d..bdd639e 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java
@@ -52,7 +52,7 @@ public class HeliumTest {
// given
File heliumConf = new File(tmpDir, "helium.conf");
Helium helium = new Helium(heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(),
- null, null, null);
+ null, null, null, null);
assertFalse(heliumConf.exists());
// when
@@ -63,14 +63,14 @@ public class HeliumTest {
// then load without exception
Helium heliumRestored = new Helium(
- heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null);
+ heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null, null);
}
@Test
public void testRestoreRegistryInstances() throws IOException, URISyntaxException, TaskRunnerException {
File heliumConf = new File(tmpDir, "helium.conf");
Helium helium = new Helium(
- heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null);
+ heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null, null);
HeliumTestRegistry registry1 = new HeliumTestRegistry("r1", "r1");
HeliumTestRegistry registry2 = new HeliumTestRegistry("r2", "r2");
helium.addRegistry(registry1);
@@ -105,7 +105,7 @@ public class HeliumTest {
public void testRefresh() throws IOException, URISyntaxException, TaskRunnerException {
File heliumConf = new File(tmpDir, "helium.conf");
Helium helium = new Helium(
- heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null);
+ heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null, null);
HeliumTestRegistry registry1 = new HeliumTestRegistry("r1", "r1");
helium.addRegistry(registry1);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java
new file mode 100644
index 0000000..21d7526
--- /dev/null
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java
@@ -0,0 +1,74 @@
+package org.apache.zeppelin.interpreter;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.display.AngularObjectRegistryListener;
+import org.apache.zeppelin.helium.ApplicationEventListener;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
+import org.junit.After;
+import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+import static org.mockito.Mockito.mock;
+
+
+/**
+ * This class will load configuration files under
+ * src/test/resources/interpreter
+ * src/test/resources/conf
+ *
+ * to construct InterpreterSettingManager and InterpreterFactory properly
+ *
+ */
+public abstract class AbstractInterpreterTest {
+ protected static final Logger LOGGER = LoggerFactory.getLogger(AbstractInterpreterTest.class);
+ private static final String INTERPRETER_SCRIPT =
+ System.getProperty("os.name").startsWith("Windows") ?
+ "../bin/interpreter.cmd" :
+ "../bin/interpreter.sh";
+
+ protected InterpreterSettingManager interpreterSettingManager;
+ protected InterpreterFactory interpreterFactory;
+ protected File testRootDir;
+ protected File interpreterDir;
+ protected File confDir;
+ protected File notebookDir;
+ protected ZeppelinConfiguration conf;
+
+ @Before
+ public void setUp() throws Exception {
+ // copy the resources files to a temp folder
+ testRootDir = new File(System.getProperty("java.io.tmpdir") + "/Zeppelin_Test_" + System.currentTimeMillis());
+ testRootDir.mkdirs();
+ LOGGER.info("Create tmp directory: {} as root folder of ZEPPELIN_INTERPRETER_DIR & ZEPPELIN_CONF_DIR", testRootDir.getAbsolutePath());
+ interpreterDir = new File(testRootDir, "interpreter");
+ confDir = new File(testRootDir, "conf");
+ notebookDir = new File(testRootDir, "notebook");
+
+ interpreterDir.mkdirs();
+ confDir.mkdirs();
+ notebookDir.mkdirs();
+
+ FileUtils.copyDirectory(new File("src/test/resources/interpreter"), interpreterDir);
+ FileUtils.copyDirectory(new File("src/test/resources/conf"), confDir);
+
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), confDir.getAbsolutePath());
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_DIR.getVarName(), interpreterDir.getAbsolutePath());
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER.getVarName(), INTERPRETER_SCRIPT);
+
+ conf = new ZeppelinConfiguration();
+ interpreterSettingManager = new InterpreterSettingManager(conf,
+ mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
+ interpreterFactory = new InterpreterFactory(interpreterSettingManager);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ interpreterSettingManager.close();
+ FileUtils.deleteDirectory(testRootDir);
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/DoubleEchoInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/DoubleEchoInterpreter.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/DoubleEchoInterpreter.java
new file mode 100644
index 0000000..be3d5be
--- /dev/null
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/DoubleEchoInterpreter.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.zeppelin.interpreter;
+
+import java.util.Properties;
+
+
+public class DoubleEchoInterpreter extends Interpreter {
+
+ public DoubleEchoInterpreter(Properties property) {
+ super(property);
+ }
+
+ @Override
+ public void open() {
+
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ return new InterpreterResult(InterpreterResult.Code.SUCCESS, st + "," + st);
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+
+ }
+
+ @Override
+ public FormType getFormType() {
+ return null;
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java
new file mode 100644
index 0000000..e7a04f3
--- /dev/null
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.zeppelin.interpreter;
+
+import java.util.Properties;
+
+/**
+ * Just return the received statement back
+ */
+public class EchoInterpreter extends Interpreter {
+
+ public EchoInterpreter(Properties property) {
+ super(property);
+ }
+
+ @Override
+ public void open() {
+
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public InterpreterResult interpret(String st, InterpreterContext context) {
+ if (Boolean.parseBoolean(property.getProperty("zeppelin.interpreter.echo.fail", "false"))) {
+ return new InterpreterResult(InterpreterResult.Code.ERROR);
+ } else {
+ return new InterpreterResult(InterpreterResult.Code.SUCCESS, st);
+ }
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) {
+
+ }
+
+ @Override
+ public FormType getFormType() {
+ return FormType.NATIVE;
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
index aaa8864..f3137d9 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
@@ -17,481 +17,50 @@
package org.apache.zeppelin.interpreter;
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.NullArgumentException;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
-import org.apache.zeppelin.dep.Dependency;
-import org.apache.zeppelin.dep.DependencyResolver;
-import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
-import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
-import org.apache.zeppelin.notebook.JobListenerFactory;
-import org.apache.zeppelin.notebook.Note;
-import org.apache.zeppelin.notebook.Notebook;
-import org.apache.zeppelin.notebook.NotebookAuthorization;
-import org.apache.zeppelin.notebook.repo.NotebookRepo;
-import org.apache.zeppelin.notebook.repo.VFSNotebookRepo;
-import org.apache.zeppelin.scheduler.SchedulerFactory;
-import org.apache.zeppelin.search.SearchService;
-import org.apache.zeppelin.user.AuthenticationInfo;
-import org.junit.After;
-import org.junit.Before;
import org.junit.Test;
-import org.mockito.Mock;
-import org.quartz.SchedulerException;
-import org.sonatype.aether.RepositoryException;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import java.io.IOException;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-public class InterpreterFactoryTest {
-
- private InterpreterFactory factory;
- private InterpreterSettingManager interpreterSettingManager;
- private File tmpDir;
- private ZeppelinConfiguration conf;
- private InterpreterContext context;
- private Notebook notebook;
- private NotebookRepo notebookRepo;
- private DependencyResolver depResolver;
- private SchedulerFactory schedulerFactory;
- private NotebookAuthorization notebookAuthorization;
- @Mock
- private JobListenerFactory jobListenerFactory;
-
- @Before
- public void setUp() throws Exception {
- tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
- tmpDir.mkdirs();
- new File(tmpDir, "conf").mkdirs();
- FileUtils.copyDirectory(new File("src/test/resources/interpreter"), new File(tmpDir, "interpreter"));
-
- System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
- System.setProperty(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(),
- "mock1,mock2,mock11,dev");
- conf = new ZeppelinConfiguration();
- schedulerFactory = new SchedulerFactory();
- depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
- context = new InterpreterContext("note", "id", null, "title", "text", null, null, null, null, null, null, null);
-
- ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
- interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
- Map<String, InterpreterProperty> intp1Properties = new HashMap<String, InterpreterProperty>();
- intp1Properties.put("PROPERTY_1",
- new InterpreterProperty("PROPERTY_1", "VALUE_1"));
- intp1Properties.put("property_2",
- new InterpreterProperty("property_2", "value_2"));
- interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), intp1Properties);
-
- ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
- interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock2", interpreterInfos2, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock2", null);
- interpreterSettingManager.createNewSetting("mock2", "mock2", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
-
- SearchService search = mock(SearchService.class);
- notebookRepo = new VFSNotebookRepo(conf);
- notebookAuthorization = NotebookAuthorization.init(conf);
- notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, jobListenerFactory, search,
- notebookAuthorization, null);
- }
- @After
- public void tearDown() throws Exception {
- FileUtils.deleteDirectory(tmpDir);
- }
+public class InterpreterFactoryTest extends AbstractInterpreterTest {
@Test
- public void testBasic() {
- List<InterpreterSetting> all = interpreterSettingManager.get();
- InterpreterSetting mock1Setting = null;
- for (InterpreterSetting setting : all) {
- if (setting.getName().equals("mock1")) {
- mock1Setting = setting;
- break;
- }
- }
-
-// mock1Setting = factory.createNewSetting("mock11", "mock1", new ArrayList<Dependency>(), new InterpreterOption(false), new Properties());
+ public void testGetFactory() throws IOException {
+ // no default interpreter because there's no interpreter setting binded to this note
+ assertNull(interpreterFactory.getInterpreter("user1", "note1", ""));
- InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("user", "sharedProcess");
- factory.createInterpretersForNote(mock1Setting, "user", "sharedProcess", "session");
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds());
+ assertTrue(interpreterFactory.getInterpreter("user1", "note1", "") instanceof RemoteInterpreter);
+ RemoteInterpreter remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("user1", "note1", "");
+ // EchoInterpreter is the default interpreter (see zeppelin-interpreter/src/test/resources/conf/interpreter.json)
+ assertEquals(EchoInterpreter.class.getName(), remoteInterpreter.getClassName());
- // get interpreter
- assertNotNull("get Interpreter", interpreterGroup.get("session").get(0));
+ assertTrue(interpreterFactory.getInterpreter("user1", "note1", "test") instanceof RemoteInterpreter);
+ remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("user1", "note1", "test");
+ assertEquals(EchoInterpreter.class.getName(), remoteInterpreter.getClassName());
- // try to get unavailable interpreter
- assertNull(interpreterSettingManager.get("unknown"));
+ assertTrue(interpreterFactory.getInterpreter("user1", "note1", "echo") instanceof RemoteInterpreter);
+ remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("user1", "note1", "echo");
+ assertEquals(EchoInterpreter.class.getName(), remoteInterpreter.getClassName());
- // restart interpreter
- interpreterSettingManager.restart(mock1Setting.getId());
- assertNull(mock1Setting.getInterpreterGroup("user", "sharedProcess").get("session"));
+ assertTrue(interpreterFactory.getInterpreter("user1", "note1", "double_echo") instanceof RemoteInterpreter);
+ remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("user1", "note1", "double_echo");
+ assertEquals(DoubleEchoInterpreter.class.getName(), remoteInterpreter.getClassName());
}
- @Test
- public void testRemoteRepl() throws Exception {
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
- interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
- Map<String, InterpreterProperty> intp1Properties = new HashMap<String, InterpreterProperty>();
- intp1Properties.put("PROPERTY_1",
- new InterpreterProperty("PROPERTY_1", "VALUE_1"));
- intp1Properties.put("property_2", new InterpreterProperty("property_2", "value_2"));
- interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), intp1Properties);
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
- List<InterpreterSetting> all = interpreterSettingManager.get();
- InterpreterSetting mock1Setting = null;
- for (InterpreterSetting setting : all) {
- if (setting.getName().equals("mock1")) {
- mock1Setting = setting;
- break;
- }
- }
- InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("user", "sharedProcess");
- factory.createInterpretersForNote(mock1Setting, "user", "sharedProcess", "session");
- // get interpreter
- assertNotNull("get Interpreter", interpreterGroup.get("session").get(0));
- assertTrue(interpreterGroup.get("session").get(0) instanceof LazyOpenInterpreter);
- LazyOpenInterpreter lazyInterpreter = (LazyOpenInterpreter)(interpreterGroup.get("session").get(0));
- assertTrue(lazyInterpreter.getInnerInterpreter() instanceof RemoteInterpreter);
- RemoteInterpreter remoteInterpreter = (RemoteInterpreter) lazyInterpreter.getInnerInterpreter();
- assertEquals("VALUE_1", remoteInterpreter.getEnv().get("PROPERTY_1"));
- assertEquals("value_2", remoteInterpreter.getProperty("property_2"));
+ @Test(expected = InterpreterException.class)
+ public void testUnknownRepl1() throws IOException {
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds());
+ interpreterFactory.getInterpreter("user1", "note1", "test.unknown_repl");
}
- /**
- * 2 users' interpreters in scoped mode. Each user has one session. Restarting user1's interpreter
- * won't affect user2's interpreter
- * @throws Exception
- */
@Test
- public void testRestartInterpreterInScopedMode() throws Exception {
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
- interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
- Map<String, InterpreterProperty> intp1Properties = new HashMap<String, InterpreterProperty>();
- intp1Properties.put("PROPERTY_1",
- new InterpreterProperty("PROPERTY_1", "VALUE_1"));
- intp1Properties.put("property_2",
- new InterpreterProperty("property_2", "value_2"));
- interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), intp1Properties);
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
- List<InterpreterSetting> all = interpreterSettingManager.get();
- InterpreterSetting mock1Setting = null;
- for (InterpreterSetting setting : all) {
- if (setting.getName().equals("mock1")) {
- mock1Setting = setting;
- break;
- }
- }
- mock1Setting.getOption().setPerUser("scoped");
- mock1Setting.getOption().setPerNote("shared");
- // set remote as false so that we won't create new remote interpreter process
- mock1Setting.getOption().setRemote(false);
- mock1Setting.getOption().setHost("localhost");
- mock1Setting.getOption().setPort(2222);
- InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("user1", "sharedProcess");
- factory.createInterpretersForNote(mock1Setting, "user1", "sharedProcess", "user1");
- factory.createInterpretersForNote(mock1Setting, "user2", "sharedProcess", "user2");
-
- LazyOpenInterpreter interpreter1 = (LazyOpenInterpreter)interpreterGroup.get("user1").get(0);
- interpreter1.open();
- LazyOpenInterpreter interpreter2 = (LazyOpenInterpreter)interpreterGroup.get("user2").get(0);
- interpreter2.open();
-
- mock1Setting.closeAndRemoveInterpreterGroup("sharedProcess", "user1");
- assertFalse(interpreter1.isOpen());
- assertTrue(interpreter2.isOpen());
- }
-
- /**
- * 2 users' interpreters in isolated mode. Each user has one interpreterGroup. Restarting user1's interpreter
- * won't affect user2's interpreter
- * @throws Exception
- */
- @Test
- public void testRestartInterpreterInIsolatedMode() throws Exception {
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
- interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
- interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
- Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
- Map<String, InterpreterProperty> intp1Properties = new HashMap<String, InterpreterProperty>();
- intp1Properties.put("PROPERTY_1",
- new InterpreterProperty("PROPERTY_1", "VALUE_1"));
- intp1Properties.put("property_2",
- new InterpreterProperty("property_2", "value_2"));
- interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), intp1Properties);
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
- List<InterpreterSetting> all = interpreterSettingManager.get();
- InterpreterSetting mock1Setting = null;
- for (InterpreterSetting setting : all) {
- if (setting.getName().equals("mock1")) {
- mock1Setting = setting;
- break;
- }
- }
- mock1Setting.getOption().setPerUser("isolated");
- mock1Setting.getOption().setPerNote("shared");
- // set remote as false so that we won't create new remote interpreter process
- mock1Setting.getOption().setRemote(false);
- mock1Setting.getOption().setHost("localhost");
- mock1Setting.getOption().setPort(2222);
- InterpreterGroup interpreterGroup1 = mock1Setting.getInterpreterGroup("user1", "note1");
- InterpreterGroup interpreterGroup2 = mock1Setting.getInterpreterGroup("user2", "note2");
- factory.createInterpretersForNote(mock1Setting, "user1", "note1", "shared_session");
- factory.createInterpretersForNote(mock1Setting, "user2", "note2", "shared_session");
-
- LazyOpenInterpreter interpreter1 = (LazyOpenInterpreter)interpreterGroup1.get("shared_session").get(0);
- interpreter1.open();
- LazyOpenInterpreter interpreter2 = (LazyOpenInterpreter)interpreterGroup2.get("shared_session").get(0);
- interpreter2.open();
-
- mock1Setting.closeAndRemoveInterpreterGroup("note1", "user1");
- assertFalse(interpreter1.isOpen());
- assertTrue(interpreter2.isOpen());
- }
-
- @Test
- public void testFactoryDefaultList() throws IOException, RepositoryException {
- // get default settings
- List<String> all = interpreterSettingManager.getDefaultInterpreterSettingList();
- assertTrue(interpreterSettingManager.get().size() >= all.size());
- }
-
- @Test
- public void testExceptions() throws InterpreterException, IOException, RepositoryException {
- List<String> all = interpreterSettingManager.getDefaultInterpreterSettingList();
- // add setting with null option & properties expected nullArgumentException.class
- try {
- interpreterSettingManager.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null);
- } catch(NullArgumentException e) {
- assertEquals("Test null option" , e.getMessage(),new NullArgumentException("option").getMessage());
- }
- try {
- interpreterSettingManager.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null);
- } catch (NullArgumentException e){
- assertEquals("Test null properties" , e.getMessage(),new NullArgumentException("properties").getMessage());
- }
- }
-
-
- @Test
- public void testSaveLoad() throws IOException, RepositoryException {
- // interpreter settings
- int numInterpreters = interpreterSettingManager.get().size();
-
- // check if file saved
- assertTrue(new File(conf.getInterpreterSettingPath()).exists());
-
- interpreterSettingManager.createNewSetting("new-mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new HashMap<String, InterpreterProperty>());
- assertEquals(numInterpreters + 1, interpreterSettingManager.get().size());
-
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
-
- /*
- Current situation, if InterpreterSettinfRef doesn't have the key of InterpreterSetting, it would be ignored.
- Thus even though interpreter.json have several interpreterSetting in that file, it would be ignored and would not be initialized from loadFromFile.
- In this case, only "mock11" would be referenced from file under interpreter/mock, and "mock11" group would be initialized.
- */
- // TODO(jl): Decide how to handle the know referenced interpreterSetting.
- assertEquals(1, interpreterSettingManager.get().size());
- }
-
- @Test
- public void testInterpreterSettingPropertyClass() throws IOException, RepositoryException {
- // check if default interpreter reference's property type is map
- Map<String, InterpreterSetting> interpreterSettingRefs = interpreterSettingManager.getAvailableInterpreterSettings();
- InterpreterSetting intpSetting = interpreterSettingRefs.get("mock1");
- Map<String, DefaultInterpreterProperty> intpProperties =
- (Map<String, DefaultInterpreterProperty>) intpSetting.getProperties();
- assertTrue(intpProperties instanceof Map);
-
- // check if interpreter instance is saved as Properties in conf/interpreter.json file
- Map<String, InterpreterProperty> properties = new HashMap<String, InterpreterProperty>();
- properties.put("key1", new InterpreterProperty("key1", "value1", "type1"));
- properties.put("key2", new InterpreterProperty("key2", "value2", "type2"));
-
- interpreterSettingManager.createNewSetting("newMock", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), properties);
-
- String confFilePath = conf.getInterpreterSettingPath();
- byte[] encoded = Files.readAllBytes(Paths.get(confFilePath));
- String json = new String(encoded, "UTF-8");
-
- InterpreterInfoSaving infoSaving = InterpreterInfoSaving.fromJson(json);
- Map<String, InterpreterSetting> interpreterSettings = infoSaving.interpreterSettings;
- for (String key : interpreterSettings.keySet()) {
- InterpreterSetting setting = interpreterSettings.get(key);
- if (setting.getName().equals("newMock")) {
- assertEquals(setting.getProperties().toString(), properties.toString());
- }
- }
- }
-
- @Test
- public void testInterpreterAliases() throws IOException, RepositoryException {
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
- final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null);
- final InterpreterInfo info2 = new InterpreterInfo("className2", "name1", true, null);
- interpreterSettingManager.add("group1", new ArrayList<InterpreterInfo>() {{
- add(info1);
- }}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path1", null);
- interpreterSettingManager.add("group2", new ArrayList<InterpreterInfo>(){{
- add(info2);
- }}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path2", null);
-
- final InterpreterSetting setting1 = interpreterSettingManager.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
- final InterpreterSetting setting2 = interpreterSettingManager.createNewSetting("test-group2", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
-
- interpreterSettingManager.setInterpreters("user", "note", new ArrayList<String>() {{
- add(setting1.getId());
- add(setting2.getId());
- }});
-
- assertEquals("className1", factory.getInterpreter("user1", "note", "test-group1").getClassName());
- assertEquals("className1", factory.getInterpreter("user1", "note", "group1").getClassName());
- }
-
- @Test
- public void testMultiUser() throws IOException, RepositoryException {
- interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
- factory = new InterpreterFactory(conf, null, null, null, depResolver, true, interpreterSettingManager);
- final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null);
- interpreterSettingManager.add("group1", new ArrayList<InterpreterInfo>(){{
- add(info1);
- }}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path1", null);
-
- InterpreterOption perUserInterpreterOption = new InterpreterOption(true, InterpreterOption.ISOLATED, InterpreterOption.SHARED);
- final InterpreterSetting setting1 = interpreterSettingManager.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), perUserInterpreterOption, new HashMap<String, InterpreterProperty>());
-
- interpreterSettingManager.setInterpreters("user1", "note", new ArrayList<String>() {{
- add(setting1.getId());
- }});
-
- interpreterSettingManager.setInterpreters("user2", "note", new ArrayList<String>() {{
- add(setting1.getId());
- }});
-
- assertNotEquals(factory.getInterpreter("user1", "note", "test-group1"), factory.getInterpreter("user2", "note", "test-group1"));
- }
-
-
- @Test
- public void testInvalidInterpreterSettingName() {
- try {
- interpreterSettingManager.createNewSetting("new.mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new HashMap<String, InterpreterProperty>());
- fail("expect fail because of invalid InterpreterSetting Name");
- } catch (IOException e) {
- assertEquals("'.' is invalid for InterpreterSetting name.", e.getMessage());
- }
- }
-
-
- @Test
- public void getEditorSetting() throws IOException, RepositoryException, SchedulerException {
- List<String> intpIds = new ArrayList<>();
- for(InterpreterSetting intpSetting: interpreterSettingManager.get()) {
- if (intpSetting.getName().startsWith("mock1")) {
- intpIds.add(intpSetting.getId());
- }
- }
- Note note = notebook.createNote(intpIds, new AuthenticationInfo("anonymous"));
-
- Interpreter interpreter = factory.getInterpreter("user1", note.getId(), "mock11");
- // get editor setting from interpreter-setting.json
- Map<String, Object> editor = interpreterSettingManager.getEditorSetting(interpreter, "user1", note.getId(), "mock11");
- assertEquals("java", editor.get("language"));
-
- // when interpreter is not loaded via interpreter-setting.json
- // or editor setting doesn't exit
- editor = interpreterSettingManager.getEditorSetting(factory.getInterpreter("user1", note.getId(), "mock1"),"user1", note.getId(), "mock1");
- assertEquals(null, editor.get("language"));
-
- // when interpreter is not bound to note
- editor = interpreterSettingManager.getEditorSetting(factory.getInterpreter("user1", note.getId(), "mock11"),"user1", note.getId(), "mock2");
- assertEquals("text", editor.get("language"));
- }
-
- @Test
- public void registerCustomInterpreterRunner() throws IOException {
- InterpreterSettingManager spyInterpreterSettingManager = spy(interpreterSettingManager);
-
- doNothing().when(spyInterpreterSettingManager).saveToFile();
-
- ArrayList<InterpreterInfo> interpreterInfos1 = new ArrayList<>();
- interpreterInfos1.add(new InterpreterInfo("name1.class", "name1", true, Maps.<String, Object>newHashMap()));
-
- spyInterpreterSettingManager.add("normalGroup1", interpreterInfos1, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, DefaultInterpreterProperty>newHashMap(), "/normalGroup1", null);
-
- spyInterpreterSettingManager.createNewSetting("normalGroup1", "normalGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
-
- ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
- interpreterInfos2.add(new InterpreterInfo("name1.class", "name1", true, Maps.<String, Object>newHashMap()));
-
- InterpreterRunner mockInterpreterRunner = mock(InterpreterRunner.class);
-
- when(mockInterpreterRunner.getPath()).thenReturn("custom-linux-path.sh");
-
- spyInterpreterSettingManager.add("customGroup1", interpreterInfos2, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, DefaultInterpreterProperty>newHashMap(), "/customGroup1", mockInterpreterRunner);
-
- spyInterpreterSettingManager.createNewSetting("customGroup1", "customGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
-
- spyInterpreterSettingManager.setInterpreters("anonymous", "noteCustome", spyInterpreterSettingManager.getDefaultInterpreterSettingList());
-
- factory.getInterpreter("anonymous", "noteCustome", "customGroup1");
-
- verify(mockInterpreterRunner, times(1)).getPath();
- }
-
- @Test
- public void interpreterRunnerTest() {
- InterpreterRunner mockInterpreterRunner = mock(InterpreterRunner.class);
- String testInterpreterRunner = "relativePath.sh";
- when(mockInterpreterRunner.getPath()).thenReturn(testInterpreterRunner); // This test only for Linux
- Interpreter i = factory.createRemoteRepl("path1", "sessionKey", "className", new Properties(), interpreterSettingManager.get().get(0).getId(), "userName", false, mockInterpreterRunner);
- String interpreterRunner = ((RemoteInterpreter) ((LazyOpenInterpreter) i).getInnerInterpreter()).getInterpreterRunner();
- assertNotEquals(interpreterRunner, testInterpreterRunner);
-
- testInterpreterRunner = "/AbsolutePath.sh";
- when(mockInterpreterRunner.getPath()).thenReturn(testInterpreterRunner);
- i = factory.createRemoteRepl("path1", "sessionKey", "className", new Properties(), interpreterSettingManager.get().get(0).getId(), "userName", false, mockInterpreterRunner);
- interpreterRunner = ((RemoteInterpreter) ((LazyOpenInterpreter) i).getInnerInterpreter()).getInterpreterRunner();
- assertEquals(interpreterRunner, testInterpreterRunner);
+ public void testUnknownRepl2() throws IOException {
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds());
+ assertNull(interpreterFactory.getInterpreter("user1", "note1", "unknown_repl"));
}
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java
new file mode 100644
index 0000000..0bcdb6f
--- /dev/null
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.zeppelin.interpreter;
+
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.dep.Dependency;
+import org.apache.zeppelin.display.AngularObjectRegistryListener;
+import org.apache.zeppelin.helium.ApplicationEventListener;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
+import org.junit.Test;
+import org.sonatype.aether.RepositoryException;
+import org.sonatype.aether.repository.RemoteRepository;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+
+
+public class InterpreterSettingManagerTest extends AbstractInterpreterTest {
+
+ @Test
+ public void testInitInterpreterSettingManager() throws IOException, RepositoryException {
+ assertEquals(5, interpreterSettingManager.get().size());
+ InterpreterSetting interpreterSetting = interpreterSettingManager.getByName("test");
+ assertEquals("test", interpreterSetting.getName());
+ assertEquals("test", interpreterSetting.getGroup());
+ assertEquals(2, interpreterSetting.getInterpreterInfos().size());
+ // 3 other builtin properties:
+ // * zeppelin.interpeter.output.limit
+ // * zeppelin.interpreter.localRepo
+ // * zeppelin.interpreter.max.poolsize
+ assertEquals(6, interpreterSetting.getJavaProperties().size());
+ assertEquals("value_1", interpreterSetting.getJavaProperties().getProperty("property_1"));
+ assertEquals("new_value_2", interpreterSetting.getJavaProperties().getProperty("property_2"));
+ assertEquals("value_3", interpreterSetting.getJavaProperties().getProperty("property_3"));
+ assertEquals("shared", interpreterSetting.getOption().perNote);
+ assertEquals("shared", interpreterSetting.getOption().perUser);
+ assertEquals(0, interpreterSetting.getDependencies().size());
+ assertNotNull(interpreterSetting.getAngularObjectRegistryListener());
+ assertNotNull(interpreterSetting.getRemoteInterpreterProcessListener());
+ assertNotNull(interpreterSetting.getAppEventListener());
+ assertNotNull(interpreterSetting.getDependencyResolver());
+ assertNotNull(interpreterSetting.getInterpreterSettingManager());
+ assertEquals("linux_runner", interpreterSetting.getInterpreterRunner().getPath());
+
+ List<RemoteRepository> repositories = interpreterSettingManager.getRepositories();
+ assertEquals(2, repositories.size());
+ assertEquals("central", repositories.get(0).getId());
+
+ // Load it again
+ InterpreterSettingManager interpreterSettingManager2 = new InterpreterSettingManager(conf,
+ mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
+ assertEquals(5, interpreterSettingManager2.get().size());
+ interpreterSetting = interpreterSettingManager2.getByName("test");
+ assertEquals("test", interpreterSetting.getName());
+ assertEquals("test", interpreterSetting.getGroup());
+ assertEquals(2, interpreterSetting.getInterpreterInfos().size());
+ assertEquals(6, interpreterSetting.getJavaProperties().size());
+ assertEquals("value_1", interpreterSetting.getJavaProperties().getProperty("property_1"));
+ assertEquals("new_value_2", interpreterSetting.getJavaProperties().getProperty("property_2"));
+ assertEquals("value_3", interpreterSetting.getJavaProperties().getProperty("property_3"));
+ assertEquals("shared", interpreterSetting.getOption().perNote);
+ assertEquals("shared", interpreterSetting.getOption().perUser);
+ assertEquals("linux_runner", interpreterSetting.getInterpreterRunner().getPath());
+ assertEquals(0, interpreterSetting.getDependencies().size());
+
+ repositories = interpreterSettingManager2.getRepositories();
+ assertEquals(2, repositories.size());
+ assertEquals("central", repositories.get(0).getId());
+
+ }
+
+ @Test
+ public void testCreateUpdateRemoveSetting() throws IOException {
+ // create new interpreter setting
+ InterpreterOption option = new InterpreterOption();
+ option.setPerNote("scoped");
+ option.setPerUser("scoped");
+ Map<String, InterpreterProperty> properties = new HashMap<>();
+ properties.put("property_4", new InterpreterProperty("property_4","value_4"));
+
+ try {
+ interpreterSettingManager.createNewSetting("test2", "test", new ArrayList<Dependency>(), option, properties);
+ fail("Should fail due to interpreter already existed");
+ } catch (IOException e) {
+ assertTrue(e.getMessage().contains("already existed"));
+ }
+
+ interpreterSettingManager.createNewSetting("test3", "test", new ArrayList<Dependency>(), option, properties);
+ assertEquals(6, interpreterSettingManager.get().size());
+ InterpreterSetting interpreterSetting = interpreterSettingManager.getByName("test3");
+ assertEquals("test3", interpreterSetting.getName());
+ assertEquals("test", interpreterSetting.getGroup());
+ // 3 other builtin properties:
+ // * zeppelin.interpeter.output.limit
+ // * zeppelin.interpreter.localRepo
+ // * zeppelin.interpreter.max.poolsize
+ assertEquals(4, interpreterSetting.getJavaProperties().size());
+ assertEquals("value_4", interpreterSetting.getJavaProperties().getProperty("property_4"));
+ assertEquals("scoped", interpreterSetting.getOption().perNote);
+ assertEquals("scoped", interpreterSetting.getOption().perUser);
+ assertEquals(0, interpreterSetting.getDependencies().size());
+ assertNotNull(interpreterSetting.getAngularObjectRegistryListener());
+ assertNotNull(interpreterSetting.getRemoteInterpreterProcessListener());
+ assertNotNull(interpreterSetting.getAppEventListener());
+ assertNotNull(interpreterSetting.getDependencyResolver());
+ assertNotNull(interpreterSetting.getInterpreterSettingManager());
+
+ // load it again, it should be saved in interpreter-setting.json. So we can restore it properly
+ InterpreterSettingManager interpreterSettingManager2 = new InterpreterSettingManager(conf,
+ mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
+ assertEquals(6, interpreterSettingManager2.get().size());
+ interpreterSetting = interpreterSettingManager2.getByName("test3");
+ assertEquals("test3", interpreterSetting.getName());
+ assertEquals("test", interpreterSetting.getGroup());
+ assertEquals(6, interpreterSetting.getJavaProperties().size());
+ assertEquals("value_4", interpreterSetting.getJavaProperties().getProperty("property_4"));
+ assertEquals("scoped", interpreterSetting.getOption().perNote);
+ assertEquals("scoped", interpreterSetting.getOption().perUser);
+ assertEquals(0, interpreterSetting.getDependencies().size());
+
+ // update interpreter setting
+ InterpreterOption newOption = new InterpreterOption();
+ newOption.setPerNote("scoped");
+ newOption.setPerUser("isolated");
+ Map<String, InterpreterProperty> newProperties = new HashMap<>(properties);
+ newProperties.put("property_4", new InterpreterProperty("property_4", "new_value_4"));
+ List<Dependency> newDependencies = new ArrayList<>();
+ newDependencies.add(new Dependency("com.databricks:spark-avro_2.11:3.1.0"));
+ interpreterSettingManager.setPropertyAndRestart(interpreterSetting.getId(), newOption, newProperties, newDependencies);
+ interpreterSetting = interpreterSettingManager.get(interpreterSetting.getId());
+ assertEquals("test3", interpreterSetting.getName());
+ assertEquals("test", interpreterSetting.getGroup());
+ assertEquals(4, interpreterSetting.getJavaProperties().size());
+ assertEquals("new_value_4", interpreterSetting.getJavaProperties().getProperty("property_4"));
+ assertEquals("scoped", interpreterSetting.getOption().perNote);
+ assertEquals("isolated", interpreterSetting.getOption().perUser);
+ assertEquals(1, interpreterSetting.getDependencies().size());
+ assertNotNull(interpreterSetting.getAngularObjectRegistryListener());
+ assertNotNull(interpreterSetting.getRemoteInterpreterProcessListener());
+ assertNotNull(interpreterSetting.getAppEventListener());
+ assertNotNull(interpreterSetting.getDependencyResolver());
+ assertNotNull(interpreterSetting.getInterpreterSettingManager());
+
+ // restart in note page
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds());
+ interpreterSettingManager.setInterpreterBinding("user2", "note2", interpreterSettingManager.getSettingIds());
+ interpreterSettingManager.setInterpreterBinding("user3", "note3", interpreterSettingManager.getSettingIds());
+ // create 3 sessions as it is scoped mode
+ interpreterSetting.getOption().setPerUser("scoped");
+ interpreterSetting.getDefaultInterpreter("user1", "note1");
+ interpreterSetting.getDefaultInterpreter("user2", "note2");
+ interpreterSetting.getDefaultInterpreter("user3", "note3");
+ InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
+ assertEquals(3, interpreterGroup.getSessionNum());
+ // only close user1's session
+ interpreterSettingManager.restart(interpreterSetting.getId(), "note1", "user1");
+ assertEquals(2, interpreterGroup.getSessionNum());
+ // close all the sessions
+ interpreterSettingManager.restart(interpreterSetting.getId(), "note1", "anonymous");
+ assertEquals(0, interpreterGroup.getSessionNum());
+
+ // remove interpreter setting
+ interpreterSettingManager.remove(interpreterSetting.getId());
+ assertEquals(5, interpreterSettingManager.get().size());
+
+ // load it again
+ InterpreterSettingManager interpreterSettingManager3 = new InterpreterSettingManager(new ZeppelinConfiguration(),
+ mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
+ assertEquals(5, interpreterSettingManager3.get().size());
+
+ }
+
+ @Test
+ public void testInterpreterBinding() throws IOException {
+ assertNull(interpreterSettingManager.getInterpreterBinding("note1"));
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds());
+ assertEquals(interpreterSettingManager.getInterpreterSettingIds(), interpreterSettingManager.getInterpreterBinding("note1"));
+ }
+
+ @Test
+ public void testUpdateInterpreterBinding_PerNoteShared() throws IOException {
+ InterpreterSetting defaultInterpreterSetting = interpreterSettingManager.get().get(0);
+ defaultInterpreterSetting.getOption().setPerNote("shared");
+
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds());
+ // create interpreter of the first binded interpreter setting
+ interpreterFactory.getInterpreter("user1", "note1", "");
+ assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
+
+ // choose the first setting
+ List<String> newSettingIds = new ArrayList<>();
+ newSettingIds.add(interpreterSettingManager.getInterpreterSettingIds().get(1));
+
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", newSettingIds);
+ assertEquals(newSettingIds, interpreterSettingManager.getInterpreterBinding("note1"));
+ // InterpreterGroup will still be alive as it is shared
+ assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
+ }
+
+ @Test
+ public void testUpdateInterpreterBinding_PerNoteIsolated() throws IOException {
+ InterpreterSetting defaultInterpreterSetting = interpreterSettingManager.get().get(0);
+ defaultInterpreterSetting.getOption().setPerNote("isolated");
+
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds());
+ // create interpreter of the first binded interpreter setting
+ interpreterFactory.getInterpreter("user1", "note1", "");
+ assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
+
+ // choose the first setting
+ List<String> newSettingIds = new ArrayList<>();
+ newSettingIds.add(interpreterSettingManager.getInterpreterSettingIds().get(1));
+
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", newSettingIds);
+ assertEquals(newSettingIds, interpreterSettingManager.getInterpreterBinding("note1"));
+ // InterpreterGroup will be closed as it is only belong to this note
+ assertEquals(0, defaultInterpreterSetting.getAllInterpreterGroups().size());
+
+ }
+
+ @Test
+ public void testUpdateInterpreterBinding_PerNoteScoped() throws IOException {
+ InterpreterSetting defaultInterpreterSetting = interpreterSettingManager.get().get(0);
+ defaultInterpreterSetting.getOption().setPerNote("scoped");
+
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds());
+ interpreterSettingManager.setInterpreterBinding("user1", "note2", interpreterSettingManager.getInterpreterSettingIds());
+ // create 2 interpreter of the first binded interpreter setting for note1 and note2
+ interpreterFactory.getInterpreter("user1", "note1", "");
+ interpreterFactory.getInterpreter("user1", "note2", "");
+ assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(2, defaultInterpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ // choose the first setting
+ List<String> newSettingIds = new ArrayList<>();
+ newSettingIds.add(interpreterSettingManager.getInterpreterSettingIds().get(1));
+
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", newSettingIds);
+ assertEquals(newSettingIds, interpreterSettingManager.getInterpreterBinding("note1"));
+ // InterpreterGroup will be still alive but session belong to note1 will be closed
+ assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
+ assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
+
+ }
+
+ @Test
+ public void testGetEditor() throws IOException {
+ interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds());
+ Interpreter echoInterpreter = interpreterFactory.getInterpreter("user1", "note1", "test.echo");
+ // get editor setting from interpreter-setting.json
+ Map<String, Object> editor = interpreterSettingManager.getEditorSetting(echoInterpreter, "user1", "note1", "test.echo");
+ assertEquals("java", editor.get("language"));
+
+ // when editor setting doesn't exit, return the default editor
+ Interpreter mock1Interpreter = interpreterFactory.getInterpreter("user1", "note1", "mock1");
+ editor = interpreterSettingManager.getEditorSetting(mock1Interpreter,"user1", "note1", "mock1");
+ assertEquals("text", editor.get("language"));
+
+ }
+}