You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by jo...@apache.org on 2017/02/24 09:00:09 UTC
[1/3] zeppelin git commit: ZEPPELIN-2057 InterpreterSettingManager
for branch-0.7
Repository: zeppelin
Updated Branches:
refs/heads/branch-0.7 ce64df799 -> 176a37f3e
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderViewTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderViewTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderViewTest.java
index 3ed14b6..cdba75d 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderViewTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderViewTest.java
@@ -19,6 +19,7 @@ package org.apache.zeppelin.notebook;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterFactory;
+import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.search.SearchService;
@@ -61,6 +62,9 @@ public class FolderViewTest {
@Mock
InterpreterFactory interpreterFactory;
+ @Mock
+ InterpreterSettingManager interpreterSettingManager;
+
FolderView folderView;
Note note1;
@@ -85,7 +89,7 @@ public class FolderViewTest {
Note abNote2;
private Note createNote() {
- Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
+ Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener);
note.setNoteNameListener(folderView);
return note;
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java
index 320a5b5..2db594d 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java
@@ -29,6 +29,7 @@ import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterSetting;
+import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.mock.MockInterpreter11;
@@ -47,6 +48,7 @@ public class NoteInterpreterLoaderTest {
private File tmpDir;
private ZeppelinConfiguration conf;
private InterpreterFactory factory;
+ private InterpreterSettingManager interpreterSettingManager;
private DependencyResolver depResolver;
@Before
@@ -66,7 +68,8 @@ public class NoteInterpreterLoaderTest {
MockInterpreter2.register("mock2", "group2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2");
depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
- factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false);
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(false));
+ factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
}
@After
@@ -77,7 +80,7 @@ public class NoteInterpreterLoaderTest {
@Test
public void testGetInterpreter() throws IOException {
- factory.setInterpreters("user", "note", factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters("user", "note", interpreterSettingManager.getDefaultInterpreterSettingList());
// when there're no interpreter selection directive
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", null).getClassName());
@@ -96,20 +99,20 @@ public class NoteInterpreterLoaderTest {
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("user", "note", "group1.mock11").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("user", "note", "group2.mock2").getClassName());
- factory.closeNote("user", "note");
+ interpreterSettingManager.closeNote("user", "note");
}
@Test
public void testNoteSession() throws IOException {
- factory.setInterpreters("user", "noteA", factory.getDefaultInterpreterSettingList());
- factory.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
+ interpreterSettingManager.setInterpreters("user", "noteA", interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
- factory.setInterpreters("user", "noteB", factory.getDefaultInterpreterSettingList());
- factory.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
+ interpreterSettingManager.setInterpreters("user", "noteB", interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
// interpreters are not created before accessing it
- assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA"));
- assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB"));
+ assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA"));
+ assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB"));
factory.getInterpreter("user", "noteA", null).open();
factory.getInterpreter("user", "noteB", null).open();
@@ -119,35 +122,35 @@ public class NoteInterpreterLoaderTest {
factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId()));
// interpreters are created after accessing it
- assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA"));
- assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB"));
+ assertNotNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA"));
+ assertNotNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB"));
// invalid close
- factory.closeNote("user", "note");
- assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA"));
- assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB"));
+ interpreterSettingManager.closeNote("user", "note");
+ assertNotNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA"));
+ assertNotNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB"));
// when
- factory.closeNote("user", "noteA");
- factory.closeNote("user", "noteB");
+ interpreterSettingManager.closeNote("user", "noteA");
+ interpreterSettingManager.closeNote("user", "noteB");
// interpreters are destroyed after close
- assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA"));
- assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB"));
+ assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA"));
+ assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB"));
}
@Test
public void testNotePerInterpreterProcess() throws IOException {
- factory.setInterpreters("user", "noteA", factory.getDefaultInterpreterSettingList());
- factory.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
+ interpreterSettingManager.setInterpreters("user", "noteA", interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
- factory.setInterpreters("user", "noteB", factory.getDefaultInterpreterSettingList());
- factory.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
+ interpreterSettingManager.setInterpreters("user", "noteB", interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
// interpreters are not created before accessing it
- assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session"));
- assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session"));
+ assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session"));
+ assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session"));
factory.getInterpreter("user", "noteA", null).open();
factory.getInterpreter("user", "noteB", null).open();
@@ -158,29 +161,29 @@ public class NoteInterpreterLoaderTest {
factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId()));
// interpreters are created after accessing it
- assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session"));
- assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session"));
+ assertNotNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session"));
+ assertNotNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session"));
// when
- factory.closeNote("user", "noteA");
- factory.closeNote("user", "noteB");
+ interpreterSettingManager.closeNote("user", "noteA");
+ interpreterSettingManager.closeNote("user", "noteB");
// interpreters are destroyed after close
- assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session"));
- assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session"));
+ assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session"));
+ assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session"));
}
@Test
public void testNoteInterpreterCloseForAll() throws IOException {
- factory.setInterpreters("user", "FitstNote", factory.getDefaultInterpreterSettingList());
- factory.getInterpreterSettings("FitstNote").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
+ interpreterSettingManager.setInterpreters("user", "FitstNote", interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.getInterpreterSettings("FitstNote").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
- factory.setInterpreters("user", "yourFirstNote", factory.getDefaultInterpreterSettingList());
- factory.getInterpreterSettings("yourFirstNote").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
+ interpreterSettingManager.setInterpreters("user", "yourFirstNote", interpreterSettingManager.getDefaultInterpreterSettingList());
+ interpreterSettingManager.getInterpreterSettings("yourFirstNote").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
// interpreters are not created before accessing it
- assertNull(factory.getInterpreterSettings("FitstNote").get(0).getInterpreterGroup("user", "FitstNote").get("FitstNote"));
- assertNull(factory.getInterpreterSettings("yourFirstNote").get(0).getInterpreterGroup("user", "yourFirstNote").get("yourFirstNote"));
+ assertNull(interpreterSettingManager.getInterpreterSettings("FitstNote").get(0).getInterpreterGroup("user", "FitstNote").get("FitstNote"));
+ assertNull(interpreterSettingManager.getInterpreterSettings("yourFirstNote").get(0).getInterpreterGroup("user", "yourFirstNote").get("yourFirstNote"));
Interpreter firstNoteIntp = factory.getInterpreter("user", "FitstNote", "group1.mock1");
Interpreter yourFirstNoteIntp = factory.getInterpreter("user", "yourFirstNote", "group1.mock1");
@@ -191,7 +194,7 @@ public class NoteInterpreterLoaderTest {
assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
- factory.closeNote("user", "FitstNote");
+ interpreterSettingManager.closeNote("user", "FitstNote");
assertFalse(((LazyOpenInterpreter)firstNoteIntp).isOpen());
assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
@@ -203,13 +206,13 @@ public class NoteInterpreterLoaderTest {
assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
// invalid check
- factory.closeNote("invalid", "Note");
+ interpreterSettingManager.closeNote("invalid", "Note");
assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
// invalid contains value check
- factory.closeNote("u", "Note");
+ interpreterSettingManager.closeNote("u", "Note");
assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java
index 786dd81..9da7783 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java
@@ -20,6 +20,7 @@ package org.apache.zeppelin.notebook;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.search.SearchService;
@@ -60,6 +61,9 @@ public class NoteTest {
@Mock
InterpreterFactory interpreterFactory;
+ @Mock
+ InterpreterSettingManager interpreterSettingManager;
+
private AuthenticationInfo anonymous = new AuthenticationInfo("anonymous");
@Test
@@ -68,7 +72,7 @@ public class NoteTest {
when(interpreter.getScheduler()).thenReturn(scheduler);
String pText = "%spark sc.version";
- Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
+ Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener);
Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
p.setText(pText);
@@ -84,7 +88,7 @@ public class NoteTest {
@Test
public void addParagraphWithEmptyReplNameTest() {
- Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
+ Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener);
Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
assertNull(p.getText());
@@ -94,7 +98,7 @@ public class NoteTest {
public void addParagraphWithLastReplNameTest() {
when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("spark"))).thenReturn(interpreter);
- Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
+ Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener);
Paragraph p1 = note.addParagraph(AuthenticationInfo.ANONYMOUS);
p1.setText("%spark ");
Paragraph p2 = note.addParagraph(AuthenticationInfo.ANONYMOUS);
@@ -106,7 +110,7 @@ public class NoteTest {
public void insertParagraphWithLastReplNameTest() {
when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("spark"))).thenReturn(interpreter);
- Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
+ Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener);
Paragraph p1 = note.addParagraph(AuthenticationInfo.ANONYMOUS);
p1.setText("%spark ");
Paragraph p2 = note.insertParagraph(note.getParagraphs().size(), AuthenticationInfo.ANONYMOUS);
@@ -118,7 +122,7 @@ public class NoteTest {
public void insertParagraphWithInvalidReplNameTest() {
when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("invalid"))).thenReturn(null);
- Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
+ Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener);
Paragraph p1 = note.addParagraph(AuthenticationInfo.ANONYMOUS);
p1.setText("%invalid ");
Paragraph p2 = note.insertParagraph(note.getParagraphs().size(), AuthenticationInfo.ANONYMOUS);
@@ -138,7 +142,7 @@ public class NoteTest {
when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("md"))).thenReturn(interpreter);
when(interpreter.getScheduler()).thenReturn(scheduler);
- Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
+ Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener);
Paragraph p1 = note.addParagraph(AuthenticationInfo.ANONYMOUS);
InterpreterResult result = new InterpreterResult(InterpreterResult.Code.SUCCESS, InterpreterResult.Type.TEXT, "result");
p1.setResult(result);
@@ -154,7 +158,7 @@ public class NoteTest {
@Test
public void getFolderIdTest() {
- Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
+ Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener);
// Ordinary case test
note.setName("this/is/a/folder/noteName");
assertEquals("this/is/a/folder", note.getFolderId());
@@ -170,7 +174,7 @@ public class NoteTest {
@Test
public void getNameWithoutPathTest() {
- Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
+ Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener);
// Notes in the root folder
note.setName("noteOnRootFolder");
assertEquals("noteOnRootFolder", note.getNameWithoutPath());
@@ -185,7 +189,7 @@ public class NoteTest {
@Test
public void isTrashTest() {
- Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
+ Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener);
// Notes in the root folder
note.setName("noteOnRootFolder");
assertFalse(note.isTrash());
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
index 4bc9459..78903a9 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
@@ -64,6 +64,7 @@ public class NotebookTest implements JobListenerFactory{
private Notebook notebook;
private NotebookRepo notebookRepo;
private InterpreterFactory factory;
+ private InterpreterSettingManager interpreterSettingManager;
private DependencyResolver depResolver;
private NotebookAuthorization notebookAuthorization;
private Credentials credentials;
@@ -91,14 +92,15 @@ public class NotebookTest implements JobListenerFactory{
MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2");
depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
- factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false);
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(false));
+ factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath());
- notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, this, search,
+ notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, this, search,
notebookAuthorization, credentials);
}
@@ -110,7 +112,7 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testSelectingReplImplementation() throws IOException {
Note note = notebook.createNote(anonymous);
- factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
// run with default repl
Paragraph p1 = note.addParagraph(AuthenticationInfo.ANONYMOUS);
@@ -236,7 +238,8 @@ public class NotebookTest implements JobListenerFactory{
Notebook notebook2 = new Notebook(
conf, notebookRepo, schedulerFactory,
- new InterpreterFactory(conf, null, null, null, depResolver, false), this, null, null, null);
+ new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager),
+ interpreterSettingManager, null, null, null, null);
assertEquals(1, notebook2.getAllNotes().size());
notebook.removeNote(note.getId(), anonymous);
@@ -292,7 +295,7 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testRunAll() throws IOException {
Note note = notebook.createNote(anonymous);
- factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
// p1
Paragraph p1 = note.addParagraph(AuthenticationInfo.ANONYMOUS);
@@ -331,7 +334,7 @@ public class NotebookTest implements JobListenerFactory{
public void testSchedule() throws InterruptedException, IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
- factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
Map config = new HashMap<>();
@@ -364,7 +367,7 @@ public class NotebookTest implements JobListenerFactory{
public void testSchedulePoolUsage() throws InterruptedException, IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
- factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
Map config = Maps.newHashMap();
p.setConfig(config);
@@ -432,7 +435,7 @@ public class NotebookTest implements JobListenerFactory{
public void testAutoRestartInterpreterAfterSchedule() throws InterruptedException, IOException{
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
- factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
Map config = new HashMap<>();
@@ -485,7 +488,7 @@ public class NotebookTest implements JobListenerFactory{
public void testExportAndImportNote() throws IOException, CloneNotSupportedException,
InterruptedException, InterpreterException, SchedulerException, RepositoryException {
Note note = notebook.createNote(anonymous);
- factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
final Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
String simpleText = "hello world";
@@ -524,7 +527,7 @@ public class NotebookTest implements JobListenerFactory{
public void testCloneNote() throws IOException, CloneNotSupportedException,
InterruptedException, InterpreterException, SchedulerException, RepositoryException {
Note note = notebook.createNote(anonymous);
- factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
final Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
p.setText("hello world");
@@ -558,7 +561,7 @@ public class NotebookTest implements JobListenerFactory{
public void testCloneNoteWithNoName() throws IOException, CloneNotSupportedException,
InterruptedException {
Note note = notebook.createNote(anonymous);
- factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
Note cloneNote = notebook.cloneNote(note.getId(), null, anonymous);
assertEquals(cloneNote.getName(), "Note " + cloneNote.getId());
@@ -570,7 +573,7 @@ public class NotebookTest implements JobListenerFactory{
public void testCloneNoteWithExceptionResult() throws IOException, CloneNotSupportedException,
InterruptedException {
Note note = notebook.createNote(anonymous);
- factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
final Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
p.setText("hello world");
@@ -595,7 +598,7 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testResourceRemovealOnParagraphNoteRemove() throws IOException {
Note note = notebook.createNote(anonymous);
- factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
intpGroup.setResourcePool(new LocalResourcePool(intpGroup.getId()));
}
@@ -624,9 +627,9 @@ public class NotebookTest implements JobListenerFactory{
IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
- factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
- AngularObjectRegistry registry = factory
+ AngularObjectRegistry registry = interpreterSettingManager
.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
@@ -657,9 +660,9 @@ public class NotebookTest implements JobListenerFactory{
IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
- factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
- AngularObjectRegistry registry = factory
+ AngularObjectRegistry registry = interpreterSettingManager
.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
@@ -691,9 +694,9 @@ public class NotebookTest implements JobListenerFactory{
IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
- factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
- AngularObjectRegistry registry = factory
+ AngularObjectRegistry registry = interpreterSettingManager
.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
@@ -703,8 +706,8 @@ public class NotebookTest implements JobListenerFactory{
registry.add("o2", "object2", null, null);
// restart interpreter
- factory.restart(factory.getInterpreterSettings(note.getId()).get(0).getId());
- registry = factory.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
+ interpreterSettingManager.restart(interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getId());
+ registry = interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
// local and global scope object should be removed
@@ -804,7 +807,7 @@ public class NotebookTest implements JobListenerFactory{
public void testAbortParagraphStatusOnInterpreterRestart() throws InterruptedException,
IOException {
Note note = notebook.createNote(anonymous);
- factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
// create three paragraphs
Paragraph p1 = note.addParagraph(anonymous);
@@ -825,7 +828,7 @@ public class NotebookTest implements JobListenerFactory{
assertEquals(Status.PENDING, p3.getStatus());
// restart interpreter
- factory.restart(factory.getInterpreterSettings(note.getId()).get(0).getId());
+ interpreterSettingManager.restart(interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getId());
// make sure three differnt status aborted well.
assertEquals(Status.FINISHED, p1.getStatus());
@@ -844,9 +847,9 @@ public class NotebookTest implements JobListenerFactory{
p1.setAuthenticationInfo(anonymous);
// restart interpreter with per user session enabled
- for (InterpreterSetting setting : factory.getInterpreterSettings(note1.getId())) {
+ for (InterpreterSetting setting : interpreterSettingManager.getInterpreterSettings(note1.getId())) {
setting.getOption().setPerNote(setting.getOption().SCOPED);
- notebook.getInterpreterFactory().restart(setting.getId());
+ notebook.getInterpreterSettingManager().restart(setting.getId());
}
note1.run(p1.getId());
@@ -892,9 +895,9 @@ public class NotebookTest implements JobListenerFactory{
// restart interpreter with per note session enabled
- for (InterpreterSetting setting : notebook.getInterpreterFactory().getInterpreterSettings(note1.getId())) {
+ for (InterpreterSetting setting : notebook.getInterpreterSettingManager().getInterpreterSettings(note1.getId())) {
setting.getOption().setPerNote(InterpreterOption.SCOPED);
- notebook.getInterpreterFactory().restart(setting.getId());
+ notebook.getInterpreterSettingManager().restart(setting.getId());
}
// run per note session enabled
@@ -935,10 +938,10 @@ public class NotebookTest implements JobListenerFactory{
assertEquals(p1.getResult().message().get(0).getData(), p2.getResult().message().get(0).getData());
// restart interpreter with scoped mode enabled
- for (InterpreterSetting setting : notebook.getInterpreterFactory().getInterpreterSettings(note1.getId())) {
+ for (InterpreterSetting setting : notebook.getInterpreterSettingManager().getInterpreterSettings(note1.getId())) {
setting.getOption().setPerNote(InterpreterOption.SCOPED);
- notebook.getInterpreterFactory().restart(setting.getId(), note1.getId());
- notebook.getInterpreterFactory().restart(setting.getId(), note2.getId());
+ notebook.getInterpreterSettingManager().restart(setting.getId(), note1.getId());
+ notebook.getInterpreterSettingManager().restart(setting.getId(), note2.getId());
}
// run per note session enabled
@@ -951,10 +954,10 @@ public class NotebookTest implements JobListenerFactory{
assertNotEquals(p1.getResult().message().get(0).getData(), p2.getResult().message().get(0).getData());
// restart interpreter with isolated mode enabled
- for (InterpreterSetting setting : notebook.getInterpreterFactory().getInterpreterSettings(note1.getId())) {
+ for (InterpreterSetting setting : notebook.getInterpreterSettingManager().getInterpreterSettings(note1.getId())) {
setting.getOption().setPerNote(InterpreterOption.ISOLATED);
- notebook.getInterpreterFactory().restart(setting.getId(), note1.getId());
- notebook.getInterpreterFactory().restart(setting.getId(), note2.getId());
+ notebook.getInterpreterSettingManager().restart(setting.getId(), note1.getId());
+ notebook.getInterpreterSettingManager().restart(setting.getId(), note2.getId());
}
// run per note process enabled
@@ -979,9 +982,9 @@ public class NotebookTest implements JobListenerFactory{
p1.setText("getId");
// restart interpreter with per note session enabled
- for (InterpreterSetting setting : factory.getInterpreterSettings(note1.getId())) {
+ for (InterpreterSetting setting : interpreterSettingManager.getInterpreterSettings(note1.getId())) {
setting.getOption().setPerNote(InterpreterOption.SCOPED);
- notebook.getInterpreterFactory().restart(setting.getId());
+ notebook.getInterpreterSettingManager().restart(setting.getId());
}
note1.run(p1.getId());
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
index f06a11a..69577e9 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
@@ -102,7 +102,7 @@ public class ParagraphTest {
final String scriptBody = "My name is ${name} and I am ${age=20} years old. " +
"My occupation is ${ job = engineer | developer | artists}";
- final Paragraph paragraph = new Paragraph(note, null, null);
+ final Paragraph paragraph = new Paragraph(note, null, null, null);
final String paragraphId = paragraph.getId();
final AngularObject nameAO = AngularObjectBuilder.build("name", "DuyHai DOAN", noteId,
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
index 0da6610..a77b646 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
@@ -37,6 +37,7 @@ import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import org.apache.zeppelin.interpreter.InterpreterResultMessage;
+import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
import org.apache.zeppelin.notebook.*;
@@ -66,6 +67,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
private Notebook notebookSync;
private NotebookRepoSync notebookRepoSync;
private InterpreterFactory factory;
+ private InterpreterSettingManager interpreterSettingManager;
private DependencyResolver depResolver;
private SearchService search;
private NotebookAuthorization notebookAuthorization;
@@ -101,13 +103,14 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2");
depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo");
- factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false);
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
+ factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
search = mock(SearchService.class);
notebookRepoSync = new NotebookRepoSync(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath());
- notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, this, search,
+ notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search,
notebookAuthorization, credentials);
anonymous = new AuthenticationInfo("anonymous");
}
@@ -236,7 +239,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC.getVarName(), "true");
conf = ZeppelinConfiguration.create();
notebookRepoSync = new NotebookRepoSync(conf);
- notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, this, search,
+ notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search,
notebookAuthorization, credentials);
// check that both storage repos are empty
@@ -284,7 +287,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
ZeppelinConfiguration vConf = ZeppelinConfiguration.create();
NotebookRepoSync vRepoSync = new NotebookRepoSync(vConf);
- Notebook vNotebookSync = new Notebook(vConf, vRepoSync, schedulerFactory, factory, this, search,
+ Notebook vNotebookSync = new Notebook(vConf, vRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search,
notebookAuthorization, credentials);
// one git versioned storage initialized
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
index 99fea49..6db85fe 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
@@ -31,6 +31,7 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterOption;
+import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.notebook.JobListenerFactory;
import org.apache.zeppelin.notebook.Note;
@@ -55,6 +56,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
private SchedulerFactory schedulerFactory;
private Notebook notebook;
private NotebookRepo notebookRepo;
+ private InterpreterSettingManager interpreterSettingManager;
private InterpreterFactory factory;
private DependencyResolver depResolver;
private NotebookAuthorization notebookAuthorization;
@@ -84,12 +86,13 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
this.schedulerFactory = new SchedulerFactory();
depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo");
- factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false);
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
+ factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
- notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, this, search,
+ notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, this, search,
notebookAuthorization, null);
}
@@ -118,7 +121,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
public void testSaveNotebook() throws IOException, InterruptedException {
AuthenticationInfo anonymous = new AuthenticationInfo("anonymous");
Note note = notebook.createNote(anonymous);
- factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
Paragraph p1 = note.addParagraph(AuthenticationInfo.ANONYMOUS);
Map<String, Object> config = p1.getConfig();
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java
index d7bf27b..cf5de75 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import org.apache.zeppelin.interpreter.InterpreterFactory;
+import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
@@ -41,6 +42,7 @@ public class LuceneSearchTest {
private static NotebookRepo notebookRepoMock;
private static InterpreterFactory interpreterFactory;
+ private static InterpreterSettingManager interpreterSettingManager;
private SearchService noteSearchService;
private AuthenticationInfo anonymous;
@@ -49,6 +51,7 @@ public class LuceneSearchTest {
public static void beforeStartUp() {
notebookRepoMock = mock(NotebookRepo.class);
interpreterFactory = mock(InterpreterFactory.class);
+ interpreterSettingManager = mock(InterpreterSettingManager.class);
// when(replLoaderMock.getInterpreterSettings())
// .thenReturn(ImmutableList.<InterpreterSetting>of());
@@ -288,7 +291,7 @@ public class LuceneSearchTest {
}
private Note newNote(String name) {
- Note note = new Note(notebookRepoMock, interpreterFactory, null, noteSearchService, null, null);
+ Note note = new Note(notebookRepoMock, interpreterFactory, interpreterSettingManager, null, noteSearchService, null, null);
note.setName(name);
return note;
}
[2/3] zeppelin git commit: ZEPPELIN-2057 InterpreterSettingManager
for branch-0.7
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
new file mode 100644
index 0000000..b141e6b
--- /dev/null
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
@@ -0,0 +1,1125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.internal.StringMap;
+import com.google.gson.reflect.TypeToken;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.lang.reflect.Type;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream.Filter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import java.nio.file.attribute.PosixFilePermission;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.dep.Dependency;
+import org.apache.zeppelin.dep.DependencyResolver;
+import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter;
+import org.apache.zeppelin.scheduler.Job;
+import org.apache.zeppelin.scheduler.Job.Status;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.sonatype.aether.RepositoryException;
+import org.sonatype.aether.repository.Authentication;
+import org.sonatype.aether.repository.Proxy;
+import org.sonatype.aether.repository.RemoteRepository;
+
+/**
+ * TBD
+ */
+public class InterpreterSettingManager {
+
+ private static final Logger logger = LoggerFactory.getLogger(InterpreterSettingManager.class);
+ private static final String SHARED_SESSION = "shared_session";
+ private static final Map<String, Object> DEFAULT_EDITOR = ImmutableMap.of(
+ "language", (Object) "text",
+ "editOnDblClick", false);
+
+ private final ZeppelinConfiguration zeppelinConfiguration;
+ private final Path interpreterDirPath;
+ private final Path interpreterBindingPath;
+
+ /**
+ * This is only references with default settings, name and properties
+ * key: InterpreterSetting.name
+ */
+ private final Map<String, InterpreterSetting> interpreterSettingsRef;
+ /**
+ * This is used by creating and running Interpreters
+ * key: InterpreterSetting.id <- This is becuase backward compatibility
+ */
+ private final Map<String, InterpreterSetting> interpreterSettings;
+ private final Map<String, List<String>> interpreterBindings;
+
+ private final DependencyResolver dependencyResolver;
+ private final List<RemoteRepository> interpreterRepositories;
+
+ private final InterpreterOption defaultOption;
+
+ private final Map<String, URLClassLoader> cleanCl;
+
+ @Deprecated
+ private String[] interpreterClassList;
+ private String[] interpreterGroupOrderList;
+ private InterpreterGroupFactory interpreterGroupFactory;
+
+ private final Gson gson;
+
+ public InterpreterSettingManager(ZeppelinConfiguration zeppelinConfiguration,
+ DependencyResolver dependencyResolver, InterpreterOption interpreterOption)
+ throws IOException, RepositoryException {
+ this.zeppelinConfiguration = zeppelinConfiguration;
+ this.interpreterDirPath = Paths.get(zeppelinConfiguration.getInterpreterDir());
+ logger.debug("InterpreterRootPath: {}", interpreterDirPath);
+ this.interpreterBindingPath = Paths.get(zeppelinConfiguration.getInterpreterSettingPath());
+ logger.debug("InterpreterBindingPath: {}", interpreterBindingPath);
+
+ this.interpreterSettingsRef = Maps.newConcurrentMap();
+ this.interpreterSettings = Maps.newConcurrentMap();
+ this.interpreterBindings = Maps.newConcurrentMap();
+
+ this.dependencyResolver = dependencyResolver;
+ this.interpreterRepositories = dependencyResolver.getRepos();
+
+ this.defaultOption = interpreterOption;
+
+ this.cleanCl = Collections.synchronizedMap(new HashMap<String, URLClassLoader>());
+
+ String replsConf = zeppelinConfiguration.getString(ConfVars.ZEPPELIN_INTERPRETERS);
+ this.interpreterClassList = replsConf.split(",");
+ String groupOrder = zeppelinConfiguration.getString(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER);
+ this.interpreterGroupOrderList = groupOrder.split(",");
+
+ GsonBuilder gsonBuilder = new GsonBuilder();
+ gsonBuilder.setPrettyPrinting();
+ this.gson = gsonBuilder.create();
+
+ init();
+ }
+
+ private void loadFromFile() {
+ if (!Files.exists(interpreterBindingPath)) {
+ // nothing to read
+ return;
+ }
+ InterpreterInfoSaving infoSaving;
+ try (BufferedReader json =
+ Files.newBufferedReader(interpreterBindingPath, StandardCharsets.UTF_8)) {
+ infoSaving = gson.fromJson(json, InterpreterInfoSaving.class);
+
+ for (String k : infoSaving.interpreterSettings.keySet()) {
+ InterpreterSetting setting = infoSaving.interpreterSettings.get(k);
+ List<InterpreterInfo> infos = setting.getInterpreterInfos();
+
+ // Convert json StringMap to Properties
+ StringMap<String> p = (StringMap<String>) setting.getProperties();
+ Properties properties = new Properties();
+ for (String key : p.keySet()) {
+ properties.put(key, p.get(key));
+ }
+ setting.setProperties(properties);
+
+ // Always use separate interpreter process
+ // While we decided to turn this feature on always (without providing
+ // enable/disable option on GUI).
+ // previously created setting should turn this feature on here.
+ setting.getOption().setRemote(true);
+
+ // Update transient information from InterpreterSettingRef
+ InterpreterSetting interpreterSettingObject =
+ interpreterSettingsRef.get(setting.getGroup());
+ if (interpreterSettingObject == null) {
+ logger.warn("can't get InterpreterSetting " +
+ "Information From loaded Interpreter Setting Ref - {} ", setting.getGroup());
+ continue;
+ }
+ String depClassPath = interpreterSettingObject.getPath();
+ setting.setPath(depClassPath);
+
+ for (InterpreterInfo info : infos) {
+ if (info.getEditor() == null) {
+ Map<String, Object> editor = getEditorFromSettingByClassName(interpreterSettingObject,
+ info.getClassName());
+ info.setEditor(editor);
+ }
+ }
+
+ setting.setInterpreterGroupFactory(interpreterGroupFactory);
+
+ loadInterpreterDependencies(setting);
+ interpreterSettings.put(k, setting);
+ }
+
+ interpreterBindings.putAll(infoSaving.interpreterBindings);
+
+ if (infoSaving.interpreterRepositories != null) {
+ for (RemoteRepository repo : infoSaving.interpreterRepositories) {
+ if (!dependencyResolver.getRepos().contains(repo)) {
+ this.interpreterRepositories.add(repo);
+ }
+ }
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public void saveToFile() throws IOException {
+ String jsonString;
+
+ synchronized (interpreterSettings) {
+ InterpreterInfoSaving info = new InterpreterInfoSaving();
+ info.interpreterBindings = interpreterBindings;
+ info.interpreterSettings = interpreterSettings;
+ info.interpreterRepositories = interpreterRepositories;
+
+ jsonString = gson.toJson(info);
+ }
+
+ if (!Files.exists(interpreterBindingPath)) {
+ Files.createFile(interpreterBindingPath);
+
+ Set<PosixFilePermission> permissions = EnumSet.of(OWNER_READ, OWNER_WRITE);
+ Files.setPosixFilePermissions(interpreterBindingPath, permissions);
+ }
+
+ FileOutputStream fos = new FileOutputStream(interpreterBindingPath.toFile(), false);
+ OutputStreamWriter out = new OutputStreamWriter(fos);
+ out.append(jsonString);
+ out.close();
+ fos.close();
+ }
+
+ //TODO(jl): Fix it to remove InterpreterGroupFactory
+ public void setInterpreterGroupFactory(InterpreterGroupFactory interpreterGroupFactory) {
+ for (InterpreterSetting setting : interpreterSettings.values()) {
+ setting.setInterpreterGroupFactory(interpreterGroupFactory);
+ }
+ this.interpreterGroupFactory = interpreterGroupFactory;
+ }
+
+ private void init() throws InterpreterException, IOException, RepositoryException {
+ String interpreterJson = zeppelinConfiguration.getInterpreterJson();
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+
+ if (Files.exists(interpreterDirPath)) {
+ for (Path interpreterDir : Files
+ .newDirectoryStream(interpreterDirPath, new Filter<Path>() {
+ @Override
+ public boolean accept(Path entry) throws IOException {
+ return Files.exists(entry) && Files.isDirectory(entry);
+ }
+ })) {
+ String interpreterDirString = interpreterDir.toString();
+
+ /**
+ * Register interpreter by the following ordering
+ * 1. Register it from path {ZEPPELIN_HOME}/interpreter/{interpreter_name}/
+ * interpreter-setting.json
+ * 2. Register it from interpreter-setting.json in classpath
+ * {ZEPPELIN_HOME}/interpreter/{interpreter_name}
+ * 3. Register it by Interpreter.register
+ */
+ if (!registerInterpreterFromPath(interpreterDirString, interpreterJson)) {
+ if (!registerInterpreterFromResource(cl, interpreterDirString, interpreterJson)) {
+ /*
+ * TODO(jongyoul)
+ * - Remove these codes below because of legacy code
+ * - Support ThreadInterpreter
+ */
+ URLClassLoader ccl = new URLClassLoader(
+ recursiveBuildLibList(interpreterDir.toFile()), cl);
+ for (String className : interpreterClassList) {
+ try {
+ // Load classes
+ Class.forName(className, true, ccl);
+ Set<String> interpreterKeys = Interpreter.registeredInterpreters.keySet();
+ for (String interpreterKey : interpreterKeys) {
+ if (className
+ .equals(Interpreter.registeredInterpreters.get(interpreterKey)
+ .getClassName())) {
+ Interpreter.registeredInterpreters.get(interpreterKey)
+ .setPath(interpreterDirString);
+ logger.info("Interpreter " + interpreterKey + " found. class=" + className);
+ cleanCl.put(interpreterDirString, ccl);
+ }
+ }
+ } catch (Throwable t) {
+ // nothing to do
+ }
+ }
+ }
+ }
+ }
+ }
+
+ for (RegisteredInterpreter registeredInterpreter : Interpreter.registeredInterpreters
+ .values()) {
+ logger
+ .debug("Registered: {} -> {}. Properties: {}", registeredInterpreter.getInterpreterKey(),
+ registeredInterpreter.getClassName(), registeredInterpreter.getProperties());
+ }
+
+ // RegisteredInterpreters -> interpreterSettingRef
+ InterpreterInfo interpreterInfo;
+ for (RegisteredInterpreter r : Interpreter.registeredInterpreters.values()) {
+ interpreterInfo =
+ new InterpreterInfo(r.getClassName(), r.getName(), r.isDefaultInterpreter(),
+ r.getEditor());
+ add(r.getGroup(), interpreterInfo, r.getProperties(), defaultOption, r.getPath(),
+ r.getRunner());
+ }
+
+ for (String settingId : interpreterSettingsRef.keySet()) {
+ InterpreterSetting setting = interpreterSettingsRef.get(settingId);
+ logger.info("InterpreterSettingRef name {}", setting.getName());
+ }
+
+ loadFromFile();
+
+ // if no interpreter settings are loaded, create default set
+ if (0 == interpreterSettings.size()) {
+ Map<String, InterpreterSetting> temp = new HashMap<>();
+ InterpreterSetting interpreterSetting;
+ for (InterpreterSetting setting : interpreterSettingsRef.values()) {
+ interpreterSetting = createFromInterpreterSettingRef(setting);
+ temp.put(setting.getName(), interpreterSetting);
+ }
+
+ for (String group : interpreterGroupOrderList) {
+ if (null != (interpreterSetting = temp.remove(group))) {
+ interpreterSettings.put(interpreterSetting.getId(), interpreterSetting);
+ }
+ }
+
+ for (InterpreterSetting setting : temp.values()) {
+ interpreterSettings.put(setting.getId(), setting);
+ }
+
+ saveToFile();
+ }
+
+ for (String settingId : interpreterSettings.keySet()) {
+ InterpreterSetting setting = interpreterSettings.get(settingId);
+ logger.info("InterpreterSetting group {} : id={}, name={}", setting.getGroup(), settingId,
+ setting.getName());
+ }
+ }
+
+ private boolean registerInterpreterFromResource(ClassLoader cl, String interpreterDir,
+ String interpreterJson) throws IOException, RepositoryException {
+ URL[] urls = recursiveBuildLibList(new File(interpreterDir));
+ ClassLoader tempClassLoader = new URLClassLoader(urls, cl);
+
+ Enumeration<URL> interpreterSettings = tempClassLoader.getResources(interpreterJson);
+ if (!interpreterSettings.hasMoreElements()) {
+ return false;
+ }
+ for (URL url : Collections.list(interpreterSettings)) {
+ try (InputStream inputStream = url.openStream()) {
+ logger.debug("Reading {} from {}", interpreterJson, url);
+ List<RegisteredInterpreter> registeredInterpreterList =
+ getInterpreterListFromJson(inputStream);
+ registerInterpreters(registeredInterpreterList, interpreterDir);
+ }
+ }
+ return true;
+ }
+
+ private boolean registerInterpreterFromPath(String interpreterDir, String interpreterJson)
+ throws IOException, RepositoryException {
+
+ Path interpreterJsonPath = Paths.get(interpreterDir, interpreterJson);
+ if (Files.exists(interpreterJsonPath)) {
+ logger.debug("Reading {}", interpreterJsonPath);
+ List<RegisteredInterpreter> registeredInterpreterList =
+ getInterpreterListFromJson(interpreterJsonPath);
+ registerInterpreters(registeredInterpreterList, interpreterDir);
+ return true;
+ }
+ return false;
+ }
+
+ private List<RegisteredInterpreter> getInterpreterListFromJson(Path filename)
+ throws FileNotFoundException {
+ return getInterpreterListFromJson(new FileInputStream(filename.toFile()));
+ }
+
+ private List<RegisteredInterpreter> getInterpreterListFromJson(InputStream stream) {
+ Type registeredInterpreterListType = new TypeToken<List<RegisteredInterpreter>>() {
+ }.getType();
+ return gson.fromJson(new InputStreamReader(stream), registeredInterpreterListType);
+ }
+
+ private void registerInterpreters(List<RegisteredInterpreter> registeredInterpreters,
+ String absolutePath) throws IOException, RepositoryException {
+
+ for (RegisteredInterpreter registeredInterpreter : registeredInterpreters) {
+ InterpreterInfo interpreterInfo =
+ new InterpreterInfo(registeredInterpreter.getClassName(), registeredInterpreter.getName(),
+ registeredInterpreter.isDefaultInterpreter(), registeredInterpreter.getEditor());
+ // use defaultOption if it is not specified in interpreter-setting.json
+ InterpreterOption option = registeredInterpreter.getOption() == null ? defaultOption :
+ registeredInterpreter.getOption();
+ add(registeredInterpreter.getGroup(), interpreterInfo, registeredInterpreter.getProperties(),
+ option, absolutePath, registeredInterpreter.getRunner());
+ }
+
+ }
+
+ public InterpreterSetting getDefaultInterpreterSetting(List<InterpreterSetting> settings) {
+ if (settings == null || settings.isEmpty()) {
+ return null;
+ }
+ return settings.get(0);
+ }
+
+ public InterpreterSetting getDefaultInterpreterSetting(String noteId) {
+ return getDefaultInterpreterSetting(getInterpreterSettings(noteId));
+ }
+
+ public List<InterpreterSetting> getInterpreterSettings(String noteId) {
+ List<String> interpreterSettingIds = getNoteInterpreterSettingBinding(noteId);
+ LinkedList<InterpreterSetting> settings = new LinkedList<>();
+
+ Iterator<String> iter = interpreterSettingIds.iterator();
+ while (iter.hasNext()) {
+ String id = iter.next();
+ InterpreterSetting setting = get(id);
+ if (setting == null) {
+ // interpreter setting is removed from factory. remove id from here, too
+ iter.remove();
+ } else {
+ settings.add(setting);
+ }
+ }
+ return settings;
+ }
+
+ private List<String> getNoteInterpreterSettingBinding(String noteId) {
+ LinkedList<String> bindings = new LinkedList<>();
+ synchronized (interpreterSettings) {
+ List<String> settingIds = interpreterBindings.get(noteId);
+ if (settingIds != null) {
+ bindings.addAll(settingIds);
+ }
+ }
+ return bindings;
+ }
+
+ private InterpreterSetting createFromInterpreterSettingRef(String name) {
+ Preconditions.checkNotNull(name, "reference name should be not null");
+ InterpreterSetting settingRef = interpreterSettingsRef.get(name);
+ return createFromInterpreterSettingRef(settingRef);
+ }
+
+ private InterpreterSetting createFromInterpreterSettingRef(InterpreterSetting o) {
+ // should return immutable objects
+ List<InterpreterInfo> infos = (null == o.getInterpreterInfos()) ?
+ new ArrayList<InterpreterInfo>() : new ArrayList<>(o.getInterpreterInfos());
+ List<Dependency> deps = (null == o.getDependencies()) ?
+ new ArrayList<Dependency>() : new ArrayList<>(o.getDependencies());
+ Properties props =
+ convertInterpreterProperties((Map<String, InterpreterProperty>) o.getProperties());
+ InterpreterOption option = InterpreterOption.fromInterpreterOption(o.getOption());
+
+ InterpreterSetting setting = new InterpreterSetting(o.getName(), o.getName(),
+ infos, props, deps, option, o.getPath(), o.getInterpreterRunner());
+ setting.setInterpreterGroupFactory(interpreterGroupFactory);
+ return setting;
+ }
+
+ private Properties convertInterpreterProperties(Map<String, InterpreterProperty> p) {
+ Properties properties = new Properties();
+ for (String key : p.keySet()) {
+ properties.put(key, p.get(key).getValue());
+ }
+ return properties;
+ }
+
+ public Map<String, Object> getEditorSetting(Interpreter interpreter, String user, String noteId,
+ String replName) {
+ Map<String, Object> editor = DEFAULT_EDITOR;
+ String group = StringUtils.EMPTY;
+ try {
+ String defaultSettingName = getDefaultInterpreterSetting(noteId).getName();
+ List<InterpreterSetting> intpSettings = getInterpreterSettings(noteId);
+ for (InterpreterSetting intpSetting : intpSettings) {
+ String[] replNameSplit = replName.split("\\.");
+ if (replNameSplit.length == 2) {
+ group = replNameSplit[0];
+ }
+ // when replName is 'name' of interpreter
+ if (defaultSettingName.equals(intpSetting.getName())) {
+ editor = getEditorFromSettingByClassName(intpSetting, interpreter.getClassName());
+ }
+ // when replName is 'alias name' of interpreter or 'group' of interpreter
+ if (replName.equals(intpSetting.getName()) || group.equals(intpSetting.getName())) {
+ editor = getEditorFromSettingByClassName(intpSetting, interpreter.getClassName());
+ break;
+ }
+ }
+ } catch (NullPointerException e) {
+ logger.warn("Couldn't get interpreter editor setting");
+ }
+ return editor;
+ }
+
+ public Map<String, Object> getEditorFromSettingByClassName(InterpreterSetting intpSetting,
+ String className) {
+ List<InterpreterInfo> intpInfos = intpSetting.getInterpreterInfos();
+ for (InterpreterInfo intpInfo : intpInfos) {
+
+ if (className.equals(intpInfo.getClassName())) {
+ if (intpInfo.getEditor() == null) {
+ break;
+ }
+ return intpInfo.getEditor();
+ }
+ }
+ return DEFAULT_EDITOR;
+ }
+
+ private void loadInterpreterDependencies(final InterpreterSetting setting) {
+ setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
+ setting.setErrorReason(null);
+ interpreterSettings.put(setting.getId(), setting);
+ synchronized (interpreterSettings) {
+ final Thread t = new Thread() {
+ public void run() {
+ try {
+ // dependencies to prevent library conflict
+ File localRepoDir = new File(zeppelinConfiguration.getInterpreterLocalRepoPath() + "/" +
+ setting.getId());
+ if (localRepoDir.exists()) {
+ try {
+ FileUtils.cleanDirectory(localRepoDir);
+ } catch (FileNotFoundException e) {
+ logger.info("A file that does not exist cannot be deleted, nothing to worry", e);
+ }
+ }
+
+ // load dependencies
+ List<Dependency> deps = setting.getDependencies();
+ if (deps != null) {
+ for (Dependency d : deps) {
+ File destDir = new File(
+ zeppelinConfiguration.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
+
+ if (d.getExclusions() != null) {
+ dependencyResolver.load(d.getGroupArtifactVersion(), d.getExclusions(),
+ new File(destDir, setting.getId()));
+ } else {
+ dependencyResolver
+ .load(d.getGroupArtifactVersion(), new File(destDir, setting.getId()));
+ }
+ }
+ }
+
+ setting.setStatus(InterpreterSetting.Status.READY);
+ setting.setErrorReason(null);
+ } catch (Exception e) {
+ logger.error(String.format("Error while downloading repos for interpreter group : %s," +
+ " go to interpreter setting page click on edit and save it again to make " +
+ "this interpreter work properly. : %s",
+ setting.getGroup(), e.getLocalizedMessage()), e);
+ setting.setErrorReason(e.getLocalizedMessage());
+ setting.setStatus(InterpreterSetting.Status.ERROR);
+ } finally {
+ interpreterSettings.put(setting.getId(), setting);
+ }
+ }
+ };
+ t.start();
+ }
+ }
+
+ /**
+ * Overwrite dependency jar under local-repo/{interpreterId}
+ * if jar file in original path is changed
+ */
+ private void copyDependenciesFromLocalPath(final InterpreterSetting setting) {
+ setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
+ interpreterSettings.put(setting.getId(), setting);
+ synchronized (interpreterSettings) {
+ final Thread t = new Thread() {
+ public void run() {
+ try {
+ List<Dependency> deps = setting.getDependencies();
+ if (deps != null) {
+ for (Dependency d : deps) {
+ File destDir = new File(
+ zeppelinConfiguration.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
+
+ int numSplits = d.getGroupArtifactVersion().split(":").length;
+ if (!(numSplits >= 3 && numSplits <= 6)) {
+ dependencyResolver.copyLocalDependency(d.getGroupArtifactVersion(),
+ new File(destDir, setting.getId()));
+ }
+ }
+ }
+ setting.setStatus(InterpreterSetting.Status.READY);
+ } catch (Exception e) {
+ logger.error(String.format("Error while copying deps for interpreter group : %s," +
+ " go to interpreter setting page click on edit and save it again to make " +
+ "this interpreter work properly.",
+ setting.getGroup()), e);
+ setting.setErrorReason(e.getLocalizedMessage());
+ setting.setStatus(InterpreterSetting.Status.ERROR);
+ } finally {
+ interpreterSettings.put(setting.getId(), setting);
+ }
+ }
+ };
+ t.start();
+ }
+ }
+
+ /**
+ * Return ordered interpreter setting list.
+ * The list does not contain more than one setting from the same interpreter class.
+ * Order by InterpreterClass (order defined by ZEPPELIN_INTERPRETERS), Interpreter setting name
+ */
+ public List<String> getDefaultInterpreterSettingList() {
+ // this list will contain default interpreter setting list
+ List<String> defaultSettings = new LinkedList<>();
+
+ // to ignore the same interpreter group
+ Map<String, Boolean> interpreterGroupCheck = new HashMap<>();
+
+ List<InterpreterSetting> sortedSettings = get();
+
+ for (InterpreterSetting setting : sortedSettings) {
+ if (defaultSettings.contains(setting.getId())) {
+ continue;
+ }
+
+ if (!interpreterGroupCheck.containsKey(setting.getName())) {
+ defaultSettings.add(setting.getId());
+ interpreterGroupCheck.put(setting.getName(), true);
+ }
+ }
+ return defaultSettings;
+ }
+
+ List<RegisteredInterpreter> getRegisteredInterpreterList() {
+ return new ArrayList<>(Interpreter.registeredInterpreters.values());
+ }
+
+
+ private boolean findDefaultInterpreter(List<InterpreterInfo> infos) {
+ for (InterpreterInfo interpreterInfo : infos) {
+ if (interpreterInfo.isDefaultInterpreter()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public InterpreterSetting createNewSetting(String name, String group,
+ List<Dependency> dependencies, InterpreterOption option, Properties p) throws IOException {
+ if (name.indexOf(".") >= 0) {
+ throw new IOException("'.' is invalid for InterpreterSetting name.");
+ }
+ InterpreterSetting setting = createFromInterpreterSettingRef(group);
+ setting.setName(name);
+ setting.setGroup(group);
+ setting.appendDependencies(dependencies);
+ setting.setInterpreterOption(option);
+ setting.setProperties(p);
+ setting.setInterpreterGroupFactory(interpreterGroupFactory);
+ interpreterSettings.put(setting.getId(), setting);
+ loadInterpreterDependencies(setting);
+ saveToFile();
+ return setting;
+ }
+
+ private InterpreterSetting add(String group, InterpreterInfo interpreterInfo,
+ Map<String, InterpreterProperty> interpreterProperties, InterpreterOption option, String path,
+ InterpreterRunner runner)
+ throws InterpreterException, IOException, RepositoryException {
+ ArrayList<InterpreterInfo> infos = new ArrayList<>();
+ infos.add(interpreterInfo);
+ return add(group, infos, new ArrayList<Dependency>(), option, interpreterProperties, path,
+ runner);
+ }
+
+ /**
+ * @param group InterpreterSetting reference name
+ */
+ public InterpreterSetting add(String group, ArrayList<InterpreterInfo> interpreterInfos,
+ List<Dependency> dependencies, InterpreterOption option,
+ Map<String, InterpreterProperty> interpreterProperties, String path,
+ InterpreterRunner runner) {
+ Preconditions.checkNotNull(group, "name should not be null");
+ Preconditions.checkNotNull(interpreterInfos, "interpreterInfos should not be null");
+ Preconditions.checkNotNull(dependencies, "dependencies should not be null");
+ Preconditions.checkNotNull(option, "option should not be null");
+ Preconditions.checkNotNull(interpreterProperties, "properties should not be null");
+
+ InterpreterSetting interpreterSetting;
+
+ synchronized (interpreterSettingsRef) {
+ if (interpreterSettingsRef.containsKey(group)) {
+ interpreterSetting = interpreterSettingsRef.get(group);
+
+ // Append InterpreterInfo
+ List<InterpreterInfo> infos = interpreterSetting.getInterpreterInfos();
+ boolean hasDefaultInterpreter = findDefaultInterpreter(infos);
+ for (InterpreterInfo interpreterInfo : interpreterInfos) {
+ if (!infos.contains(interpreterInfo)) {
+ if (!hasDefaultInterpreter && interpreterInfo.isDefaultInterpreter()) {
+ hasDefaultInterpreter = true;
+ infos.add(0, interpreterInfo);
+ } else {
+ infos.add(interpreterInfo);
+ }
+ }
+ }
+
+ // Append dependencies
+ List<Dependency> dependencyList = interpreterSetting.getDependencies();
+ for (Dependency dependency : dependencies) {
+ if (!dependencyList.contains(dependency)) {
+ dependencyList.add(dependency);
+ }
+ }
+
+ // Append properties
+ Map<String, InterpreterProperty> properties =
+ (Map<String, InterpreterProperty>) interpreterSetting.getProperties();
+ for (String key : interpreterProperties.keySet()) {
+ if (!properties.containsKey(key)) {
+ properties.put(key, interpreterProperties.get(key));
+ }
+ }
+
+ } else {
+ interpreterSetting =
+ new InterpreterSetting(group, null, interpreterInfos, interpreterProperties,
+ dependencies, option, path, runner);
+ interpreterSettingsRef.put(group, interpreterSetting);
+ }
+ }
+
+ if (dependencies.size() > 0) {
+ loadInterpreterDependencies(interpreterSetting);
+ }
+
+ interpreterSetting.setInterpreterGroupFactory(interpreterGroupFactory);
+ return interpreterSetting;
+ }
+
+ /**
+ * map interpreter ids into noteId
+ *
+ * @param noteId note id
+ * @param ids InterpreterSetting id list
+ */
+ public void setInterpreters(String user, String noteId, List<String> ids) throws IOException {
+ putNoteInterpreterSettingBinding(user, noteId, ids);
+ }
+
+ private void putNoteInterpreterSettingBinding(String user, String noteId,
+ List<String> settingList) throws IOException {
+ List<String> unBindedSettings = new LinkedList<>();
+
+ synchronized (interpreterSettings) {
+ List<String> oldSettings = interpreterBindings.get(noteId);
+ if (oldSettings != null) {
+ for (String oldSettingId : oldSettings) {
+ if (!settingList.contains(oldSettingId)) {
+ unBindedSettings.add(oldSettingId);
+ }
+ }
+ }
+ interpreterBindings.put(noteId, settingList);
+ saveToFile();
+
+ for (String settingId : unBindedSettings) {
+ InterpreterSetting setting = get(settingId);
+ removeInterpretersForNote(setting, user, noteId);
+ }
+ }
+ }
+
+ public void removeInterpretersForNote(InterpreterSetting interpreterSetting, String user,
+ String noteId) {
+ InterpreterOption option = interpreterSetting.getOption();
+ if (option.isProcess()) {
+ interpreterSetting.closeAndRemoveInterpreterGroupByNoteId(noteId);
+ } else if (option.isSession()) {
+ InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId);
+ String key = getInterpreterSessionKey(user, noteId, interpreterSetting);
+ interpreterGroup.close(key);
+ synchronized (interpreterGroup) {
+ interpreterGroup.remove(key);
+ interpreterGroup.notifyAll(); // notify createInterpreterForNote()
+ }
+ logger.info("Interpreter instance {} for note {} is removed", interpreterSetting.getName(),
+ noteId);
+ }
+ }
+
+ public String getInterpreterSessionKey(String user, String noteId, InterpreterSetting setting) {
+ InterpreterOption option = setting.getOption();
+ String key;
+ if (option.isExistingProcess()) {
+ key = Constants.EXISTING_PROCESS;
+ } else if (option.perNoteScoped() && option.perUserScoped()) {
+ key = user + ":" + noteId;
+ } else if (option.perUserScoped()) {
+ key = user;
+ } else if (option.perNoteScoped()) {
+ key = noteId;
+ } else {
+ key = SHARED_SESSION;
+ }
+
+ logger.debug("Interpreter session key: {}, for note: {}, user: {}, InterpreterSetting Name: " +
+ "{}", key, noteId, user, setting.getName());
+ return key;
+ }
+
+
+ public List<String> getInterpreters(String noteId) {
+ return getNoteInterpreterSettingBinding(noteId);
+ }
+
+ public void closeNote(String user, String noteId) {
+ // close interpreters in this note session
+ List<InterpreterSetting> settings = getInterpreterSettings(noteId);
+ if (settings == null || settings.size() == 0) {
+ return;
+ }
+
+ logger.info("closeNote: {}", noteId);
+ for (InterpreterSetting setting : settings) {
+ removeInterpretersForNote(setting, user, noteId);
+ }
+ }
+
+ public Map<String, InterpreterSetting> getAvailableInterpreterSettings() {
+ return interpreterSettingsRef;
+ }
+
+ private URL[] recursiveBuildLibList(File path) throws MalformedURLException {
+ URL[] urls = new URL[0];
+ if (path == null || !path.exists()) {
+ return urls;
+ } else if (path.getName().startsWith(".")) {
+ return urls;
+ } else if (path.isDirectory()) {
+ File[] files = path.listFiles();
+ if (files != null) {
+ for (File f : files) {
+ urls = (URL[]) ArrayUtils.addAll(urls, recursiveBuildLibList(f));
+ }
+ }
+ return urls;
+ } else {
+ return new URL[]{path.toURI().toURL()};
+ }
+ }
+
+ public List<RemoteRepository> getRepositories() {
+ return this.interpreterRepositories;
+ }
+
+ public void addRepository(String id, String url, boolean snapshot, Authentication auth,
+ Proxy proxy) throws IOException {
+ dependencyResolver.addRepo(id, url, snapshot, auth, proxy);
+ saveToFile();
+ }
+
+ public void removeRepository(String id) throws IOException {
+ dependencyResolver.delRepo(id);
+ saveToFile();
+ }
+
+ public void removeNoteInterpreterSettingBinding(String user, String noteId) {
+ synchronized (interpreterSettings) {
+ List<String> settingIds = (interpreterBindings.containsKey(noteId) ?
+ interpreterBindings.remove(noteId) :
+ Collections.<String>emptyList());
+ for (String settingId : settingIds) {
+ this.removeInterpretersForNote(get(settingId), user, noteId);
+ }
+ }
+ }
+
+ /**
+ * Change interpreter property and restart
+ */
+ public void setPropertyAndRestart(String id, InterpreterOption option, Properties properties,
+ List<Dependency> dependencies) throws IOException {
+ synchronized (interpreterSettings) {
+ InterpreterSetting intpSetting = interpreterSettings.get(id);
+ if (intpSetting != null) {
+ try {
+ stopJobAllInterpreter(intpSetting);
+
+ intpSetting.closeAndRemoveAllInterpreterGroups();
+ intpSetting.setOption(option);
+ intpSetting.setProperties(properties);
+ intpSetting.setDependencies(dependencies);
+ loadInterpreterDependencies(intpSetting);
+
+ saveToFile();
+ } catch (Exception e) {
+ throw e;
+ } finally {
+ loadFromFile();
+ }
+ } else {
+ throw new InterpreterException("Interpreter setting id " + id + " not found");
+ }
+ }
+ }
+
+ public void restart(String settingId, String noteId, String user) {
+ InterpreterSetting intpSetting = interpreterSettings.get(settingId);
+ Preconditions.checkNotNull(intpSetting);
+
+ // restart interpreter setting in note page
+ if (noteIdIsExist(noteId) && intpSetting.getOption().isProcess()) {
+ intpSetting.closeAndRemoveInterpreterGroupByNoteId(noteId);
+ return;
+ } else {
+ // restart interpreter setting in interpreter setting page
+ restart(settingId, user);
+ }
+
+ }
+
+ private boolean noteIdIsExist(String noteId) {
+ return noteId == null ? false : true;
+ }
+
+ public void restart(String id, String user) {
+ synchronized (interpreterSettings) {
+ InterpreterSetting intpSetting = interpreterSettings.get(id);
+ // Check if dependency in specified path is changed
+ // If it did, overwrite old dependency jar with new one
+ if (intpSetting != null) {
+ //clean up metaInfos
+ intpSetting.setInfos(null);
+ copyDependenciesFromLocalPath(intpSetting);
+
+ stopJobAllInterpreter(intpSetting);
+ if (user.equals("anonymous")) {
+ intpSetting.closeAndRemoveAllInterpreterGroups();
+ } else {
+ intpSetting.closeAndRemoveInterpreterGroupByUser(user);
+ }
+
+ } else {
+ throw new InterpreterException("Interpreter setting id " + id + " not found");
+ }
+ }
+ }
+
+ public void restart(String id) {
+ restart(id, "anonymous");
+ }
+
+ private void stopJobAllInterpreter(InterpreterSetting intpSetting) {
+ if (intpSetting != null) {
+ for (InterpreterGroup intpGroup : intpSetting.getAllInterpreterGroups()) {
+ for (List<Interpreter> interpreters : intpGroup.values()) {
+ for (Interpreter intp : interpreters) {
+ for (Job job : intp.getScheduler().getJobsRunning()) {
+ job.abort();
+ job.setStatus(Status.ABORT);
+ logger.info("Job " + job.getJobName() + " aborted ");
+ }
+ for (Job job : intp.getScheduler().getJobsWaiting()) {
+ job.abort();
+ job.setStatus(Status.ABORT);
+ logger.info("Job " + job.getJobName() + " aborted ");
+ }
+ }
+ }
+ }
+ }
+ }
+
+ public InterpreterSetting get(String name) {
+ synchronized (interpreterSettings) {
+ return interpreterSettings.get(name);
+ }
+ }
+
+ public void remove(String id) throws IOException {
+ synchronized (interpreterSettings) {
+ if (interpreterSettings.containsKey(id)) {
+ InterpreterSetting intp = interpreterSettings.get(id);
+ intp.closeAndRemoveAllInterpreterGroups();
+
+ interpreterSettings.remove(id);
+ for (List<String> settings : interpreterBindings.values()) {
+ Iterator<String> it = settings.iterator();
+ while (it.hasNext()) {
+ String settingId = it.next();
+ if (settingId.equals(id)) {
+ it.remove();
+ }
+ }
+ }
+ saveToFile();
+ }
+ }
+
+ File localRepoDir = new File(zeppelinConfiguration.getInterpreterLocalRepoPath() + "/" + id);
+ FileUtils.deleteDirectory(localRepoDir);
+ }
+
+ /**
+ * Get interpreter settings
+ */
+ public List<InterpreterSetting> get() {
+ synchronized (interpreterSettings) {
+ List<InterpreterSetting> orderedSettings = new LinkedList<>();
+
+ Map<String, List<InterpreterSetting>> nameInterpreterSettingMap = new HashMap<>();
+ for (InterpreterSetting interpreterSetting : interpreterSettings.values()) {
+ String group = interpreterSetting.getGroup();
+ if (!nameInterpreterSettingMap.containsKey(group)) {
+ nameInterpreterSettingMap.put(group, new ArrayList<InterpreterSetting>());
+ }
+ nameInterpreterSettingMap.get(group).add(interpreterSetting);
+ }
+
+ for (String groupName : interpreterGroupOrderList) {
+ List<InterpreterSetting> interpreterSettingList =
+ nameInterpreterSettingMap.remove(groupName);
+ if (null != interpreterSettingList) {
+ for (InterpreterSetting interpreterSetting : interpreterSettingList) {
+ orderedSettings.add(interpreterSetting);
+ }
+ }
+ }
+
+ List<InterpreterSetting> settings = new ArrayList<>();
+
+ for (List<InterpreterSetting> interpreterSettingList : nameInterpreterSettingMap.values()) {
+ for (InterpreterSetting interpreterSetting : interpreterSettingList) {
+ settings.add(interpreterSetting);
+ }
+ }
+
+ Collections.sort(settings, new Comparator<InterpreterSetting>() {
+ @Override
+ public int compare(InterpreterSetting o1, InterpreterSetting o2) {
+ return o1.getName().compareTo(o2.getName());
+ }
+ });
+
+ orderedSettings.addAll(settings);
+
+ return orderedSettings;
+ }
+ }
+
+ public void close() {
+ List<Thread> closeThreads = new LinkedList<>();
+ synchronized (interpreterSettings) {
+ Collection<InterpreterSetting> intpSettings = interpreterSettings.values();
+ for (final InterpreterSetting intpSetting : intpSettings) {
+ Thread t = new Thread() {
+ public void run() {
+ intpSetting.closeAndRemoveAllInterpreterGroups();
+ }
+ };
+ t.start();
+ closeThreads.add(t);
+ }
+ }
+
+ for (Thread t : closeThreads) {
+ try {
+ t.join();
+ } catch (InterruptedException e) {
+ logger.error("Can't close interpreterGroup", e);
+ }
+ }
+ }
+
+ public void shutdown() {
+ List<Thread> closeThreads = new LinkedList<>();
+ synchronized (interpreterSettings) {
+ Collection<InterpreterSetting> intpSettings = interpreterSettings.values();
+ for (final InterpreterSetting intpSetting : intpSettings) {
+ Thread t = new Thread() {
+ public void run() {
+ intpSetting.shutdownAndRemoveAllInterpreterGroups();
+ }
+ };
+ t.start();
+ closeThreads.add(t);
+ }
+ }
+
+ for (Thread t : closeThreads) {
+ try {
+ t.join();
+ } catch (InterruptedException e) {
+ logger.error("Can't close interpreterGroup", e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
index 73279ff..35f32f3 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
@@ -77,6 +77,7 @@ public class Note implements Serializable, ParagraphJobListener {
private Map<String, List<AngularObject>> angularObjects = new HashMap<>();
private transient InterpreterFactory factory;
+ private transient InterpreterSettingManager interpreterSettingManager;
private transient JobListenerFactory jobListenerFactory;
private transient NotebookRepo repo;
private transient SearchService index;
@@ -101,10 +102,12 @@ public class Note implements Serializable, ParagraphJobListener {
public Note() {
}
- public Note(NotebookRepo repo, InterpreterFactory factory, JobListenerFactory jlFactory,
+ public Note(NotebookRepo repo, InterpreterFactory factory,
+ InterpreterSettingManager interpreterSettingManager, JobListenerFactory jlFactory,
SearchService noteIndex, Credentials credentials, NoteEventListener noteEventListener) {
this.repo = repo;
this.factory = factory;
+ this.interpreterSettingManager = interpreterSettingManager;
this.jobListenerFactory = jlFactory;
this.index = noteIndex;
this.noteEventListener = noteEventListener;
@@ -117,7 +120,7 @@ public class Note implements Serializable, ParagraphJobListener {
}
private String getDefaultInterpreterName() {
- InterpreterSetting setting = factory.getDefaultInterpreterSetting(getId());
+ InterpreterSetting setting = interpreterSettingManager.getDefaultInterpreterSetting(getId());
return null != setting ? setting.getName() : StringUtils.EMPTY;
}
@@ -220,6 +223,15 @@ public class Note implements Serializable, ParagraphJobListener {
}
}
+ void setInterpreterSettingManager(InterpreterSettingManager interpreterSettingManager) {
+ this.interpreterSettingManager = interpreterSettingManager;
+ synchronized (paragraphs) {
+ for (Paragraph p : paragraphs) {
+ p.setInterpreterSettingManager(interpreterSettingManager);
+ }
+ }
+ }
+
public void initializeJobListenerForParagraph(Paragraph paragraph) {
final Note paragraphNote = paragraph.getNote();
if (!paragraphNote.getId().equals(this.getId())) {
@@ -272,7 +284,7 @@ public class Note implements Serializable, ParagraphJobListener {
* Add paragraph last.
*/
public Paragraph addParagraph(AuthenticationInfo authenticationInfo) {
- Paragraph p = new Paragraph(this, this, factory);
+ Paragraph p = new Paragraph(this, this, factory, interpreterSettingManager);
p.setAuthenticationInfo(authenticationInfo);
setParagraphMagic(p, paragraphs.size());
synchronized (paragraphs) {
@@ -292,7 +304,8 @@ public class Note implements Serializable, ParagraphJobListener {
void addCloneParagraph(Paragraph srcParagraph) {
// Keep paragraph original ID
- final Paragraph newParagraph = new Paragraph(srcParagraph.getId(), this, this, factory);
+ final Paragraph newParagraph = new Paragraph(srcParagraph.getId(), this, this, factory,
+ interpreterSettingManager);
Map<String, Object> config = new HashMap<>(srcParagraph.getConfig());
Map<String, Object> param = new HashMap<>(srcParagraph.settings.getParams());
@@ -329,7 +342,7 @@ public class Note implements Serializable, ParagraphJobListener {
* @param index index of paragraphs
*/
public Paragraph insertParagraph(int index, AuthenticationInfo authenticationInfo) {
- Paragraph p = new Paragraph(this, this, factory);
+ Paragraph p = new Paragraph(this, this, factory, interpreterSettingManager);
p.setAuthenticationInfo(authenticationInfo);
setParagraphMagic(p, index);
synchronized (paragraphs) {
@@ -622,7 +635,7 @@ public class Note implements Serializable, ParagraphJobListener {
private void snapshotAngularObjectRegistry(String user) {
angularObjects = new HashMap<>();
- List<InterpreterSetting> settings = factory.getInterpreterSettings(getId());
+ List<InterpreterSetting> settings = interpreterSettingManager.getInterpreterSettings(getId());
if (settings == null || settings.size() == 0) {
return;
}
@@ -637,7 +650,7 @@ public class Note implements Serializable, ParagraphJobListener {
private void removeAllAngularObjectInParagraph(String user, String paragraphId) {
angularObjects = new HashMap<>();
- List<InterpreterSetting> settings = factory.getInterpreterSettings(getId());
+ List<InterpreterSetting> settings = interpreterSettingManager.getInterpreterSettings(getId());
if (settings == null || settings.size() == 0) {
return;
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
index 8b946f2..474f98c 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
@@ -77,6 +77,7 @@ public class Notebook implements NoteEventListener {
private SchedulerFactory schedulerFactory;
private InterpreterFactory replFactory;
+ private InterpreterSettingManager interpreterSettingManager;
/**
* Keep the order.
*/
@@ -102,13 +103,14 @@ public class Notebook implements NoteEventListener {
*/
public Notebook(ZeppelinConfiguration conf, NotebookRepo notebookRepo,
SchedulerFactory schedulerFactory, InterpreterFactory replFactory,
- JobListenerFactory jobListenerFactory, SearchService noteSearchService,
- NotebookAuthorization notebookAuthorization, Credentials credentials)
- throws IOException, SchedulerException {
+ InterpreterSettingManager interpreterSettingManager, JobListenerFactory jobListenerFactory,
+ SearchService noteSearchService, NotebookAuthorization notebookAuthorization,
+ Credentials credentials) throws IOException, SchedulerException {
this.conf = conf;
this.notebookRepo = notebookRepo;
this.schedulerFactory = schedulerFactory;
this.replFactory = replFactory;
+ this.interpreterSettingManager = interpreterSettingManager;
this.jobListenerFactory = jobListenerFactory;
this.noteSearchService = noteSearchService;
this.notebookAuthorization = notebookAuthorization;
@@ -138,7 +140,7 @@ public class Notebook implements NoteEventListener {
Preconditions.checkNotNull(subject, "AuthenticationInfo should not be null");
Note note;
if (conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING)) {
- note = createNote(replFactory.getDefaultInterpreterSettingList(), subject);
+ note = createNote(interpreterSettingManager.getDefaultInterpreterSettingList(), subject);
} else {
note = createNote(null, subject);
}
@@ -154,7 +156,7 @@ public class Notebook implements NoteEventListener {
public Note createNote(List<String> interpreterIds, AuthenticationInfo subject)
throws IOException {
Note note =
- new Note(notebookRepo, replFactory, jobListenerFactory,
+ new Note(notebookRepo, replFactory, interpreterSettingManager, jobListenerFactory,
noteSearchService, credentials, this);
note.setNoteNameListener(folders);
@@ -270,14 +272,15 @@ public class Notebook implements NoteEventListener {
throws IOException {
Note note = getNote(id);
if (note != null) {
- List<InterpreterSetting> currentBindings = replFactory.getInterpreterSettings(id);
+ List<InterpreterSetting> currentBindings =
+ interpreterSettingManager.getInterpreterSettings(id);
for (InterpreterSetting setting : currentBindings) {
if (!interpreterSettingIds.contains(setting.getId())) {
fireUnbindInterpreter(note, setting);
}
}
- replFactory.setInterpreters(user, note.getId(), interpreterSettingIds);
+ interpreterSettingManager.setInterpreters(user, note.getId(), interpreterSettingIds);
// comment out while note.getNoteReplLoader().setInterpreters(...) do the same
// replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds);
}
@@ -286,7 +289,7 @@ public class Notebook implements NoteEventListener {
List<String> getBindedInterpreterSettingsIds(String id) {
Note note = getNote(id);
if (note != null) {
- return getInterpreterFactory().getInterpreters(note.getId());
+ return interpreterSettingManager.getInterpreters(note.getId());
} else {
return new LinkedList<>();
}
@@ -295,7 +298,7 @@ public class Notebook implements NoteEventListener {
public List<InterpreterSetting> getBindedInterpreterSettings(String id) {
Note note = getNote(id);
if (note != null) {
- return replFactory.getInterpreterSettings(note.getId());
+ return interpreterSettingManager.getInterpreterSettings(note.getId());
} else {
return new LinkedList<>();
}
@@ -328,12 +331,12 @@ public class Notebook implements NoteEventListener {
note = notes.remove(id);
folders.removeNote(note);
}
- replFactory.removeNoteInterpreterSettingBinding(subject.getUser(), id);
+ interpreterSettingManager.removeNoteInterpreterSettingBinding(subject.getUser(), id);
noteSearchService.deleteIndexDocs(note);
notebookAuthorization.removeNote(id);
// remove from all interpreter instance's angular object registry
- for (InterpreterSetting settings : replFactory.get()) {
+ for (InterpreterSetting settings : interpreterSettingManager.get()) {
AngularObjectRegistry registry =
settings.getInterpreterGroup(subject.getUser(), id).getAngularObjectRegistry();
if (registry instanceof RemoteAngularObjectRegistry) {
@@ -467,6 +470,7 @@ public class Notebook implements NoteEventListener {
note.setCredentials(this.credentials);
note.setInterpreterFactory(replFactory);
+ note.setInterpreterSettingManager(interpreterSettingManager);
note.setJobListenerFactory(jobListenerFactory);
note.setNotebookRepo(notebookRepo);
@@ -509,7 +513,7 @@ public class Notebook implements NoteEventListener {
for (String name : angularObjectSnapshot.keySet()) {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
- List<InterpreterSetting> settings = replFactory.get();
+ List<InterpreterSetting> settings = interpreterSettingManager.get();
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(subject.getUser(), note.getId());
if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
@@ -753,9 +757,10 @@ public class Notebook implements NoteEventListener {
// set interpreter bind type
String interpreterGroupName = null;
- if (replFactory.getInterpreterSettings(jobNote.getId()) != null
- && replFactory.getInterpreterSettings(jobNote.getId()).size() >= 1) {
- interpreterGroupName = replFactory.getInterpreterSettings(jobNote.getId()).get(0).getName();
+ if (interpreterSettingManager.getInterpreterSettings(jobNote.getId()) != null
+ && interpreterSettingManager.getInterpreterSettings(jobNote.getId()).size() >= 1) {
+ interpreterGroupName =
+ interpreterSettingManager.getInterpreterSettings(jobNote.getId()).get(0).getName();
}
// note json object root information.
@@ -829,9 +834,10 @@ public class Notebook implements NoteEventListener {
// set interpreter bind type
String interpreterGroupName = null;
- if (replFactory.getInterpreterSettings(note.getId()) != null
- && replFactory.getInterpreterSettings(note.getId()).size() >= 1) {
- interpreterGroupName = replFactory.getInterpreterSettings(note.getId()).get(0).getName();
+ if (interpreterSettingManager.getInterpreterSettings(note.getId()) != null
+ && interpreterSettingManager.getInterpreterSettings(note.getId()).size() >= 1) {
+ interpreterGroupName =
+ interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getName();
}
// not update and not running -> pass
@@ -881,9 +887,9 @@ public class Notebook implements NoteEventListener {
logger.error(e.getMessage(), e);
}
if (releaseResource) {
- for (InterpreterSetting setting : notebook.getInterpreterFactory()
+ for (InterpreterSetting setting : notebook.getInterpreterSettingManager()
.getInterpreterSettings(note.getId())) {
- notebook.getInterpreterFactory().restart(setting.getId());
+ notebook.getInterpreterSettingManager().restart(setting.getId());
}
}
}
@@ -948,6 +954,10 @@ public class Notebook implements NoteEventListener {
return replFactory;
}
+ public InterpreterSettingManager getInterpreterSettingManager() {
+ return interpreterSettingManager;
+ }
+
public NotebookAuthorization getNotebookAuthorization() {
return notebookAuthorization;
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
index 28b6ab3..f609ecb 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
@@ -53,6 +53,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
private static Logger logger = LoggerFactory.getLogger(Paragraph.class);
private transient InterpreterFactory factory;
+ private transient InterpreterSettingManager interpreterSettingManager;
private transient Note note;
private transient AuthenticationInfo authenticationInfo;
private transient Map<String, Paragraph> userParagraphMap = Maps.newHashMap(); // personalized
@@ -84,10 +85,11 @@ public class Paragraph extends Job implements Serializable, Cloneable {
}
public Paragraph(String paragraphId, Note note, JobListener listener,
- InterpreterFactory factory) {
+ InterpreterFactory factory, InterpreterSettingManager interpreterSettingManager) {
super(paragraphId, generateId(), listener);
this.note = note;
this.factory = factory;
+ this.interpreterSettingManager = interpreterSettingManager;
title = null;
text = null;
authenticationInfo = null;
@@ -97,10 +99,12 @@ public class Paragraph extends Job implements Serializable, Cloneable {
config = new HashMap<>();
}
- public Paragraph(Note note, JobListener listener, InterpreterFactory factory) {
+ public Paragraph(Note note, JobListener listener, InterpreterFactory factory,
+ InterpreterSettingManager interpreterSettingManager) {
super(generateId(), listener);
this.note = note;
this.factory = factory;
+ this.interpreterSettingManager = interpreterSettingManager;
title = null;
text = null;
authenticationInfo = null;
@@ -249,7 +253,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
public List<InterpreterCompletion> getInterpreterCompletion() {
List<InterpreterCompletion> completion = new LinkedList();
- for (InterpreterSetting intp : factory.getInterpreterSettings(note.getId())) {
+ for (InterpreterSetting intp : interpreterSettingManager.getInterpreterSettings(note.getId())) {
List<InterpreterInfo> intInfo = intp.getInterpreterInfos();
if (intInfo.size() > 1) {
for (InterpreterInfo info : intInfo) {
@@ -292,6 +296,10 @@ public class Paragraph extends Job implements Serializable, Cloneable {
this.factory = factory;
}
+ public void setInterpreterSettingManager(InterpreterSettingManager interpreterSettingManager) {
+ this.interpreterSettingManager = interpreterSettingManager;
+ }
+
public InterpreterResult getResult() {
return (InterpreterResult) getReturn();
}
@@ -416,7 +424,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
}
private boolean noteHasInterpreters() {
- return !factory.getInterpreterSettings(note.getId()).isEmpty();
+ return !interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty();
}
private boolean interpreterHasUser(InterpreterSetting intp) {
@@ -430,7 +438,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
private InterpreterSetting getInterpreterSettingById(String id) {
InterpreterSetting setting = null;
- for (InterpreterSetting i : factory.getInterpreterSettings(note.getId())) {
+ for (InterpreterSetting i : interpreterSettingManager.getInterpreterSettings(note.getId())) {
if (id.startsWith(i.getId())) {
setting = i;
break;
@@ -504,8 +512,9 @@ public class Paragraph extends Job implements Serializable, Cloneable {
AngularObjectRegistry registry = null;
ResourcePool resourcePool = null;
- if (!factory.getInterpreterSettings(note.getId()).isEmpty()) {
- InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0);
+ if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) {
+ InterpreterSetting intpGroup =
+ interpreterSettingManager.getInterpreterSettings(note.getId()).get(0);
registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
}
@@ -532,8 +541,9 @@ public class Paragraph extends Job implements Serializable, Cloneable {
AngularObjectRegistry registry = null;
ResourcePool resourcePool = null;
- if (!factory.getInterpreterSettings(note.getId()).isEmpty()) {
- InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0);
+ if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) {
+ InterpreterSetting intpGroup =
+ interpreterSettingManager.getInterpreterSettings(note.getId()).get(0);
registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
index 2588c4c..3940fc3 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
@@ -49,6 +49,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
private SchedulerFactory schedulerFactory;
private DependencyResolver depResolver;
private InterpreterFactory factory;
+ private InterpreterSettingManager interpreterSettingManager;
private VFSNotebookRepo notebookRepo;
private Notebook notebook;
private HeliumApplicationFactory heliumAppFactory;
@@ -84,8 +85,8 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
heliumAppFactory = new HeliumApplicationFactory();
depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
- factory = new InterpreterFactory(conf,
- new InterpreterOption(true), null, null, heliumAppFactory, depResolver, false);
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
+ factory = new InterpreterFactory(conf, null, null, heliumAppFactory, depResolver, false, interpreterSettingManager);
HashMap<String, String> env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
factory.setEnv(env);
@@ -98,6 +99,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
notebookRepo,
schedulerFactory,
factory,
+ interpreterSettingManager,
this,
search,
notebookAuthorization,
@@ -112,7 +114,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
@After
public void tearDown() throws Exception {
- List<InterpreterSetting> settings = factory.get();
+ List<InterpreterSetting> settings = interpreterSettingManager.get();
for (InterpreterSetting setting : settings) {
for (InterpreterGroup intpGroup : setting.getAllInterpreterGroups()) {
intpGroup.close();
@@ -138,7 +140,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
- factory.setInterpreters("user", note1.getId(),factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters("user", note1.getId(),interpreterSettingManager.getDefaultInterpreterSettingList());
Paragraph p1 = note1.addParagraph(AuthenticationInfo.ANONYMOUS);
@@ -184,7 +186,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
- factory.setInterpreters("user", note1.getId(), factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
Paragraph p1 = note1.addParagraph(AuthenticationInfo.ANONYMOUS);
@@ -224,7 +226,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
- notebook.bindInterpretersToNote("user", note1.getId(), factory.getDefaultInterpreterSettingList());
+ notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
Paragraph p1 = note1.addParagraph(AuthenticationInfo.ANONYMOUS);
@@ -285,7 +287,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
- notebook.bindInterpretersToNote("user", note1.getId(), factory.getDefaultInterpreterSettingList());
+ notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
String mock1IntpSettingId = null;
for (InterpreterSetting setting : notebook.getBindedInterpreterSettings(note1.getId())) {
if (setting.getName().equals("mock1")) {
@@ -312,7 +314,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
Thread.yield();
}
// when restart interpreter
- factory.restart(mock1IntpSettingId);
+ interpreterSettingManager.restart(mock1IntpSettingId);
while (app.getStatus() == ApplicationState.Status.LOADED) {
Thread.yield();
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
index 7522366..a8da7d3 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
@@ -68,6 +68,7 @@ import org.mockito.Mock;
public class InterpreterFactoryTest {
private InterpreterFactory factory;
+ private InterpreterSettingManager interpreterSettingManager;
private File tmpDir;
private ZeppelinConfiguration conf;
private InterpreterContext context;
@@ -102,13 +103,14 @@ public class InterpreterFactoryTest {
conf = new ZeppelinConfiguration();
schedulerFactory = new SchedulerFactory();
depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
- factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false);
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
+ factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
context = new InterpreterContext("note", "id", null, "title", "text", null, null, null, null, null, null, null);
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
- notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, jobListenerFactory, search,
+ notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, jobListenerFactory, search,
notebookAuthorization, null);
}
@@ -119,7 +121,7 @@ public class InterpreterFactoryTest {
@Test
public void testBasic() {
- List<InterpreterSetting> all = factory.get();
+ List<InterpreterSetting> all = interpreterSettingManager.get();
InterpreterSetting mock1Setting = null;
for (InterpreterSetting setting : all) {
if (setting.getName().equals("mock1")) {
@@ -137,17 +139,18 @@ public class InterpreterFactoryTest {
assertNotNull("get Interpreter", interpreterGroup.get("session").get(0));
// try to get unavailable interpreter
- assertNull(factory.get("unknown"));
+ assertNull(interpreterSettingManager.get("unknown"));
// restart interpreter
- factory.restart(mock1Setting.getId());
+ interpreterSettingManager.restart(mock1Setting.getId());
assertNull(mock1Setting.getInterpreterGroup("user", "sharedProcess").get("session"));
}
@Test
public void testRemoteRepl() throws Exception {
- factory = new InterpreterFactory(conf, new InterpreterOption(true), null, null, null, depResolver, false);
- List<InterpreterSetting> all = factory.get();
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
+ factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
+ List<InterpreterSetting> all = interpreterSettingManager.get();
InterpreterSetting mock1Setting = null;
for (InterpreterSetting setting : all) {
if (setting.getName().equals("mock1")) {
@@ -174,8 +177,9 @@ public class InterpreterFactoryTest {
*/
@Test
public void testRestartInterpreterInScopedMode() throws Exception {
- factory = new InterpreterFactory(conf, new InterpreterOption(true), null, null, null, depResolver, false);
- List<InterpreterSetting> all = factory.get();
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
+ factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
+ List<InterpreterSetting> all = interpreterSettingManager.get();
InterpreterSetting mock1Setting = null;
for (InterpreterSetting setting : all) {
if (setting.getName().equals("mock1")) {
@@ -210,8 +214,9 @@ public class InterpreterFactoryTest {
*/
@Test
public void testRestartInterpreterInIsolatedMode() throws Exception {
- factory = new InterpreterFactory(conf, new InterpreterOption(true), null, null, null, depResolver, false);
- List<InterpreterSetting> all = factory.get();
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
+ factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
+ List<InterpreterSetting> all = interpreterSettingManager.get();
InterpreterSetting mock1Setting = null;
for (InterpreterSetting setting : all) {
if (setting.getName().equals("mock1")) {
@@ -243,21 +248,21 @@ public class InterpreterFactoryTest {
@Test
public void testFactoryDefaultList() throws IOException, RepositoryException {
// get default settings
- List<String> all = factory.getDefaultInterpreterSettingList();
- assertTrue(factory.get().size() >= all.size());
+ List<String> all = interpreterSettingManager.getDefaultInterpreterSettingList();
+ assertTrue(interpreterSettingManager.get().size() >= all.size());
}
@Test
public void testExceptions() throws InterpreterException, IOException, RepositoryException {
- List<String> all = factory.getDefaultInterpreterSettingList();
+ List<String> all = interpreterSettingManager.getDefaultInterpreterSettingList();
// add setting with null option & properties expected nullArgumentException.class
try {
- factory.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null);
+ interpreterSettingManager.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null);
} catch(NullArgumentException e) {
assertEquals("Test null option" , e.getMessage(),new NullArgumentException("option").getMessage());
}
try {
- factory.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null);
+ interpreterSettingManager.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null);
} catch (NullArgumentException e){
assertEquals("Test null properties" , e.getMessage(),new NullArgumentException("properties").getMessage());
}
@@ -267,22 +272,23 @@ public class InterpreterFactoryTest {
@Test
public void testSaveLoad() throws IOException, RepositoryException {
// interpreter settings
- int numInterpreters = factory.get().size();
+ int numInterpreters = interpreterSettingManager.get().size();
// check if file saved
assertTrue(new File(conf.getInterpreterSettingPath()).exists());
- factory.createNewSetting("new-mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties());
- assertEquals(numInterpreters + 1, factory.get().size());
+ interpreterSettingManager.createNewSetting("new-mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties());
+ assertEquals(numInterpreters + 1, interpreterSettingManager.get().size());
- InterpreterFactory factory2 = new InterpreterFactory(conf, null, null, null, depResolver, false);
- assertEquals(numInterpreters + 1, factory2.get().size());
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
+
+ assertEquals(numInterpreters + 1, interpreterSettingManager.get().size());
}
@Test
public void testInterpreterSettingPropertyClass() throws IOException, RepositoryException {
// check if default interpreter reference's property type is map
- Map<String, InterpreterSetting> interpreterSettingRefs = factory.getAvailableInterpreterSettings();
+ Map<String, InterpreterSetting> interpreterSettingRefs = interpreterSettingManager.getAvailableInterpreterSettings();
InterpreterSetting intpSetting = interpreterSettingRefs.get("mock1");
Map<String, InterpreterProperty> intpProperties =
(Map<String, InterpreterProperty>) intpSetting.getProperties();
@@ -293,7 +299,7 @@ public class InterpreterFactoryTest {
properties.put("key1", "value1");
properties.put("key2", "value2");
- factory.createNewSetting("newMock", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), properties);
+ interpreterSettingManager.createNewSetting("newMock", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), properties);
String confFilePath = conf.getInterpreterSettingPath();
byte[] encoded = Files.readAllBytes(Paths.get(confFilePath));
@@ -312,20 +318,21 @@ public class InterpreterFactoryTest {
@Test
public void testInterpreterAliases() throws IOException, RepositoryException {
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false);
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
+ factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null);
final InterpreterInfo info2 = new InterpreterInfo("className2", "name1", true, null);
- factory.add("group1", new ArrayList<InterpreterInfo>() {{
+ interpreterSettingManager.add("group1", new ArrayList<InterpreterInfo>() {{
add(info1);
}}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path1", null);
- factory.add("group2", new ArrayList<InterpreterInfo>(){{
+ interpreterSettingManager.add("group2", new ArrayList<InterpreterInfo>(){{
add(info2);
}}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path2", null);
- final InterpreterSetting setting1 = factory.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new Properties());
- final InterpreterSetting setting2 = factory.createNewSetting("test-group2", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new Properties());
+ final InterpreterSetting setting1 = interpreterSettingManager.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new Properties());
+ final InterpreterSetting setting2 = interpreterSettingManager.createNewSetting("test-group2", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new Properties());
- factory.setInterpreters("user", "note", new ArrayList<String>() {{
+ interpreterSettingManager.setInterpreters("user", "note", new ArrayList<String>() {{
add(setting1.getId());
add(setting2.getId());
}});
@@ -336,20 +343,21 @@ public class InterpreterFactoryTest {
@Test
public void testMultiUser() throws IOException, RepositoryException {
- factory = new InterpreterFactory(conf, null, null, null, depResolver, true);
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
+ factory = new InterpreterFactory(conf, null, null, null, depResolver, true, interpreterSettingManager);
final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null);
- factory.add("group1", new ArrayList<InterpreterInfo>(){{
+ interpreterSettingManager.add("group1", new ArrayList<InterpreterInfo>(){{
add(info1);
}}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path1", null);
InterpreterOption perUserInterpreterOption = new InterpreterOption(true, InterpreterOption.ISOLATED, InterpreterOption.SHARED);
- final InterpreterSetting setting1 = factory.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), perUserInterpreterOption, new Properties());
+ final InterpreterSetting setting1 = interpreterSettingManager.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), perUserInterpreterOption, new Properties());
- factory.setInterpreters("user1", "note", new ArrayList<String>() {{
+ interpreterSettingManager.setInterpreters("user1", "note", new ArrayList<String>() {{
add(setting1.getId());
}});
- factory.setInterpreters("user2", "note", new ArrayList<String>() {{
+ interpreterSettingManager.setInterpreters("user2", "note", new ArrayList<String>() {{
add(setting1.getId());
}});
@@ -360,7 +368,7 @@ public class InterpreterFactoryTest {
@Test
public void testInvalidInterpreterSettingName() {
try {
- factory.createNewSetting("new.mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties());
+ interpreterSettingManager.createNewSetting("new.mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties());
fail("expect fail because of invalid InterpreterSetting Name");
} catch (IOException e) {
assertEquals("'.' is invalid for InterpreterSetting name.", e.getMessage());
@@ -371,39 +379,40 @@ public class InterpreterFactoryTest {
@Test
public void getEditorSetting() throws IOException, RepositoryException, SchedulerException {
List<String> intpIds = new ArrayList<>();
- for(InterpreterSetting intpSetting: factory.get()) {
+ for(InterpreterSetting intpSetting: interpreterSettingManager.get()) {
if (intpSetting.getName().startsWith("mock1")) {
intpIds.add(intpSetting.getId());
}
}
Note note = notebook.createNote(intpIds, new AuthenticationInfo("anonymous"));
+ Interpreter interpreter = factory.getInterpreter("user1", note.getId(), "mock11");
// get editor setting from interpreter-setting.json
- Map<String, Object> editor = factory.getEditorSetting("user1", note.getId(), "mock11");
+ Map<String, Object> editor = interpreterSettingManager.getEditorSetting(interpreter, "user1", note.getId(), "mock11");
assertEquals("java", editor.get("language"));
// when interpreter is not loaded via interpreter-setting.json
// or editor setting doesn't exit
- editor = factory.getEditorSetting("user1", note.getId(), "mock1");
+ editor = interpreterSettingManager.getEditorSetting(factory.getInterpreter("user1", note.getId(), "mock1"),"user1", note.getId(), "mock1");
assertEquals(null, editor.get("language"));
// when interpreter is not bound to note
- editor = factory.getEditorSetting("user1", note.getId(), "mock2");
+ editor = interpreterSettingManager.getEditorSetting(factory.getInterpreter("user1", note.getId(), "mock11"),"user1", note.getId(), "mock2");
assertEquals("text", editor.get("language"));
}
@Test
public void registerCustomInterpreterRunner() throws IOException {
- InterpreterFactory spyFactory = spy(factory);
+ InterpreterSettingManager spyInterpreterSettingManager = spy(interpreterSettingManager);
- doNothing().when(spyFactory).saveToFile();
+ doNothing().when(spyInterpreterSettingManager).saveToFile();
ArrayList<InterpreterInfo> interpreterInfos1 = new ArrayList<>();
interpreterInfos1.add(new InterpreterInfo("name1.class", "name1", true, Maps.<String, Object>newHashMap()));
- spyFactory.add("normalGroup1", interpreterInfos1, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, InterpreterProperty>newHashMap(), "/normalGroup1", null);
+ spyInterpreterSettingManager.add("normalGroup1", interpreterInfos1, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, InterpreterProperty>newHashMap(), "/normalGroup1", null);
- spyFactory.createNewSetting("normalGroup1", "normalGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new Properties());
+ spyInterpreterSettingManager.createNewSetting("normalGroup1", "normalGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new Properties());
ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
interpreterInfos2.add(new InterpreterInfo("name1.class", "name1", true, Maps.<String, Object>newHashMap()));
@@ -412,13 +421,13 @@ public class InterpreterFactoryTest {
when(mockInterpreterRunner.getPath()).thenReturn("custom-linux-path.sh");
- spyFactory.add("customGroup1", interpreterInfos2, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, InterpreterProperty>newHashMap(), "/customGroup1", mockInterpreterRunner);
+ spyInterpreterSettingManager.add("customGroup1", interpreterInfos2, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, InterpreterProperty>newHashMap(), "/customGroup1", mockInterpreterRunner);
- spyFactory.createNewSetting("customGroup1", "customGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new Properties());
+ spyInterpreterSettingManager.createNewSetting("customGroup1", "customGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new Properties());
- spyFactory.setInterpreters("anonymous", "noteCustome", spyFactory.getDefaultInterpreterSettingList());
+ spyInterpreterSettingManager.setInterpreters("anonymous", "noteCustome", spyInterpreterSettingManager.getDefaultInterpreterSettingList());
- spyFactory.getInterpreter("anonymous", "noteCustome", "customGroup1");
+ factory.getInterpreter("anonymous", "noteCustome", "customGroup1");
verify(mockInterpreterRunner, times(1)).getPath();
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java
index 8325e8a..27aa633 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java
@@ -19,6 +19,7 @@ package org.apache.zeppelin.notebook;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterFactory;
+import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.search.SearchService;
@@ -59,6 +60,9 @@ public class FolderTest {
@Mock
InterpreterFactory interpreterFactory;
+ @Mock
+ InterpreterSettingManager interpreterSettingManager;
+
Folder folder;
Note note1;
@@ -67,13 +71,13 @@ public class FolderTest {
@Before
public void createFolderAndNotes() {
- note1 = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
+ note1 = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener);
note1.setName("this/is/a/folder/note1");
- note2 = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
+ note2 = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener);
note2.setName("this/is/a/folder/note2");
- note3 = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
+ note3 = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener);
note3.setName("this/is/a/folder/note3");
folder = new Folder("this/is/a/folder");
@@ -114,7 +118,7 @@ public class FolderTest {
@Test
public void addNoteTest() {
- Note note4 = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
+ Note note4 = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener);
note4.setName("this/is/a/folder/note4");
folder.addNote(note4);
[3/3] zeppelin git commit: ZEPPELIN-2057 InterpreterSettingManager
for branch-0.7
Posted by jo...@apache.org.
ZEPPELIN-2057 InterpreterSettingManager for branch-0.7
### What is this PR for?
ZEPPELIN-2057 is already merged into master but it's conflicted when it merges into branch-0.7.
### What type of PR is it?
[Refactoring]
### Todos
* [x] - Fix it to fit branch-0.7
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-2057
### How should this be tested?
N/A
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jongyoul Lee <jo...@gmail.com>
Closes #2057 from jongyoul/ZEPPELIN-2057-branch-0.7 and squashes the following commits:
1ae7daa [Jongyoul Lee] Fixed remote option for test
9413739 [Jongyoul Lee] ZEPPELIN-2057 Extract InterpreterSetting functions from InterpreterFactory
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/176a37f3
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/176a37f3
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/176a37f3
Branch: refs/heads/branch-0.7
Commit: 176a37f3e772204f4dc53eb98c1ce3eaab835a2c
Parents: ce64df7
Author: Jongyoul Lee <jo...@gmail.com>
Authored: Thu Feb 23 03:31:05 2017 +0900
Committer: Jongyoul Lee <jo...@apache.org>
Committed: Fri Feb 24 17:59:59 2017 +0900
----------------------------------------------------------------------
.../zeppelin/rest/InterpreterRestApi.java | 36 +-
.../apache/zeppelin/server/ZeppelinServer.java | 17 +-
.../apache/zeppelin/socket/NotebookServer.java | 26 +-
.../zeppelin/utils/InterpreterBindingUtils.java | 2 +-
.../zeppelin/rest/AbstractTestRestApi.java | 11 +-
.../zeppelin/rest/InterpreterRestApiTest.java | 6 +-
.../zeppelin/rest/ZeppelinSparkClusterTest.java | 4 +-
.../zeppelin/socket/NotebookServerTest.java | 6 +-
.../interpreter/InterpreterFactory.java | 1029 +---------------
.../interpreter/InterpreterSettingManager.java | 1125 ++++++++++++++++++
.../java/org/apache/zeppelin/notebook/Note.java | 27 +-
.../org/apache/zeppelin/notebook/Notebook.java | 50 +-
.../org/apache/zeppelin/notebook/Paragraph.java | 28 +-
.../helium/HeliumApplicationFactoryTest.java | 18 +-
.../interpreter/InterpreterFactoryTest.java | 103 +-
.../apache/zeppelin/notebook/FolderTest.java | 12 +-
.../zeppelin/notebook/FolderViewTest.java | 6 +-
.../notebook/NoteInterpreterLoaderTest.java | 81 +-
.../org/apache/zeppelin/notebook/NoteTest.java | 22 +-
.../apache/zeppelin/notebook/NotebookTest.java | 73 +-
.../apache/zeppelin/notebook/ParagraphTest.java | 2 +-
.../notebook/repo/NotebookRepoSyncTest.java | 11 +-
.../notebook/repo/VFSNotebookRepoTest.java | 9 +-
.../zeppelin/search/LuceneSearchTest.java | 5 +-
24 files changed, 1460 insertions(+), 1249 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
index 06d4752..06da4fc 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
@@ -37,6 +37,7 @@ import javax.ws.rs.core.Response.Status;
import com.google.gson.Gson;
import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.rest.message.RestartInterpreterRequest;
import org.apache.zeppelin.utils.SecurityUtils;
import org.slf4j.Logger;
@@ -46,7 +47,6 @@ import org.sonatype.aether.repository.RemoteRepository;
import org.apache.zeppelin.annotation.ZeppelinApi;
import org.apache.zeppelin.dep.Repository;
import org.apache.zeppelin.interpreter.InterpreterException;
-import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.rest.message.NewInterpreterSettingRequest;
import org.apache.zeppelin.rest.message.UpdateInterpreterSettingRequest;
@@ -60,15 +60,15 @@ import org.apache.zeppelin.server.JsonResponse;
public class InterpreterRestApi {
private static final Logger logger = LoggerFactory.getLogger(InterpreterRestApi.class);
- private InterpreterFactory interpreterFactory;
+ private InterpreterSettingManager interpreterSettingManager;
Gson gson = new Gson();
public InterpreterRestApi() {
}
- public InterpreterRestApi(InterpreterFactory interpreterFactory) {
- this.interpreterFactory = interpreterFactory;
+ public InterpreterRestApi(InterpreterSettingManager interpreterSettingManager) {
+ this.interpreterSettingManager = interpreterSettingManager;
}
/**
@@ -78,7 +78,7 @@ public class InterpreterRestApi {
@Path("setting")
@ZeppelinApi
public Response listSettings() {
- return new JsonResponse<>(Status.OK, "", interpreterFactory.get()).build();
+ return new JsonResponse<>(Status.OK, "", interpreterSettingManager.get()).build();
}
/**
@@ -89,7 +89,7 @@ public class InterpreterRestApi {
@ZeppelinApi
public Response getSetting(@PathParam("settingId") String settingId) {
try {
- InterpreterSetting setting = interpreterFactory.get(settingId);
+ InterpreterSetting setting = interpreterSettingManager.get(settingId);
if (setting == null) {
return new JsonResponse<>(Status.NOT_FOUND).build();
} else {
@@ -119,7 +119,7 @@ public class InterpreterRestApi {
}
Properties p = new Properties();
p.putAll(request.getProperties());
- InterpreterSetting interpreterSetting = interpreterFactory
+ InterpreterSetting interpreterSetting = interpreterSettingManager
.createNewSetting(request.getName(), request.getGroup(), request.getDependencies(),
request.getOption(), p);
logger.info("new setting created with {}", interpreterSetting.getId());
@@ -140,7 +140,7 @@ public class InterpreterRestApi {
try {
UpdateInterpreterSettingRequest request =
gson.fromJson(message, UpdateInterpreterSettingRequest.class);
- interpreterFactory
+ interpreterSettingManager
.setPropertyAndRestart(settingId, request.getOption(), request.getProperties(),
request.getDependencies());
} catch (InterpreterException e) {
@@ -152,7 +152,7 @@ public class InterpreterRestApi {
return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, e.getMessage(),
ExceptionUtils.getStackTrace(e)).build();
}
- InterpreterSetting setting = interpreterFactory.get(settingId);
+ InterpreterSetting setting = interpreterSettingManager.get(settingId);
if (setting == null) {
return new JsonResponse<>(Status.NOT_FOUND, "", settingId).build();
}
@@ -167,7 +167,7 @@ public class InterpreterRestApi {
@ZeppelinApi
public Response removeSetting(@PathParam("settingId") String settingId) throws IOException {
logger.info("Remove interpreterSetting {}", settingId);
- interpreterFactory.remove(settingId);
+ interpreterSettingManager.remove(settingId);
return new JsonResponse(Status.OK).build();
}
@@ -183,14 +183,14 @@ public class InterpreterRestApi {
RestartInterpreterRequest request = gson.fromJson(message, RestartInterpreterRequest.class);
String noteId = request == null ? null : request.getNoteId();
- interpreterFactory.restart(settingId, noteId, SecurityUtils.getPrincipal());
+ interpreterSettingManager.restart(settingId, noteId, SecurityUtils.getPrincipal());
} catch (InterpreterException e) {
logger.error("Exception in InterpreterRestApi while restartSetting ", e);
return new JsonResponse<>(Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e))
.build();
}
- InterpreterSetting setting = interpreterFactory.get(settingId);
+ InterpreterSetting setting = interpreterSettingManager.get(settingId);
if (setting == null) {
return new JsonResponse<>(Status.NOT_FOUND, "", settingId).build();
}
@@ -203,7 +203,7 @@ public class InterpreterRestApi {
@GET
@ZeppelinApi
public Response listInterpreter(String message) {
- Map<String, InterpreterSetting> m = interpreterFactory.getAvailableInterpreterSettings();
+ Map<String, InterpreterSetting> m = interpreterSettingManager.getAvailableInterpreterSettings();
return new JsonResponse<>(Status.OK, "", m).build();
}
@@ -214,7 +214,7 @@ public class InterpreterRestApi {
@Path("repository")
@ZeppelinApi
public Response listRepositories() {
- List<RemoteRepository> interpreterRepositories = interpreterFactory.getRepositories();
+ List<RemoteRepository> interpreterRepositories = interpreterSettingManager.getRepositories();
return new JsonResponse<>(Status.OK, "", interpreterRepositories).build();
}
@@ -229,8 +229,8 @@ public class InterpreterRestApi {
public Response addRepository(String message) {
try {
Repository request = gson.fromJson(message, Repository.class);
- interpreterFactory.addRepository(request.getId(), request.getUrl(), request.isSnapshot(),
- request.getAuthentication(), request.getProxy());
+ interpreterSettingManager.addRepository(request.getId(), request.getUrl(),
+ request.isSnapshot(), request.getAuthentication(), request.getProxy());
logger.info("New repository {} added", request.getId());
} catch (Exception e) {
logger.error("Exception in InterpreterRestApi while adding repository ", e);
@@ -252,7 +252,7 @@ public class InterpreterRestApi {
return new JsonResponse<>(Status.BAD_REQUEST).build();
}
String propValue = null;
- InterpreterSetting interpreterSetting = interpreterFactory.get(settingId);
+ InterpreterSetting interpreterSetting = interpreterSettingManager.get(settingId);
Map<String, String> infos = interpreterSetting.getInfos();
if (infos != null) {
propValue = infos.get(propName);
@@ -276,7 +276,7 @@ public class InterpreterRestApi {
public Response removeRepository(@PathParam("repoId") String repoId) {
logger.info("Remove repository {}", repoId);
try {
- interpreterFactory.removeRepository(repoId);
+ interpreterSettingManager.removeRepository(repoId);
} catch (Exception e) {
logger.error("Exception in InterpreterRestApi while removing repository ", e);
return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, e.getMessage(),
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
index b173d04..4bcb422 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
@@ -37,6 +37,9 @@ import org.apache.zeppelin.helium.Helium;
import org.apache.zeppelin.helium.HeliumApplicationFactory;
import org.apache.zeppelin.helium.HeliumVisualizationFactory;
import org.apache.zeppelin.interpreter.InterpreterFactory;
+import org.apache.zeppelin.interpreter.InterpreterOption;
+import org.apache.zeppelin.interpreter.InterpreterOutput;
+import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.notebook.Notebook;
import org.apache.zeppelin.notebook.NotebookAuthorization;
import org.apache.zeppelin.notebook.repo.NotebookRepoSync;
@@ -84,6 +87,7 @@ public class ZeppelinServer extends Application {
public static NotebookServer notebookWsServer;
public static Helium helium;
+ private final InterpreterSettingManager interpreterSettingManager;
private SchedulerFactory schedulerFactory;
private InterpreterFactory replFactory;
private SearchService noteSearchService;
@@ -132,14 +136,17 @@ public class ZeppelinServer extends Application {
}
this.schedulerFactory = new SchedulerFactory();
+ this.interpreterSettingManager = new InterpreterSettingManager(conf, depResolver,
+ new InterpreterOption(true));
this.replFactory = new InterpreterFactory(conf, notebookWsServer,
- notebookWsServer, heliumApplicationFactory, depResolver, SecurityUtils.isAuthenticated());
+ notebookWsServer, heliumApplicationFactory, depResolver, SecurityUtils.isAuthenticated(),
+ interpreterSettingManager);
this.notebookRepo = new NotebookRepoSync(conf);
this.noteSearchService = new LuceneSearch();
this.notebookAuthorization = NotebookAuthorization.init(conf);
this.credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath());
notebook = new Notebook(conf,
- notebookRepo, schedulerFactory, replFactory, notebookWsServer,
+ notebookRepo, schedulerFactory, replFactory, interpreterSettingManager, notebookWsServer,
noteSearchService, notebookAuthorization, credentials);
// to update notebook from application event from remote process.
@@ -187,7 +194,7 @@ public class ZeppelinServer extends Application {
LOG.info("Shutting down Zeppelin Server ... ");
try {
jettyWebServer.stop();
- notebook.getInterpreterFactory().shutdown();
+ notebook.getInterpreterSettingManager().shutdown();
notebook.close();
Thread.sleep(3000);
} catch (Exception e) {
@@ -210,7 +217,7 @@ public class ZeppelinServer extends Application {
}
jettyWebServer.join();
- ZeppelinServer.notebook.getInterpreterFactory().close();
+ ZeppelinServer.notebook.getInterpreterSettingManager().close();
}
private static Server setupJettyServer(ZeppelinConfiguration conf) {
@@ -367,7 +374,7 @@ public class ZeppelinServer extends Application {
HeliumRestApi heliumApi = new HeliumRestApi(helium, notebook);
singletons.add(heliumApi);
- InterpreterRestApi interpreterApi = new InterpreterRestApi(replFactory);
+ InterpreterRestApi interpreterApi = new InterpreterRestApi(interpreterSettingManager);
singletons.add(interpreterApi);
CredentialRestApi credentialApi = new CredentialRestApi(credentials);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
index 6e58e3d..1729ed7 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
@@ -46,6 +46,7 @@ import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.helium.HeliumPackage;
+import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
@@ -450,7 +451,7 @@ public class NotebookServer extends WebSocketServlet
Notebook notebook = notebook();
List<Note> notes = notebook.getAllNotes();
for (Note note : notes) {
- List<String> ids = notebook.getInterpreterFactory().getInterpreters(note.getId());
+ List<String> ids = notebook.getInterpreterSettingManager().getInterpreters(note.getId());
for (String id : ids) {
if (id.equals(interpreterGroupId)) {
broadcast(note.getId(), m);
@@ -920,7 +921,7 @@ public class NotebookServer extends WebSocketServlet
if (!StringUtils.isEmpty(defaultInterpreterId)) {
List<String> interpreterSettingIds = new LinkedList<>();
interpreterSettingIds.add(defaultInterpreterId);
- for (String interpreterSettingId : notebook.getInterpreterFactory().
+ for (String interpreterSettingId : notebook.getInterpreterSettingManager().
getDefaultInterpreterSettingList()) {
if (!interpreterSettingId.equals(defaultInterpreterId)) {
interpreterSettingIds.add(interpreterSettingId);
@@ -1270,7 +1271,7 @@ public class NotebookServer extends WebSocketServlet
Note note = notebook.getNote(noteId);
if (note != null) {
List<InterpreterSetting> settings =
- notebook.getInterpreterFactory().getInterpreterSettings(note.getId());
+ notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getInterpreterGroup(user, note.getId()) == null) {
continue;
@@ -1312,7 +1313,7 @@ public class NotebookServer extends WebSocketServlet
// interpreter.
for (Note n : notebook.getAllNotes()) {
List<InterpreterSetting> settings =
- notebook.getInterpreterFactory().getInterpreterSettings(note.getId());
+ notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getInterpreterGroup(user, n.getId()) == null) {
continue;
@@ -2087,7 +2088,7 @@ public class NotebookServer extends WebSocketServlet
private void sendAllAngularObjects(Note note, String user, NotebookSocket conn)
throws IOException {
List<InterpreterSetting> settings =
- notebook().getInterpreterFactory().getInterpreterSettings(note.getId());
+ notebook().getInterpreterSettingManager().getInterpreterSettings(note.getId());
if (settings == null || settings.size() == 0) {
return;
}
@@ -2125,7 +2126,7 @@ public class NotebookServer extends WebSocketServlet
}
List<InterpreterSetting> intpSettings =
- notebook.getInterpreterFactory().getInterpreterSettings(note.getId());
+ notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId());
if (intpSettings.isEmpty()) {
continue;
}
@@ -2145,7 +2146,8 @@ public class NotebookServer extends WebSocketServlet
continue;
}
- List<String> settingIds = notebook.getInterpreterFactory().getInterpreters(note.getId());
+ List<String> settingIds =
+ notebook.getInterpreterSettingManager().getInterpreters(note.getId());
for (String id : settingIds) {
if (interpreterGroupId.contains(id)) {
broadcast(note.getId(),
@@ -2164,21 +2166,25 @@ public class NotebookServer extends WebSocketServlet
String user = fromMessage.principal;
Message resp = new Message(OP.EDITOR_SETTING);
resp.put("paragraphId", paragraphId);
- resp.put("editor", notebook().getInterpreterFactory().getEditorSetting(user, noteId, replName));
+ Interpreter interpreter =
+ notebook().getInterpreterFactory().getInterpreter(user, noteId, replName);
+ resp.put("editor", notebook().getInterpreterSettingManager().
+ getEditorSetting(interpreter, user, noteId, replName));
conn.send(serializeMessage(resp));
return;
}
private void getInterpreterSettings(NotebookSocket conn, AuthenticationInfo subject)
throws IOException {
- List<InterpreterSetting> availableSettings = notebook().getInterpreterFactory().get();
+ List<InterpreterSetting> availableSettings = notebook().getInterpreterSettingManager().get();
conn.send(serializeMessage(
new Message(OP.INTERPRETER_SETTINGS).put("interpreterSettings", availableSettings)));
}
@Override
public void onMetaInfosReceived(String settingId, Map<String, String> metaInfos) {
- InterpreterSetting interpreterSetting = notebook().getInterpreterFactory().get(settingId);
+ InterpreterSetting interpreterSetting =
+ notebook().getInterpreterSettingManager().get(settingId);
interpreterSetting.setInfos(metaInfos);
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-server/src/main/java/org/apache/zeppelin/utils/InterpreterBindingUtils.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/utils/InterpreterBindingUtils.java b/zeppelin-server/src/main/java/org/apache/zeppelin/utils/InterpreterBindingUtils.java
index 9333afd..94d97fd 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/utils/InterpreterBindingUtils.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/utils/InterpreterBindingUtils.java
@@ -37,7 +37,7 @@ public class InterpreterBindingUtils {
setting.getInterpreterInfos(), true));
}
- List<InterpreterSetting> availableSettings = notebook.getInterpreterFactory().get();
+ List<InterpreterSetting> availableSettings = notebook.getInterpreterSettingManager().get();
for (InterpreterSetting setting : availableSettings) {
boolean selected = false;
for (InterpreterSetting selectedSetting : selectedSettings) {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
index d1fb501..19e40bc 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
@@ -183,7 +183,8 @@ public abstract class AbstractTestRestApi {
// assume first one is spark
InterpreterSetting sparkIntpSetting = null;
- for(InterpreterSetting intpSetting : ZeppelinServer.notebook.getInterpreterFactory().get()) {
+ for(InterpreterSetting intpSetting :
+ ZeppelinServer.notebook.getInterpreterSettingManager().get()) {
if (intpSetting.getName().equals("spark")) {
sparkIntpSetting = intpSetting;
}
@@ -203,7 +204,7 @@ public abstract class AbstractTestRestApi {
sparkIntpSetting.setProperties(sparkProperties);
pySpark = true;
sparkR = true;
- ZeppelinServer.notebook.getInterpreterFactory().restart(sparkIntpSetting.getId());
+ ZeppelinServer.notebook.getInterpreterSettingManager().restart(sparkIntpSetting.getId());
} else {
String sparkHome = getSparkHome();
if (sparkHome != null) {
@@ -220,7 +221,7 @@ public abstract class AbstractTestRestApi {
sparkR = true;
}
- ZeppelinServer.notebook.getInterpreterFactory().restart(sparkIntpSetting.getId());
+ ZeppelinServer.notebook.getInterpreterSettingManager().restart(sparkIntpSetting.getId());
}
}
}
@@ -287,10 +288,10 @@ public abstract class AbstractTestRestApi {
protected static void shutDown() throws Exception {
if (!wasRunning) {
// restart interpreter to stop all interpreter processes
- List<String> settingList = ZeppelinServer.notebook.getInterpreterFactory()
+ List<String> settingList = ZeppelinServer.notebook.getInterpreterSettingManager()
.getDefaultInterpreterSettingList();
for (String setting : settingList) {
- ZeppelinServer.notebook.getInterpreterFactory().restart(setting);
+ ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting);
}
if (shiroIni != null) {
FileUtils.deleteQuietly(shiroIni);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
index ac0d106..8be4574 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
@@ -80,7 +80,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
// then
assertThat(get, isAllowed());
- assertEquals(ZeppelinServer.notebook.getInterpreterFactory().getAvailableInterpreterSettings().size(),
+ assertEquals(ZeppelinServer.notebook.getInterpreterSettingManager().getAvailableInterpreterSettings().size(),
body.entrySet().size());
get.releaseConnection();
}
@@ -258,7 +258,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
assertEquals(p.getResult().message().get(0).getData(), getSimulatedMarkdownResult("markdown"));
// when: restart interpreter
- for (InterpreterSetting setting : ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
+ for (InterpreterSetting setting : ZeppelinServer.notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId())) {
if (setting.getName().equals("md")) {
// call restart interpreter API
PutMethod put = httpPut("/interpreter/setting/restart/" + setting.getId(), "");
@@ -304,7 +304,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
// when: get md interpreter
InterpreterSetting mdIntpSetting = null;
- for (InterpreterSetting setting : ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
+ for (InterpreterSetting setting : ZeppelinServer.notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId())) {
if (setting.getName().equals("md")) {
mdIntpSetting = setting;
break;
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
index 4816e4f..1602b64 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
@@ -161,7 +161,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
for (InterpreterSetting setting : settings) {
if (setting.getName().equals("spark")) {
- ZeppelinServer.notebook.getInterpreterFactory().restart(setting.getId());
+ ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting.getId());
break;
}
}
@@ -393,7 +393,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
for (InterpreterSetting setting : settings) {
if (setting.getName().equals("spark")) {
- ZeppelinServer.notebook.getInterpreterFactory().restart(setting.getId());
+ ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting.getId());
break;
}
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
index 9e64e40..c339140 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
@@ -101,7 +101,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
// get reference to interpreterGroup
InterpreterGroup interpreterGroup = null;
- List<InterpreterSetting> settings = notebook.getInterpreterFactory().getInterpreterSettings(note1.getId());
+ List<InterpreterSetting> settings = notebook.getInterpreterSettingManager().getInterpreterSettings(note1.getId());
for (InterpreterSetting setting : settings) {
if (setting.getName().equals("md")) {
interpreterGroup = setting.getInterpreterGroup("anonymous", "sharedProcess");
@@ -374,7 +374,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
String noteName = "Note with millis " + System.currentTimeMillis();
String defaultInterpreterId = "";
- List<InterpreterSetting> settings = notebook.getInterpreterFactory().get();
+ List<InterpreterSetting> settings = notebook.getInterpreterSettingManager().get();
if (settings.size() > 1) {
defaultInterpreterId = settings.get(1).getId();
}
@@ -396,7 +396,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
}
if (settings.size() > 1) {
- assertEquals(notebook.getInterpreterFactory().getDefaultInterpreterSetting(
+ assertEquals(notebook.getInterpreterSettingManager().getDefaultInterpreterSetting(
createdNote.getId()).getId(), defaultInterpreterId);
}
notebook.removeNote(createdNote.getId(), anonymous);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
index e8b6868..ec3cad3 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
@@ -48,6 +48,8 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import java.nio.file.attribute.PosixFilePermission;
+import java.util.Enumeration;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
@@ -85,625 +87,49 @@ import org.apache.zeppelin.scheduler.Job.Status;
*/
public class InterpreterFactory implements InterpreterGroupFactory {
private static final Logger logger = LoggerFactory.getLogger(InterpreterFactory.class);
- private static final String SHARED_SESSION = "shared_session";
private Map<String, URLClassLoader> cleanCl =
Collections.synchronizedMap(new HashMap<String, URLClassLoader>());
private ZeppelinConfiguration conf;
- @Deprecated
- private String[] interpreterClassList;
- private String[] interpreterGroupOrderList;
- /**
- * This is only references with default settings, name and properties
- * key: InterpreterSetting.name
- */
- private final Map<String, InterpreterSetting> interpreterSettingsRef = new HashMap<>();
-
- /**
- * This is used by creating and running Interpreters
- * key: InterpreterSetting.id <- This is becuase backward compatibility
- */
- private final Map<String, InterpreterSetting> interpreterSettings = new HashMap<>();
-
- private Map<String, List<String>> interpreterBindings = new HashMap<>();
- private List<RemoteRepository> interpreterRepositories;
+ private final InterpreterSettingManager interpreterSettingManager;
private Gson gson;
- private InterpreterOption defaultOption;
-
private AngularObjectRegistryListener angularObjectRegistryListener;
private final RemoteInterpreterProcessListener remoteInterpreterProcessListener;
private final ApplicationEventListener appEventListener;
- private DependencyResolver depResolver;
-
private boolean shiroEnabled;
private Map<String, String> env = new HashMap<>();
private Interpreter devInterpreter;
- private static final Map<String, Object> DEFAULT_EDITOR = ImmutableMap.of(
- "language", (Object) "text",
- "editOnDblClick", false);
-
public InterpreterFactory(ZeppelinConfiguration conf,
AngularObjectRegistryListener angularObjectRegistryListener,
RemoteInterpreterProcessListener remoteInterpreterProcessListener,
ApplicationEventListener appEventListener, DependencyResolver depResolver,
- boolean shiroEnabled) throws InterpreterException, IOException, RepositoryException {
- this(conf, new InterpreterOption(true), angularObjectRegistryListener,
- remoteInterpreterProcessListener, appEventListener, depResolver, shiroEnabled);
- }
-
-
- public InterpreterFactory(ZeppelinConfiguration conf, InterpreterOption defaultOption,
- AngularObjectRegistryListener angularObjectRegistryListener,
- RemoteInterpreterProcessListener remoteInterpreterProcessListener,
- ApplicationEventListener appEventListener, DependencyResolver depResolver,
- boolean shiroEnabled) throws InterpreterException, IOException, RepositoryException {
+ boolean shiroEnabled, InterpreterSettingManager interpreterSettingManager)
+ throws InterpreterException, IOException, RepositoryException {
this.conf = conf;
- this.defaultOption = defaultOption;
this.angularObjectRegistryListener = angularObjectRegistryListener;
- this.depResolver = depResolver;
- this.interpreterRepositories = depResolver.getRepos();
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
this.appEventListener = appEventListener;
this.shiroEnabled = shiroEnabled;
- String replsConf = conf.getString(ConfVars.ZEPPELIN_INTERPRETERS);
- interpreterClassList = replsConf.split(",");
- String groupOrder = conf.getString(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER);
- interpreterGroupOrderList = groupOrder.split(",");
GsonBuilder builder = new GsonBuilder();
builder.setPrettyPrinting();
gson = builder.create();
- init();
+ this.interpreterSettingManager = interpreterSettingManager;
+ //TODO(jl): Fix it not to use InterpreterGroupFactory
+ interpreterSettingManager.setInterpreterGroupFactory(this);
logger.info("shiroEnabled: {}", shiroEnabled);
}
- private void init() throws InterpreterException, IOException, RepositoryException {
- String interpreterJson = conf.getInterpreterJson();
- ClassLoader cl = Thread.currentThread().getContextClassLoader();
-
- Path interpretersDir = Paths.get(conf.getInterpreterDir());
- if (Files.exists(interpretersDir)) {
- for (Path interpreterDir : Files
- .newDirectoryStream(interpretersDir, new DirectoryStream.Filter<Path>() {
- @Override
- public boolean accept(Path entry) throws IOException {
- return Files.exists(entry) && Files.isDirectory(entry);
- }
- })) {
- String interpreterDirString = interpreterDir.toString();
-
- /**
- * Register interpreter by the following ordering
- * 1. Register it from path {ZEPPELIN_HOME}/interpreter/{interpreter_name}/
- * interpreter-setting.json
- * 2. Register it from interpreter-setting.json in classpath
- * {ZEPPELIN_HOME}/interpreter/{interpreter_name}
- * 3. Register it by Interpreter.register
- */
- if (!registerInterpreterFromPath(interpreterDirString, interpreterJson)) {
- if (!registerInterpreterFromResource(cl, interpreterDirString, interpreterJson)) {
- /*
- * TODO(jongyoul)
- * - Remove these codes below because of legacy code
- * - Support ThreadInterpreter
- */
- URLClassLoader ccl = new URLClassLoader(
- recursiveBuildLibList(interpreterDir.toFile()), cl);
- for (String className : interpreterClassList) {
- try {
- // Load classes
- Class.forName(className, true, ccl);
- Set<String> interpreterKeys = Interpreter.registeredInterpreters.keySet();
- for (String interpreterKey : interpreterKeys) {
- if (className
- .equals(Interpreter.registeredInterpreters.get(interpreterKey)
- .getClassName())) {
- Interpreter.registeredInterpreters.get(interpreterKey)
- .setPath(interpreterDirString);
- logger.info("Interpreter " + interpreterKey + " found. class=" + className);
- cleanCl.put(interpreterDirString, ccl);
- }
- }
- } catch (Throwable t) {
- // nothing to do
- }
- }
- }
- }
- }
- }
-
- for (RegisteredInterpreter registeredInterpreter : Interpreter.registeredInterpreters
- .values()) {
- logger
- .debug("Registered: {} -> {}. Properties: {}", registeredInterpreter.getInterpreterKey(),
- registeredInterpreter.getClassName(), registeredInterpreter.getProperties());
- }
-
- // RegisteredInterpreters -> interpreterSettingRef
- InterpreterInfo interpreterInfo;
- for (RegisteredInterpreter r : Interpreter.registeredInterpreters.values()) {
- interpreterInfo =
- new InterpreterInfo(r.getClassName(), r.getName(), r.isDefaultInterpreter(),
- r.getEditor());
- add(r.getGroup(), interpreterInfo, r.getProperties(), defaultOption, r.getPath(),
- r.getRunner());
- }
-
- for (String settingId : interpreterSettingsRef.keySet()) {
- InterpreterSetting setting = interpreterSettingsRef.get(settingId);
- logger.info("InterpreterSettingRef name {}", setting.getName());
- }
-
- loadFromFile();
-
- // if no interpreter settings are loaded, create default set
- if (0 == interpreterSettings.size()) {
- Map<String, InterpreterSetting> temp = new HashMap<>();
- InterpreterSetting interpreterSetting;
- for (InterpreterSetting setting : interpreterSettingsRef.values()) {
- interpreterSetting = createFromInterpreterSettingRef(setting);
- temp.put(setting.getName(), interpreterSetting);
- }
-
- for (String group : interpreterGroupOrderList) {
- if (null != (interpreterSetting = temp.remove(group))) {
- interpreterSettings.put(interpreterSetting.getId(), interpreterSetting);
- }
- }
-
- for (InterpreterSetting setting : temp.values()) {
- interpreterSettings.put(setting.getId(), setting);
- }
-
- saveToFile();
- }
-
- for (String settingId : interpreterSettings.keySet()) {
- InterpreterSetting setting = interpreterSettings.get(settingId);
- logger.info("InterpreterSetting group {} : id={}, name={}", setting.getGroup(), settingId,
- setting.getName());
- }
- }
-
- private InterpreterSetting createFromInterpreterSettingRef(String name) {
- Preconditions.checkNotNull(name, "reference name should be not null");
- InterpreterSetting settingRef = interpreterSettingsRef.get(name);
- return createFromInterpreterSettingRef(settingRef);
- }
-
- private InterpreterSetting createFromInterpreterSettingRef(InterpreterSetting o) {
- // should return immutable objects
- List<InterpreterInfo> infos = (null == o.getInterpreterInfos()) ?
- new ArrayList<InterpreterInfo>() : new ArrayList<>(o.getInterpreterInfos());
- List<Dependency> deps = (null == o.getDependencies()) ?
- new ArrayList<Dependency>() : new ArrayList<>(o.getDependencies());
- Properties props =
- convertInterpreterProperties((Map<String, InterpreterProperty>) o.getProperties());
- InterpreterOption option = InterpreterOption.fromInterpreterOption(o.getOption());
-
- InterpreterSetting setting = new InterpreterSetting(o.getName(), o.getName(),
- infos, props, deps, option, o.getPath(), o.getInterpreterRunner());
- setting.setInterpreterGroupFactory(this);
- return setting;
- }
-
- private Properties convertInterpreterProperties(Map<String, InterpreterProperty> p) {
- Properties properties = new Properties();
- for (String key : p.keySet()) {
- properties.put(key, p.get(key).getValue());
- }
- return properties;
- }
-
- private boolean registerInterpreterFromResource(ClassLoader cl, String interpreterDir,
- String interpreterJson) throws IOException, RepositoryException {
- URL[] urls = recursiveBuildLibList(new File(interpreterDir));
- ClassLoader tempClassLoader = new URLClassLoader(urls, cl);
-
- InputStream inputStream = tempClassLoader.getResourceAsStream(interpreterJson);
-
- if (null != inputStream) {
- logger.debug("Reading {} from resources in {}", interpreterJson, interpreterDir);
- List<RegisteredInterpreter> registeredInterpreterList =
- getInterpreterListFromJson(inputStream);
- registerInterpreters(registeredInterpreterList, interpreterDir);
- return true;
- }
- return false;
- }
-
- private boolean registerInterpreterFromPath(String interpreterDir, String interpreterJson)
- throws IOException, RepositoryException {
-
- Path interpreterJsonPath = Paths.get(interpreterDir, interpreterJson);
- if (Files.exists(interpreterJsonPath)) {
- logger.debug("Reading {}", interpreterJsonPath);
- List<RegisteredInterpreter> registeredInterpreterList =
- getInterpreterListFromJson(interpreterJsonPath);
- registerInterpreters(registeredInterpreterList, interpreterDir);
- return true;
- }
- return false;
- }
-
- private List<RegisteredInterpreter> getInterpreterListFromJson(Path filename)
- throws FileNotFoundException {
- return getInterpreterListFromJson(new FileInputStream(filename.toFile()));
- }
-
- private List<RegisteredInterpreter> getInterpreterListFromJson(InputStream stream) {
- Type registeredInterpreterListType = new TypeToken<List<RegisteredInterpreter>>() {
- }.getType();
- return gson.fromJson(new InputStreamReader(stream), registeredInterpreterListType);
- }
-
- private void registerInterpreters(List<RegisteredInterpreter> registeredInterpreters,
- String absolutePath) throws IOException, RepositoryException {
-
- for (RegisteredInterpreter registeredInterpreter : registeredInterpreters) {
- InterpreterInfo interpreterInfo =
- new InterpreterInfo(registeredInterpreter.getClassName(), registeredInterpreter.getName(),
- registeredInterpreter.isDefaultInterpreter(), registeredInterpreter.getEditor());
- // use defaultOption if it is not specified in interpreter-setting.json
- InterpreterOption option = registeredInterpreter.getOption() == null ? defaultOption :
- registeredInterpreter.getOption();
- add(registeredInterpreter.getGroup(), interpreterInfo, registeredInterpreter.getProperties(),
- option, absolutePath, registeredInterpreter.getRunner());
- }
-
- }
-
- private void loadFromFile() throws IOException {
- File settingFile = new File(conf.getInterpreterSettingPath());
- if (!settingFile.exists()) {
- // nothing to read
- return;
- }
- FileInputStream fis = new FileInputStream(settingFile);
- InputStreamReader isr = new InputStreamReader(fis);
- BufferedReader bufferedReader = new BufferedReader(isr);
- StringBuilder sb = new StringBuilder();
- InterpreterSetting interpreterSettingObject;
- String depClassPath = StringUtils.EMPTY;
- String line;
- while ((line = bufferedReader.readLine()) != null) {
- sb.append(line);
- }
- isr.close();
- fis.close();
-
- String json = sb.toString();
- InterpreterInfoSaving infoSaving = gson.fromJson(json, InterpreterInfoSaving.class);
-
- for (String k : infoSaving.interpreterSettings.keySet()) {
- InterpreterSetting setting = infoSaving.interpreterSettings.get(k);
- List<InterpreterInfo> infos = setting.getInterpreterInfos();
-
- // Convert json StringMap to Properties
- StringMap<String> p = (StringMap<String>) setting.getProperties();
- Properties properties = new Properties();
- for (String key : p.keySet()) {
- properties.put(key, p.get(key));
- }
- setting.setProperties(properties);
-
- // Always use separate interpreter process
- // While we decided to turn this feature on always (without providing
- // enable/disable option on GUI).
- // previously created setting should turn this feature on here.
- setting.getOption().setRemote(true);
-
- // Update transient information from InterpreterSettingRef
- interpreterSettingObject = interpreterSettingsRef.get(setting.getGroup());
- if (interpreterSettingObject == null) {
- logger.warn("can't get InterpreterSetting " +
- "Information From loaded Interpreter Setting Ref - {} ", setting.getGroup());
- continue;
- }
- depClassPath = interpreterSettingObject.getPath();
- setting.setPath(depClassPath);
-
- for (InterpreterInfo info : infos) {
- if (info.getEditor() == null) {
- Map<String, Object> editor = getEditorFromSettingByClassName(interpreterSettingObject,
- info.getClassName());
- info.setEditor(editor);
- }
- }
-
- setting.setInterpreterGroupFactory(this);
- loadInterpreterDependencies(setting);
- interpreterSettings.put(k, setting);
- }
-
- this.interpreterBindings = infoSaving.interpreterBindings;
-
- if (infoSaving.interpreterRepositories != null) {
- for (RemoteRepository repo : infoSaving.interpreterRepositories) {
- if (!depResolver.getRepos().contains(repo)) {
- this.interpreterRepositories.add(repo);
- }
- }
- }
- }
-
- public Map<String, Object> getEditorFromSettingByClassName(InterpreterSetting intpSetting,
- String className) {
- List<InterpreterInfo> intpInfos = intpSetting.getInterpreterInfos();
- for (InterpreterInfo intpInfo : intpInfos) {
-
- if (className.equals(intpInfo.getClassName())) {
- if (intpInfo.getEditor() == null) {
- break;
- }
- return intpInfo.getEditor();
- }
- }
- return DEFAULT_EDITOR;
- }
-
- private void loadInterpreterDependencies(final InterpreterSetting setting) {
- setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
- setting.setErrorReason(null);
- interpreterSettings.put(setting.getId(), setting);
- synchronized (interpreterSettings) {
- final Thread t = new Thread() {
- public void run() {
- try {
- // dependencies to prevent library conflict
- File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" +
- setting.getId());
- if (localRepoDir.exists()) {
- try {
- FileUtils.cleanDirectory(localRepoDir);
- } catch (FileNotFoundException e) {
- logger.info("A file that does not exist cannot be deleted, nothing to worry", e);
- }
- }
-
- // load dependencies
- List<Dependency> deps = setting.getDependencies();
- if (deps != null) {
- for (Dependency d : deps) {
- File destDir = new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
-
- if (d.getExclusions() != null) {
- depResolver.load(d.getGroupArtifactVersion(), d.getExclusions(),
- new File(destDir, setting.getId()));
- } else {
- depResolver.load(d.getGroupArtifactVersion(), new File(destDir, setting.getId()));
- }
- }
- }
-
- setting.setStatus(InterpreterSetting.Status.READY);
- setting.setErrorReason(null);
- } catch (Exception e) {
- logger.error(String.format("Error while downloading repos for interpreter group : %s," +
- " go to interpreter setting page click on edit and save it again to make " +
- "this interpreter work properly. : %s",
- setting.getGroup(), e.getLocalizedMessage()), e);
- setting.setErrorReason(e.getLocalizedMessage());
- setting.setStatus(InterpreterSetting.Status.ERROR);
- } finally {
- interpreterSettings.put(setting.getId(), setting);
- }
- }
- };
- t.start();
- }
- }
-
- /**
- * Overwrite dependency jar under local-repo/{interpreterId}
- * if jar file in original path is changed
- */
- private void copyDependenciesFromLocalPath(final InterpreterSetting setting) {
- setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
- interpreterSettings.put(setting.getId(), setting);
- synchronized (interpreterSettings) {
- final Thread t = new Thread() {
- public void run() {
- try {
- List<Dependency> deps = setting.getDependencies();
- if (deps != null) {
- for (Dependency d : deps) {
- File destDir = new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
-
- int numSplits = d.getGroupArtifactVersion().split(":").length;
- if (!(numSplits >= 3 && numSplits <= 6)) {
- depResolver.copyLocalDependency(d.getGroupArtifactVersion(),
- new File(destDir, setting.getId()));
- }
- }
- }
- setting.setStatus(InterpreterSetting.Status.READY);
- } catch (Exception e) {
- logger.error(String.format("Error while copying deps for interpreter group : %s," +
- " go to interpreter setting page click on edit and save it again to make " +
- "this interpreter work properly.",
- setting.getGroup()), e);
- setting.setErrorReason(e.getLocalizedMessage());
- setting.setStatus(InterpreterSetting.Status.ERROR);
- } finally {
- interpreterSettings.put(setting.getId(), setting);
- }
- }
- };
- t.start();
- }
- }
-
- void saveToFile() throws IOException {
- String jsonString;
-
- synchronized (interpreterSettings) {
- InterpreterInfoSaving info = new InterpreterInfoSaving();
- info.interpreterBindings = interpreterBindings;
- info.interpreterSettings = interpreterSettings;
- info.interpreterRepositories = interpreterRepositories;
-
- jsonString = gson.toJson(info);
- }
-
- File settingFile = new File(conf.getInterpreterSettingPath());
- if (!settingFile.exists()) {
- settingFile.createNewFile();
- }
-
- FileOutputStream fos = new FileOutputStream(settingFile, false);
- OutputStreamWriter out = new OutputStreamWriter(fos);
- out.append(jsonString);
- out.close();
- fos.close();
- }
-
- /**
- * Return ordered interpreter setting list.
- * The list does not contain more than one setting from the same interpreter class.
- * Order by InterpreterClass (order defined by ZEPPELIN_INTERPRETERS), Interpreter setting name
- */
- public List<String> getDefaultInterpreterSettingList() {
- // this list will contain default interpreter setting list
- List<String> defaultSettings = new LinkedList<>();
-
- // to ignore the same interpreter group
- Map<String, Boolean> interpreterGroupCheck = new HashMap<>();
-
- List<InterpreterSetting> sortedSettings = get();
-
- for (InterpreterSetting setting : sortedSettings) {
- if (defaultSettings.contains(setting.getId())) {
- continue;
- }
-
- if (!interpreterGroupCheck.containsKey(setting.getName())) {
- defaultSettings.add(setting.getId());
- interpreterGroupCheck.put(setting.getName(), true);
- }
- }
- return defaultSettings;
- }
-
- List<RegisteredInterpreter> getRegisteredInterpreterList() {
- return new ArrayList<>(Interpreter.registeredInterpreters.values());
- }
-
-
- private boolean findDefaultInterpreter(List<InterpreterInfo> infos) {
- for (InterpreterInfo interpreterInfo : infos) {
- if (interpreterInfo.isDefaultInterpreter()) {
- return true;
- }
- }
- return false;
- }
-
- public InterpreterSetting createNewSetting(String name, String group,
- List<Dependency> dependencies, InterpreterOption option, Properties p) throws IOException {
- if (name.indexOf(".") >= 0) {
- throw new IOException("'.' is invalid for InterpreterSetting name.");
- }
- InterpreterSetting setting = createFromInterpreterSettingRef(group);
- setting.setName(name);
- setting.setGroup(group);
- setting.appendDependencies(dependencies);
- setting.setInterpreterOption(option);
- setting.setProperties(p);
- setting.setInterpreterGroupFactory(this);
- interpreterSettings.put(setting.getId(), setting);
- loadInterpreterDependencies(setting);
- saveToFile();
- return setting;
- }
-
- private InterpreterSetting add(String group, InterpreterInfo interpreterInfo,
- Map<String, InterpreterProperty> interpreterProperties, InterpreterOption option, String path,
- InterpreterRunner runner)
- throws InterpreterException, IOException, RepositoryException {
- ArrayList<InterpreterInfo> infos = new ArrayList<>();
- infos.add(interpreterInfo);
- return add(group, infos, new ArrayList<Dependency>(), option, interpreterProperties, path,
- runner);
- }
-
- /**
- * @param group InterpreterSetting reference name
- */
- public InterpreterSetting add(String group, ArrayList<InterpreterInfo> interpreterInfos,
- List<Dependency> dependencies, InterpreterOption option,
- Map<String, InterpreterProperty> interpreterProperties, String path,
- InterpreterRunner runner) {
- Preconditions.checkNotNull(group, "name should not be null");
- Preconditions.checkNotNull(interpreterInfos, "interpreterInfos should not be null");
- Preconditions.checkNotNull(dependencies, "dependencies should not be null");
- Preconditions.checkNotNull(option, "option should not be null");
- Preconditions.checkNotNull(interpreterProperties, "properties should not be null");
-
- InterpreterSetting interpreterSetting;
-
- synchronized (interpreterSettingsRef) {
- if (interpreterSettingsRef.containsKey(group)) {
- interpreterSetting = interpreterSettingsRef.get(group);
-
- // Append InterpreterInfo
- List<InterpreterInfo> infos = interpreterSetting.getInterpreterInfos();
- boolean hasDefaultInterpreter = findDefaultInterpreter(infos);
- for (InterpreterInfo interpreterInfo : interpreterInfos) {
- if (!infos.contains(interpreterInfo)) {
- if (!hasDefaultInterpreter && interpreterInfo.isDefaultInterpreter()) {
- hasDefaultInterpreter = true;
- infos.add(0, interpreterInfo);
- } else {
- infos.add(interpreterInfo);
- }
- }
- }
-
- // Append dependencies
- List<Dependency> dependencyList = interpreterSetting.getDependencies();
- for (Dependency dependency : dependencies) {
- if (!dependencyList.contains(dependency)) {
- dependencyList.add(dependency);
- }
- }
-
- // Append properties
- Map<String, InterpreterProperty> properties =
- (Map<String, InterpreterProperty>) interpreterSetting.getProperties();
- for (String key : interpreterProperties.keySet()) {
- if (!properties.containsKey(key)) {
- properties.put(key, interpreterProperties.get(key));
- }
- }
-
- } else {
- interpreterSetting =
- new InterpreterSetting(group, null, interpreterInfos, interpreterProperties,
- dependencies, option, path, runner);
- interpreterSettingsRef.put(group, interpreterSetting);
- }
- }
-
- if (dependencies.size() > 0) {
- loadInterpreterDependencies(interpreterSetting);
- }
-
- interpreterSetting.setInterpreterGroupFactory(this);
- return interpreterSetting;
- }
-
/**
* @param id interpreterGroup id. Combination of interpreterSettingId + noteId/userId/shared
* depends on interpreter mode
@@ -733,24 +159,6 @@ public class InterpreterFactory implements InterpreterGroupFactory {
return interpreterGroup;
}
- public void removeInterpretersForNote(InterpreterSetting interpreterSetting, String user,
- String noteId) {
- InterpreterOption option = interpreterSetting.getOption();
- if (option.isProcess()) {
- interpreterSetting.closeAndRemoveInterpreterGroupByNoteId(noteId);
- } else if (option.isSession()) {
- InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId);
- String key = getInterpreterSessionKey(user, noteId, interpreterSetting);
- interpreterGroup.close(key);
- synchronized (interpreterGroup) {
- interpreterGroup.remove(key);
- interpreterGroup.notifyAll(); // notify createInterpreterForNote()
- }
- logger.info("Interpreter instance {} for note {} is removed", interpreterSetting.getName(),
- noteId);
- }
- }
-
public void createInterpretersForNote(InterpreterSetting interpreterSetting, String user,
String noteId, String interpreterSessionKey) {
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId);
@@ -815,274 +223,6 @@ public class InterpreterFactory implements InterpreterGroupFactory {
}
}
-
- public void remove(String id) throws IOException {
- synchronized (interpreterSettings) {
- if (interpreterSettings.containsKey(id)) {
- InterpreterSetting intp = interpreterSettings.get(id);
- intp.closeAndRemoveAllInterpreterGroups();
-
- interpreterSettings.remove(id);
- for (List<String> settings : interpreterBindings.values()) {
- Iterator<String> it = settings.iterator();
- while (it.hasNext()) {
- String settingId = it.next();
- if (settingId.equals(id)) {
- it.remove();
- }
- }
- }
- saveToFile();
- }
- }
-
- File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + id);
- FileUtils.deleteDirectory(localRepoDir);
- }
-
- /**
- * Get interpreter settings
- */
- public List<InterpreterSetting> get() {
- synchronized (interpreterSettings) {
- List<InterpreterSetting> orderedSettings = new LinkedList<>();
-
- Map<String, List<InterpreterSetting>> nameInterpreterSettingMap = new HashMap<>();
- for (InterpreterSetting interpreterSetting : interpreterSettings.values()) {
- String name = interpreterSetting.getName();
- if (!nameInterpreterSettingMap.containsKey(name)) {
- nameInterpreterSettingMap.put(name, new ArrayList<InterpreterSetting>());
- }
- nameInterpreterSettingMap.get(name).add(interpreterSetting);
- }
-
- for (String groupName : interpreterGroupOrderList) {
- List<InterpreterSetting> interpreterSettingList =
- nameInterpreterSettingMap.remove(groupName);
- if (null != interpreterSettingList) {
- for (InterpreterSetting interpreterSetting : interpreterSettingList) {
- orderedSettings.add(interpreterSetting);
- }
- }
- }
-
- List<InterpreterSetting> settings = new ArrayList<>();
-
- for (List<InterpreterSetting> interpreterSettingList : nameInterpreterSettingMap.values()) {
- for (InterpreterSetting interpreterSetting : interpreterSettingList) {
- settings.add(interpreterSetting);
- }
- }
-
- Collections.sort(settings, new Comparator<InterpreterSetting>() {
- @Override
- public int compare(InterpreterSetting o1, InterpreterSetting o2) {
- return o1.getName().compareTo(o2.getName());
- }
- });
-
- orderedSettings.addAll(settings);
-
- return orderedSettings;
- }
- }
-
- public InterpreterSetting get(String name) {
- synchronized (interpreterSettings) {
- return interpreterSettings.get(name);
- }
- }
-
- private void putNoteInterpreterSettingBinding(String user, String noteId,
- List<String> settingList) throws IOException {
- List<String> unBindedSettings = new LinkedList<>();
-
- synchronized (interpreterSettings) {
- List<String> oldSettings = interpreterBindings.get(noteId);
- if (oldSettings != null) {
- for (String oldSettingId : oldSettings) {
- if (!settingList.contains(oldSettingId)) {
- unBindedSettings.add(oldSettingId);
- }
- }
- }
- interpreterBindings.put(noteId, settingList);
- saveToFile();
-
- for (String settingId : unBindedSettings) {
- InterpreterSetting setting = get(settingId);
- removeInterpretersForNote(setting, user, noteId);
- }
- }
- }
-
- public void removeNoteInterpreterSettingBinding(String user, String noteId) {
- synchronized (interpreterSettings) {
- List<String> settingIds = (interpreterBindings.containsKey(noteId) ?
- interpreterBindings.remove(noteId) :
- Collections.<String>emptyList());
- for (String settingId : settingIds) {
- this.removeInterpretersForNote(get(settingId), user, noteId);
- }
- }
- }
-
- private List<String> getNoteInterpreterSettingBinding(String noteId) {
- LinkedList<String> bindings = new LinkedList<>();
- synchronized (interpreterSettings) {
- List<String> settingIds = interpreterBindings.get(noteId);
- if (settingIds != null) {
- bindings.addAll(settingIds);
- }
- }
- return bindings;
- }
-
- /**
- * Change interpreter property and restart
- */
- public void setPropertyAndRestart(String id, InterpreterOption option, Properties properties,
- List<Dependency> dependencies) throws IOException {
- synchronized (interpreterSettings) {
- InterpreterSetting intpSetting = interpreterSettings.get(id);
- if (intpSetting != null) {
- try {
- stopJobAllInterpreter(intpSetting);
-
- intpSetting.closeAndRemoveAllInterpreterGroups();
- intpSetting.setOption(option);
- intpSetting.setProperties(properties);
- intpSetting.setDependencies(dependencies);
- loadInterpreterDependencies(intpSetting);
-
- saveToFile();
- } catch (Exception e) {
- throw e;
- } finally {
- loadFromFile();
- }
- } else {
- throw new InterpreterException("Interpreter setting id " + id + " not found");
- }
- }
- }
-
- private boolean noteIdIsExist(String noteId) {
- return noteId == null ? false : true;
- }
-
- public void restart(String settingId, String noteId, String user) {
- InterpreterSetting intpSetting = interpreterSettings.get(settingId);
- Preconditions.checkNotNull(intpSetting);
-
- // restart interpreter setting in note page
- if (noteIdIsExist(noteId) && intpSetting.getOption().isProcess()) {
- intpSetting.closeAndRemoveInterpreterGroupByNoteId(noteId);
- return;
- } else {
- // restart interpreter setting in interpreter setting page
- restart(settingId, user);
- }
-
-
- }
-
- public void restart(String id, String user) {
- synchronized (interpreterSettings) {
- InterpreterSetting intpSetting = interpreterSettings.get(id);
- // Check if dependency in specified path is changed
- // If it did, overwrite old dependency jar with new one
- if (intpSetting != null) {
- //clean up metaInfos
- intpSetting.setInfos(null);
- copyDependenciesFromLocalPath(intpSetting);
-
- stopJobAllInterpreter(intpSetting);
- if (user.equals("anonymous")) {
- intpSetting.closeAndRemoveAllInterpreterGroups();
- } else {
- intpSetting.closeAndRemoveInterpreterGroupByUser(user);
- }
-
- } else {
- throw new InterpreterException("Interpreter setting id " + id + " not found");
- }
- }
- }
-
- public void restart(String id) {
- restart(id, "anonymous");
- }
-
- private void stopJobAllInterpreter(InterpreterSetting intpSetting) {
- if (intpSetting != null) {
- for (InterpreterGroup intpGroup : intpSetting.getAllInterpreterGroups()) {
- for (List<Interpreter> interpreters : intpGroup.values()) {
- for (Interpreter intp : interpreters) {
- for (Job job : intp.getScheduler().getJobsRunning()) {
- job.abort();
- job.setStatus(Status.ABORT);
- logger.info("Job " + job.getJobName() + " aborted ");
- }
- for (Job job : intp.getScheduler().getJobsWaiting()) {
- job.abort();
- job.setStatus(Status.ABORT);
- logger.info("Job " + job.getJobName() + " aborted ");
- }
- }
- }
- }
- }
- }
-
- public void close() {
- List<Thread> closeThreads = new LinkedList<>();
- synchronized (interpreterSettings) {
- Collection<InterpreterSetting> intpSettings = interpreterSettings.values();
- for (final InterpreterSetting intpSetting : intpSettings) {
- Thread t = new Thread() {
- public void run() {
- intpSetting.closeAndRemoveAllInterpreterGroups();
- }
- };
- t.start();
- closeThreads.add(t);
- }
- }
-
- for (Thread t : closeThreads) {
- try {
- t.join();
- } catch (InterruptedException e) {
- logger.error("Can't close interpreterGroup", e);
- }
- }
- }
-
- public void shutdown() {
- List<Thread> closeThreads = new LinkedList<>();
- synchronized (interpreterSettings) {
- Collection<InterpreterSetting> intpSettings = interpreterSettings.values();
- for (final InterpreterSetting intpSetting : intpSettings) {
- Thread t = new Thread() {
- public void run() {
- intpSetting.shutdownAndRemoveAllInterpreterGroups();
- }
- };
- t.start();
- closeThreads.add(t);
- }
- }
-
- for (Thread t : closeThreads) {
- try {
- t.join();
- } catch (InterruptedException e) {
- logger.error("Can't close interpreterGroup", e);
- }
- }
- }
-
private Interpreter createRepl(String dirName, String className, Properties property)
throws InterpreterException {
logger.info("Create repl {} from {}", className, dirName);
@@ -1181,76 +321,12 @@ public class InterpreterFactory implements InterpreterGroupFactory {
return new LazyOpenInterpreter(remoteInterpreter);
}
- /**
- * map interpreter ids into noteId
- *
- * @param noteId note id
- * @param ids InterpreterSetting id list
- */
- public void setInterpreters(String user, String noteId, List<String> ids) throws IOException {
- putNoteInterpreterSettingBinding(user, noteId, ids);
- }
-
- public List<String> getInterpreters(String noteId) {
- return getNoteInterpreterSettingBinding(noteId);
- }
-
- public List<InterpreterSetting> getInterpreterSettings(String noteId) {
- List<String> interpreterSettingIds = getNoteInterpreterSettingBinding(noteId);
- LinkedList<InterpreterSetting> settings = new LinkedList<>();
-
- Iterator<String> iter = interpreterSettingIds.iterator();
- while (iter.hasNext()) {
- String id = iter.next();
- InterpreterSetting setting = get(id);
- if (setting == null) {
- // interpreter setting is removed from factory. remove id from here, too
- iter.remove();
- } else {
- settings.add(setting);
- }
- }
- return settings;
- }
-
- public void closeNote(String user, String noteId) {
- // close interpreters in this note session
- List<InterpreterSetting> settings = getInterpreterSettings(noteId);
- if (settings == null || settings.size() == 0) {
- return;
- }
-
- logger.info("closeNote: {}", noteId);
- for (InterpreterSetting setting : settings) {
- removeInterpretersForNote(setting, user, noteId);
- }
- }
-
- private String getInterpreterSessionKey(String user, String noteId, InterpreterSetting setting) {
- InterpreterOption option = setting.getOption();
- String key;
- if (option.isExistingProcess()) {
- key = Constants.EXISTING_PROCESS;
- } else if (option.perNoteScoped() && option.perUserScoped()) {
- key = user + ":" + noteId;
- } else if (option.perUserScoped()) {
- key = user;
- } else if (option.perNoteScoped()) {
- key = noteId;
- } else {
- key = SHARED_SESSION;
- }
-
- logger.debug("Interpreter session key: {}, for note: {}, user: {}, InterpreterSetting Name: " +
- "{}", key, noteId, user, setting.getName());
- return key;
- }
-
private List<Interpreter> createOrGetInterpreterList(String user, String noteId,
InterpreterSetting setting) {
InterpreterGroup interpreterGroup = setting.getInterpreterGroup(user, noteId);
synchronized (interpreterGroup) {
- String interpreterSessionKey = getInterpreterSessionKey(user, noteId, setting);
+ String interpreterSessionKey =
+ interpreterSettingManager.getInterpreterSessionKey(user, noteId, setting);
if (!interpreterGroup.containsKey(interpreterSessionKey)) {
createInterpretersForNote(setting, user, noteId, interpreterSessionKey);
}
@@ -1258,18 +334,6 @@ public class InterpreterFactory implements InterpreterGroupFactory {
}
}
- private InterpreterSetting getDefaultInterpreterSetting(List<InterpreterSetting> settings) {
- if (settings == null || settings.isEmpty()) {
- return null;
- }
- return settings.get(0);
- }
-
- public InterpreterSetting getDefaultInterpreterSetting(String noteId) {
- return getDefaultInterpreterSetting(getInterpreterSettings(noteId));
- }
-
-
private InterpreterSetting getInterpreterSettingByGroup(List<InterpreterSetting> settings,
String group) {
Preconditions.checkNotNull(group, "group should be not null");
@@ -1314,7 +378,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
}
public Interpreter getInterpreter(String user, String noteId, String replName) {
- List<InterpreterSetting> settings = getInterpreterSettings(noteId);
+ List<InterpreterSetting> settings = interpreterSettingManager.getInterpreterSettings(noteId);
InterpreterSetting setting;
Interpreter interpreter;
@@ -1325,7 +389,8 @@ public class InterpreterFactory implements InterpreterGroupFactory {
if (replName == null || replName.trim().length() == 0) {
// get default settings (first available)
// TODO(jl): Fix it in case of returning null
- InterpreterSetting defaultSettings = getDefaultInterpreterSetting(settings);
+ InterpreterSetting defaultSettings = interpreterSettingManager
+ .getDefaultInterpreterSetting(settings);
return createOrGetInterpreterList(user, noteId, defaultSettings).get(0);
}
@@ -1352,7 +417,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
// first assume replName is 'name' of interpreter. ('groupName' is ommitted)
// search 'name' from first (default) interpreter group
// TODO(jl): Handle with noteId to support defaultInterpreter per note.
- setting = getDefaultInterpreterSetting(settings);
+ setting = interpreterSettingManager.getDefaultInterpreterSetting(settings);
interpreter = getInterpreter(user, noteId, setting, replName);
@@ -1385,44 +450,6 @@ public class InterpreterFactory implements InterpreterGroupFactory {
return null;
}
- public Map<String, InterpreterSetting> getAvailableInterpreterSettings() {
- return interpreterSettingsRef;
- }
-
- private URL[] recursiveBuildLibList(File path) throws MalformedURLException {
- URL[] urls = new URL[0];
- if (path == null || !path.exists()) {
- return urls;
- } else if (path.getName().startsWith(".")) {
- return urls;
- } else if (path.isDirectory()) {
- File[] files = path.listFiles();
- if (files != null) {
- for (File f : files) {
- urls = (URL[]) ArrayUtils.addAll(urls, recursiveBuildLibList(f));
- }
- }
- return urls;
- } else {
- return new URL[]{path.toURI().toURL()};
- }
- }
-
- public List<RemoteRepository> getRepositories() {
- return this.interpreterRepositories;
- }
-
- public void addRepository(String id, String url, boolean snapshot, Authentication auth,
- Proxy proxy) throws IOException {
- depResolver.addRepo(id, url, snapshot, auth, proxy);
- saveToFile();
- }
-
- public void removeRepository(String id) throws IOException {
- depResolver.delRepo(id);
- saveToFile();
- }
-
public Map<String, String> getEnv() {
return env;
}
@@ -1431,31 +458,5 @@ public class InterpreterFactory implements InterpreterGroupFactory {
this.env = env;
}
- public Map<String, Object> getEditorSetting(String user, String noteId, String replName) {
- Interpreter intp = getInterpreter(user, noteId, replName);
- Map<String, Object> editor = DEFAULT_EDITOR;
- String group = StringUtils.EMPTY;
- try {
- String defaultSettingName = getDefaultInterpreterSetting(noteId).getName();
- List<InterpreterSetting> intpSettings = getInterpreterSettings(noteId);
- for (InterpreterSetting intpSetting : intpSettings) {
- String[] replNameSplit = replName.split("\\.");
- if (replNameSplit.length == 2) {
- group = replNameSplit[0];
- }
- // when replName is 'name' of interpreter
- if (defaultSettingName.equals(intpSetting.getName())) {
- editor = getEditorFromSettingByClassName(intpSetting, intp.getClassName());
- }
- // when replName is 'alias name' of interpreter or 'group' of interpreter
- if (replName.equals(intpSetting.getName()) || group.equals(intpSetting.getName())) {
- editor = getEditorFromSettingByClassName(intpSetting, intp.getClassName());
- break;
- }
- }
- } catch (NullPointerException e) {
- logger.warn("Couldn't get interpreter editor setting");
- }
- return editor;
- }
+
}