You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2016/02/01 03:08:01 UTC
[1/2] incubator-zeppelin git commit: [Zeppelin-630] Introduce new way
of dependency loading to intepreter
Repository: incubator-zeppelin
Updated Branches:
refs/heads/master e87f5b1ab -> 218a3b5bc
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
new file mode 100644
index 0000000..8886add
--- /dev/null
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.rest;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.httpclient.methods.DeleteMethod;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.methods.PostMethod;
+import org.apache.commons.httpclient.methods.PutMethod;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.zeppelin.interpreter.InterpreterSetting;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.Paragraph;
+import org.apache.zeppelin.scheduler.Job.Status;
+import org.apache.zeppelin.server.ZeppelinServer;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+
+import static org.junit.Assert.*;
+
+/**
+ * Zeppelin interpreter rest api tests
+ */
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class InterpreterRestApiTest extends AbstractTestRestApi {
+ Gson gson = new Gson();
+
+ @BeforeClass
+ public static void init() throws Exception {
+ AbstractTestRestApi.startUp();
+ }
+
+ @AfterClass
+ public static void destroy() throws Exception {
+ AbstractTestRestApi.shutDown();
+ }
+
+ @Test
+ public void getAvailableInterpreters() throws IOException {
+ // when
+ GetMethod get = httpGet("/interpreter");
+
+ // then
+ assertThat(get, isAllowed());
+ Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
+ }.getType());
+ Map<String, Object> body = (Map<String, Object>) resp.get("body");
+ assertEquals(ZeppelinServer.notebook.getInterpreterFactory().getRegisteredInterpreterList().size(), body.size());
+ get.releaseConnection();
+ }
+
+ @Test
+ public void getSettings() throws IOException {
+ // when
+ GetMethod get = httpGet("/interpreter/setting");
+
+ // then
+ Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
+ }.getType());
+ assertThat(get, isAllowed());
+ get.releaseConnection();
+ }
+
+ @Test
+ public void testSettingsCRUD() throws IOException {
+ // Call Create Setting REST API
+ String jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"propvalue\"}," +
+ "\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
+ "\"dependencies\":[]}";
+ PostMethod post = httpPost("/interpreter/setting/", jsonRequest);
+ LOG.info("testSettingCRUD create response\n" + post.getResponseBodyAsString());
+ assertThat("test create method:", post, isCreated());
+
+ Map<String, Object> resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
+ }.getType());
+ Map<String, Object> body = (Map<String, Object>) resp.get("body");
+ //extract id from body string {id=2AWMQDNX7, name=md2, group=md,
+ String newSettingId = body.toString().split(",")[0].split("=")[1];
+ post.releaseConnection();
+
+ // Call Update Setting REST API
+ jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"Otherpropvalue\"}," +
+ "\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
+ "\"dependencies\":[]}";
+ PutMethod put = httpPut("/interpreter/setting/" + newSettingId, jsonRequest);
+ LOG.info("testSettingCRUD update response\n" + put.getResponseBodyAsString());
+ assertThat("test update method:", put, isAllowed());
+ put.releaseConnection();
+
+ // Call Delete Setting REST API
+ DeleteMethod delete = httpDelete("/interpreter/setting/" + newSettingId);
+ LOG.info("testSettingCRUD delete response\n" + delete.getResponseBodyAsString());
+ assertThat("Test delete method:", delete, isAllowed());
+ delete.releaseConnection();
+ }
+
+ @Test
+ public void testInterpreterAutoBinding() throws IOException {
+ // create note
+ Note note = ZeppelinServer.notebook.createNote();
+
+ // check interpreter is binded
+ GetMethod get = httpGet("/notebook/interpreter/bind/" + note.id());
+ assertThat(get, isAllowed());
+ get.addRequestHeader("Origin", "http://localhost");
+ Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
+ }.getType());
+ List<Map<String, String>> body = (List<Map<String, String>>) resp.get("body");
+ assertTrue(0 < body.size());
+
+ get.releaseConnection();
+ //cleanup
+ ZeppelinServer.notebook.removeNote(note.getId());
+ }
+
+ @Test
+ public void testInterpreterRestart() throws IOException, InterruptedException {
+ // create new note
+ Note note = ZeppelinServer.notebook.createNote();
+ note.addParagraph();
+ Paragraph p = note.getLastParagraph();
+ Map config = p.getConfig();
+ config.put("enabled", true);
+
+ // run markdown paragraph
+ p.setConfig(config);
+ p.setText("%md markdown");
+ note.run(p.getId());
+ while (p.getStatus() != Status.FINISHED) {
+ Thread.sleep(100);
+ }
+ assertEquals("<p>markdown</p>\n", p.getResult().message());
+
+
+ // restart interpreter
+ for (InterpreterSetting setting : note.getNoteReplLoader().getInterpreterSettings()) {
+ if (setting.getName().equals("md")) {
+ // Call Restart Interpreter REST API
+ PutMethod put = httpPut("/interpreter/setting/restart/" + setting.id(), "");
+ assertThat("test interpreter restart:", put, isAllowed());
+ put.releaseConnection();
+ break;
+ }
+ }
+
+ // run markdown paragraph, again
+ p = note.addParagraph();
+ p.setConfig(config);
+ p.setText("%md markdown restarted");
+ note.run(p.getId());
+ while (p.getStatus() != Status.FINISHED) {
+ Thread.sleep(100);
+ }
+ assertEquals("<p>markdown restarted</p>\n", p.getResult().message());
+ //cleanup
+ ZeppelinServer.notebook.removeNote(note.getId());
+ }
+
+ @Test
+ public void testListRepository() throws IOException {
+ GetMethod get = httpGet("/interpreter/repository");
+ assertThat(get, isAllowed());
+ get.releaseConnection();
+ }
+
+ @Test
+ public void testAddDeleteRepository() throws IOException {
+ // Call create repository REST API
+ String repoId = "securecentral";
+ String jsonRequest = "{\"id\":\"" + repoId +
+ "\",\"url\":\"https://repo1.maven.org/maven2\",\"snapshot\":\"false\"}";
+
+ PostMethod post = httpPost("/interpreter/repository/", jsonRequest);
+ assertThat("Test create method:", post, isCreated());
+ post.releaseConnection();
+
+ // Call delete repository REST API
+ DeleteMethod delete = httpDelete("/interpreter/repository/" + repoId);
+ assertThat("Test delete method:", delete, isAllowed());
+ delete.releaseConnection();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java
index d8049cc..3c7c7d0 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java
@@ -72,124 +72,6 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
httpGetRoot.releaseConnection();
}
-
- @Test
- public void getAvailableInterpreters() throws IOException {
- // when
- GetMethod get = httpGet("/interpreter");
-
- // then
- assertThat(get, isAllowed());
- Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
- }.getType());
- Map<String, Object> body = (Map<String, Object>) resp.get("body");
- assertEquals(ZeppelinServer.notebook.getInterpreterFactory().getRegisteredInterpreterList().size(), body.size());
- get.releaseConnection();
- }
-
- @Test
- public void getSettings() throws IOException {
- // when
- GetMethod get = httpGet("/interpreter/setting");
-
- // then
- Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
- }.getType());
- assertThat(get, isAllowed());
- get.releaseConnection();
- }
-
- @Test
- public void testSettingsCRUD() throws IOException {
- // Call Create Setting REST API
- String jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"propvalue\"},\"" +
- "interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]}";
- PostMethod post = httpPost("/interpreter/setting/", jsonRequest);
- LOG.info("testSettingCRUD create response\n" + post.getResponseBodyAsString());
- assertThat("test create method:", post, isCreated());
-
- Map<String, Object> resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
- }.getType());
- Map<String, Object> body = (Map<String, Object>) resp.get("body");
- //extract id from body string {id=2AWMQDNX7, name=md2, group=md,
- String newSettingId = body.toString().split(",")[0].split("=")[1];
- post.releaseConnection();
-
- // Call Update Setting REST API
- jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"Otherpropvalue\"},\"" +
- "interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]}";
- PutMethod put = httpPut("/interpreter/setting/" + newSettingId, jsonRequest);
- LOG.info("testSettingCRUD update response\n" + put.getResponseBodyAsString());
- assertThat("test update method:", put, isAllowed());
- put.releaseConnection();
-
- // Call Delete Setting REST API
- DeleteMethod delete = httpDelete("/interpreter/setting/" + newSettingId);
- LOG.info("testSettingCRUD delete response\n" + delete.getResponseBodyAsString());
- assertThat("Test delete method:", delete, isAllowed());
- delete.releaseConnection();
- }
- @Test
- public void testInterpreterAutoBinding() throws IOException {
- // create note
- Note note = ZeppelinServer.notebook.createNote();
-
- // check interpreter is binded
- GetMethod get = httpGet("/notebook/interpreter/bind/"+note.id());
- assertThat(get, isAllowed());
- get.addRequestHeader("Origin", "http://localhost");
- Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>(){}.getType());
- List<Map<String, String>> body = (List<Map<String, String>>) resp.get("body");
- assertTrue(0 < body.size());
-
- get.releaseConnection();
- //cleanup
- ZeppelinServer.notebook.removeNote(note.getId());
- }
-
- @Test
- public void testInterpreterRestart() throws IOException, InterruptedException {
- // create new note
- Note note = ZeppelinServer.notebook.createNote();
- note.addParagraph();
- Paragraph p = note.getLastParagraph();
- Map config = p.getConfig();
- config.put("enabled", true);
-
- // run markdown paragraph
- p.setConfig(config);
- p.setText("%md markdown");
- note.run(p.getId());
- while (p.getStatus() != Status.FINISHED) {
- Thread.sleep(100);
- }
- assertEquals("<p>markdown</p>\n", p.getResult().message());
-
-
- // restart interpreter
- for (InterpreterSetting setting : note.getNoteReplLoader().getInterpreterSettings()) {
- if (setting.getName().equals("md")) {
- // Call Restart Interpreter REST API
- PutMethod put = httpPut("/interpreter/setting/restart/" + setting.id(), "");
- assertThat("test interpreter restart:", put, isAllowed());
- put.releaseConnection();
- break;
- }
- }
-
- // run markdown paragraph, again
- p = note.addParagraph();
- p.setConfig(config);
- p.setText("%md markdown restarted");
- note.run(p.getId());
- while (p.getStatus() != Status.FINISHED) {
- Thread.sleep(100);
- }
- assertEquals("<p>markdown restarted</p>\n", p.getResult().message());
- //cleanup
- ZeppelinServer.notebook.removeNote(note.getId());
- }
-
@Test
public void testGetNotebookInfo() throws IOException {
LOG.info("testGetNotebookInfo");
@@ -303,7 +185,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
}
@Test
- public void testDeleteNote() throws IOException {
+ public void testDeleteNote() throws IOException {
LOG.info("testDeleteNote");
//Create note and get ID
Note note = ZeppelinServer.notebook.createNote();
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-web/src/app/app.js
----------------------------------------------------------------------
diff --git a/zeppelin-web/src/app/app.js b/zeppelin-web/src/app/app.js
index d13faf0..33a9daf 100644
--- a/zeppelin-web/src/app/app.js
+++ b/zeppelin-web/src/app/app.js
@@ -81,7 +81,8 @@
ngToastProvider.configure({
dismissButton: true,
dismissOnClick: false,
- timeout: 6000
+ timeout: 6000,
+ verticalPosition: 'bottom'
});
});
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-web/src/app/home/home.css
----------------------------------------------------------------------
diff --git a/zeppelin-web/src/app/home/home.css b/zeppelin-web/src/app/home/home.css
index 0202142..dada898 100644
--- a/zeppelin-web/src/app/home/home.css
+++ b/zeppelin-web/src/app/home/home.css
@@ -276,31 +276,6 @@ kbd {
}
/*
- ngToast Style
-*/
-
-.ng-toast .alert {
- color: white !important;
- border: none !important;
-}
-
-.ng-toast .alert-danger {
- background: #A94442 !important;
-}
-
-.ng-toast .alert-warning {
- background: #CE9532 !important;
-}
-
-.ng-toast .alert-info {
- background: #589EC1 !important;
-}
-
-.ng-toast .alert-success {
- background: #428443 !important;
-}
-
-/*
temporary fix for bootstrap issue (https://github.com/twbs/bootstrap/issues/5865)
This part should be removed when new version of bootstrap handles this issue.
*/
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html
----------------------------------------------------------------------
diff --git a/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html b/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html
index 38ae7cc..6f46c4e 100644
--- a/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html
+++ b/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html
@@ -12,10 +12,10 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
<div>
- <div class="row">
- <div class="col-md-12">
- <div class="interpreterSettingAdd" ng-show="showAddNewSetting">
- <hr />
+ <div class="row interpreter">
+ <div class="col-md-12" ng-show="showAddNewSetting">
+ <hr />
+ <div class="interpreterSettingAdd">
<h4>Create new interpreter</h4>
<div class="form-group" style="width:200px">
@@ -67,6 +67,50 @@ limitations under the License.
</tr>
</table>
+ <b>Dependencies</b>
+ <table class="table table-striped properties">
+ <tr>
+ <th>artifact</th>
+ <th>exclude</th>
+ <th>action</th>
+ </tr>
+
+ <tr ng-repeat="dep in newInterpreterSetting.dependencies">
+ <td>
+ <input ng-model="dep.groupArtifactVersion" style="width:100%" />
+ </td>
+ <td>
+ <textarea msd-elastic ng-model="dep.exclusions"
+ ng-list
+ placeholder="(Optional) comma separated groupId:artifactId list">
+ </textarea>
+ </td>
+ <td>
+ <div class="btn btn-default btn-sm fa fa-remove"
+ ng-click="removeInterpreterDependency(dep.groupArtifactVersion)">
+ </div>
+ </td>
+ </tr>
+
+ <tr>
+ <td>
+ <input ng-model="newInterpreterSetting.depArtifact"
+ placeholder="groupId:artifactId:version or local file path"
+ style="width: 100%" />
+ </td>
+ <td>
+ <textarea msd-elastic ng-model="newInterpreterSetting.depExclude"
+ ng-list
+ placeholder="(Optional) comma separated groupId:artifactId list">
+ </textarea>
+ </td>
+ <td>
+ <div class="btn btn-default btn-sm fa fa-plus" ng-click="addNewInterpreterDependency()">
+ </div>
+ </td>
+ </tr>
+ </table>
+
<span class="btn btn-primary" ng-click="addNewInterpreterSetting()">
Save
</span>
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-web/src/app/interpreter/interpreter.controller.js
----------------------------------------------------------------------
diff --git a/zeppelin-web/src/app/interpreter/interpreter.controller.js b/zeppelin-web/src/app/interpreter/interpreter.controller.js
index 396a552..1e3800a 100644
--- a/zeppelin-web/src/app/interpreter/interpreter.controller.js
+++ b/zeppelin-web/src/app/interpreter/interpreter.controller.js
@@ -15,37 +15,42 @@
'use strict';
angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope, $route, $routeParams, $location, $rootScope,
- $http, baseUrlSrv) {
+ $http, baseUrlSrv, ngToast) {
var interpreterSettingsTmp = [];
$scope.interpreterSettings = [];
$scope.availableInterpreters = {};
$scope.showAddNewSetting = false;
+ $scope.showRepositoryInfo = false;
$scope._ = _;
var getInterpreterSettings = function() {
$http.get(baseUrlSrv.getRestApiBase()+'/interpreter/setting').
- success(function(data, status, headers, config) {
- $scope.interpreterSettings = data.body;
- }).
- error(function(data, status, headers, config) {
- console.log('Error %o %o', status, data.message);
- });
+ success(function(data, status, headers, config) {
+ $scope.interpreterSettings = data.body;
+ }).
+ error(function(data, status, headers, config) {
+ console.log('Error %o %o', status, data.message);
+ });
};
var getAvailableInterpreters = function() {
$http.get(baseUrlSrv.getRestApiBase()+'/interpreter').
- success(function(data, status, headers, config) {
- $scope.availableInterpreters = data.body;
- }).
- error(function(data, status, headers, config) {
- console.log('Error %o %o', status, data.message);
- });
+ success(function(data, status, headers, config) {
+ $scope.availableInterpreters = data.body;
+ }).
+ error(function(data, status, headers, config) {
+ console.log('Error %o %o', status, data.message);
+ });
};
var emptyNewProperty = function(object) {
angular.extend(object, {propertyValue: '', propertyKey: ''});
};
+ var emptyNewDependency = function(object) {
+ angular.extend(object, {depArtifact: '', depExclude: ''});
+ };
+
var removeTMPSettings = function(index) {
interpreterSettingsTmp.splice(index, 1);
};
@@ -55,29 +60,34 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope,
interpreterSettingsTmp[index] = angular.copy($scope.interpreterSettings[index]);
};
- $scope.updateInterpreterSetting = function(settingId) {
- BootstrapDialog.confirm({
- closable: true,
- title: '',
- message: 'Do you want to update this interpreter and restart with new settings?',
- callback: function(result) {
- if (result) {
- var index = _.findIndex($scope.interpreterSettings, {'id': settingId});
- var request = {
- properties: angular.copy($scope.interpreterSettings[index].properties),
- };
-
- $http.put(baseUrlSrv.getRestApiBase() + '/interpreter/setting/' + settingId, request).
- success(function (data, status, headers, config) {
- $scope.interpreterSettings[index] = data.body;
- removeTMPSettings(index);
- }).
- error(function (data, status, headers, config) {
- console.log('Error %o %o', status, data.message);
- });
- }
+ $scope.updateInterpreterSetting = function(form, settingId) {
+ var result = confirm('Do you want to update this interpreter and restart with new settings?');
+ if (result) {
+ var index = _.findIndex($scope.interpreterSettings, {'id': settingId});
+ var setting = $scope.interpreterSettings[index];
+ if (setting.propertyKey !== '' || setting.propertyKey) {
+ $scope.addNewInterpreterProperty(settingId);
}
- });
+ if (setting.depArtifact !== '' || setting.depArtifact) {
+ $scope.addNewInterpreterDependency(settingId);
+ }
+
+ var request = {
+ properties: angular.copy(setting.properties),
+ dependencies: angular.copy(setting.dependencies)
+ };
+
+ $http.put(baseUrlSrv.getRestApiBase() + '/interpreter/setting/' + settingId, request).
+ success(function (data, status, headers, config) {
+ $scope.interpreterSettings[index] = data.body;
+ removeTMPSettings(index);
+ }).
+ error(function (data, status, headers, config) {
+ console.log('Error %o %o', status, data.message);
+ ngToast.danger(data.message);
+ form.$show();
+ });
+ }
};
$scope.resetInterpreterSetting = function(settingId){
@@ -117,8 +127,8 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope,
var intpInfo = el[i];
for (var key in intpInfo) {
properties[key] = {
- value : intpInfo[key].defaultValue,
- description : intpInfo[key].description
+ value: intpInfo[key].defaultValue,
+ description: intpInfo[key].description
};
}
}
@@ -165,32 +175,46 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope,
return;
}
- var newSetting = angular.copy($scope.newInterpreterSetting);
+ var newSetting = $scope.newInterpreterSetting;
+ if (newSetting.propertyKey !== '' || newSetting.propertyKey) {
+ $scope.addNewInterpreterProperty();
+ }
+ if (newSetting.depArtifact !== '' || newSetting.depArtifact) {
+ $scope.addNewInterpreterDependency();
+ }
+
+ var request = angular.copy($scope.newInterpreterSetting);
- for (var p in $scope.newInterpreterSetting.properties) {
- newSetting.properties[p] = $scope.newInterpreterSetting.properties[p].value;
+ // Change properties to proper request format
+ var newProperties = {};
+ for (var p in newSetting.properties) {
+ newProperties[p] = newSetting.properties[p].value;
}
+ request.properties = newProperties;
- $http.post(baseUrlSrv.getRestApiBase()+'/interpreter/setting', newSetting).
- success(function(data, status, headers, config) {
- $scope.resetNewInterpreterSetting();
- getInterpreterSettings();
- $scope.showAddNewSetting = false;
- }).
- error(function(data, status, headers, config) {
- console.log('Error %o %o', status, data.message);
- });
+ $http.post(baseUrlSrv.getRestApiBase() + '/interpreter/setting', request).
+ success(function(data, status, headers, config) {
+ $scope.resetNewInterpreterSetting();
+ getInterpreterSettings();
+ $scope.showAddNewSetting = false;
+ }).
+ error(function(data, status, headers, config) {
+ console.log('Error %o %o', status, data.message);
+ ngToast.danger(data.message);
+ });
};
$scope.cancelInterpreterSetting = function() {
$scope.showAddNewSetting = false;
+ $scope.resetNewInterpreterSetting();
};
$scope.resetNewInterpreterSetting = function() {
$scope.newInterpreterSetting = {
- name : undefined,
- group : undefined,
- properties : {}
+ name: undefined,
+ group: undefined,
+ properties: {},
+ dependencies: []
};
emptyNewProperty($scope.newInterpreterSetting);
};
@@ -205,6 +229,21 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope,
}
};
+ $scope.removeInterpreterDependency = function(artifact, settingId) {
+ if (settingId === undefined) {
+ $scope.newInterpreterSetting.dependencies = _.reject($scope.newInterpreterSetting.dependencies,
+ function(el) {
+ return el.groupArtifactVersion === artifact;
+ });
+ } else {
+ var index = _.findIndex($scope.interpreterSettings, {'id': settingId});
+ $scope.interpreterSettings[index].dependencies = _.reject($scope.interpreterSettings[index].dependencies,
+ function(el) {
+ return el.groupArtifactVersion === artifact;
+ });
+ }
+ };
+
$scope.addNewInterpreterProperty = function(settingId) {
if(settingId === undefined) {
// Add new property from create form
@@ -230,10 +269,126 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope,
}
};
+ $scope.addNewInterpreterDependency = function(settingId) {
+ if(settingId === undefined) {
+ // Add new dependency from create form
+ if (!$scope.newInterpreterSetting.depArtifact || $scope.newInterpreterSetting.depArtifact === '') {
+ return;
+ }
+
+ // overwrite if artifact already exists
+ var newSetting = $scope.newInterpreterSetting;
+ for(var d in newSetting.dependencies) {
+ if (newSetting.dependencies[d].groupArtifactVersion === newSetting.depArtifact) {
+ newSetting.dependencies[d] = {
+ 'groupArtifactVersion': newSetting.depArtifact,
+ 'exclusions': newSetting.depExclude
+ };
+ newSetting.dependencies.splice(d, 1);
+ }
+ }
+
+ newSetting.dependencies.push({
+ 'groupArtifactVersion': newSetting.depArtifact,
+ 'exclusions': (newSetting.depExclude === '')? []: newSetting.depExclude
+ });
+ emptyNewDependency(newSetting);
+ }
+ else {
+ // Add new dependency from edit form
+ var index = _.findIndex($scope.interpreterSettings, { 'id': settingId });
+ var setting = $scope.interpreterSettings[index];
+ if (!setting.depArtifact || setting.depArtifact === '') {
+ return;
+ }
+
+ // overwrite if artifact already exists
+ for(var dep in setting.dependencies) {
+ if (setting.dependencies[dep].groupArtifactVersion === setting.depArtifact) {
+ setting.dependencies[dep] = {
+ 'groupArtifactVersion': setting.depArtifact,
+ 'exclusions': setting.depExclude
+ };
+ setting.dependencies.splice(dep, 1);
+ }
+ }
+
+ setting.dependencies.push({
+ 'groupArtifactVersion': setting.depArtifact,
+ 'exclusions': (setting.depExclude === '')? []: setting.depExclude
+ });
+ emptyNewDependency(setting);
+ }
+ };
+
+ $scope.resetNewRepositorySetting = function() {
+ $scope.newRepoSetting = {
+ id: undefined,
+ url: undefined,
+ snapshot: false,
+ username: undefined,
+ password: undefined
+ };
+ };
+
+ var getRepositories = function() {
+ $http.get(baseUrlSrv.getRestApiBase() + '/interpreter/repository').
+ success(function(data, status, headers, config) {
+ $scope.repositories = data.body;
+ }).
+ error(function(data, status, headers, config) {
+ console.log('Error %o %o', status, data.message);
+ });
+ };
+
+ $scope.addNewRepository = function() {
+ var request = angular.copy($scope.newRepoSetting);
+
+ $http.post(baseUrlSrv.getRestApiBase() + '/interpreter/repository', request).
+ success(function(data, status, headers, config) {
+ getRepositories();
+ $scope.resetNewRepositorySetting();
+ angular.element('#repoModal').modal('hide');
+ }).
+ error(function(data, status, headers, config) {
+ console.log('Error %o %o', headers, config);
+ });
+ };
+
+ $scope.removeRepository = function(repoId) {
+ BootstrapDialog.confirm({
+ closable: true,
+ title: '',
+ message: 'Do you want to delete this repository?',
+ callback: function(result) {
+ if (result) {
+ $http.delete(baseUrlSrv.getRestApiBase()+'/interpreter/repository/' + repoId).
+ success(function(data, status, headers, config) {
+ var index = _.findIndex($scope.repositories, { 'id': repoId });
+ $scope.repositories.splice(index, 1);
+ }).
+ error(function(data, status, headers, config) {
+ console.log('Error %o %o', status, data.message);
+ });
+ }
+ }
+ });
+ };
+
+ $scope.isDefaultRepository = function(repoId) {
+ if (repoId === 'central' || repoId === 'local') {
+ return true;
+ } else {
+ return false;
+ }
+ };
+
var init = function() {
$scope.resetNewInterpreterSetting();
+ $scope.resetNewRepositorySetting();
getInterpreterSettings();
getAvailableInterpreters();
+ getRepositories();
};
init();
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-web/src/app/interpreter/interpreter.css
----------------------------------------------------------------------
diff --git a/zeppelin-web/src/app/interpreter/interpreter.css b/zeppelin-web/src/app/interpreter/interpreter.css
index b6fe133..8695f94 100644
--- a/zeppelin-web/src/app/interpreter/interpreter.css
+++ b/zeppelin-web/src/app/interpreter/interpreter.css
@@ -34,6 +34,13 @@
font-size: 12px;
}
+.interpreter input {
+ width: 100%;
+ display: block;
+ height: 23px;
+ border: 1px solid #CCCCCC;
+}
+
.interpreter .interpreter-title {
font-size: 20px;
font-weight: bold;
@@ -81,6 +88,11 @@
float: left;
}
-.empty-properties-message {
+.gray40-message {
color: #666;
}
+
+.blackOpc:hover {
+ color: #000;
+ opacity: .5;
+}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-web/src/app/interpreter/interpreter.html
----------------------------------------------------------------------
diff --git a/zeppelin-web/src/app/interpreter/interpreter.html b/zeppelin-web/src/app/interpreter/interpreter.html
index 57984da..f1f9a02 100644
--- a/zeppelin-web/src/app/interpreter/interpreter.html
+++ b/zeppelin-web/src/app/interpreter/interpreter.html
@@ -18,11 +18,18 @@ limitations under the License.
<h3 class="new_h3">
Interpreters
</h3>
- <span class="btn btn-default fa fa-plus"
- ng-click="showAddNewSetting = !showAddNewSetting"
- style="float:right;margin-top:10px;">
- Create
- </span>
+ <div class="pull-right" style="margin-top:10px;">
+ <span class="btn btn-default fa fa-plus"
+ ng-click="showAddNewSetting = !showAddNewSetting"
+ style="margin-right:6px;">
+ Create
+ </span>
+ <span style="cursor:pointer;margin-right:4px;"
+ ng-click="showRepositoryInfo = !showRepositoryInfo"
+ tooltip-placement="bottom" tooltip="Repository information">
+ <i class="fa fa-cog" ng-style="{color: showRepositoryInfo ? '#3071A9' : 'black' }"></i>
+ </span>
+ </div>
</div>
</div>
<div class="row">
@@ -31,6 +38,42 @@ limitations under the License.
</div>
</div>
</div>
+
+ <div class="row" ng-if="showRepositoryInfo">
+ <div class="col-md-12">
+ <hr />
+ <h4>Repositories</h4>
+ <p>Available repository lists. These repositories are used to resolve external dependencies of interpreter.</p>
+ <ul class="noDot">
+ <li class="liVertical" ng-repeat="repo in repositories">
+ <a tabindex="0" class="btn btn-info" role="button"
+ popover-trigger="focus"
+ popover-placement="right"
+ popover-html-unsafe="
+ <label>URL: </label>
+ {{repo.url}}<br>
+ <label>Snapshot: </label>
+ {{repo.snapshotPolicy.enabled}}<br>
+ <label>Username: </label>
+ {{repo.authentication.username}}">
+ <span class="fa fa-database"></span>
+ {{repo.id}}
+ <span ng-if="!isDefaultRepository(repo.id)"
+ class="fa fa-close blackOpc" ng-click="removeRepository(repo.id)"></span>
+ </a>
+ </li>
+ <li class="liVertical">
+ <div ng-include src="'components/repository-create/repository-dialog.html'"></div>
+ <div class="btn btn-default"
+ data-toggle="modal"
+ data-target="#repoModal">
+ <span class="fa fa-plus"></span>
+ </div>
+ </li>
+ </ul>
+ </div>
+ </div>
+
<div ng-include src="'app/interpreter/interpreter-create/interpreter-create.html'"></div>
</div>
@@ -66,8 +109,8 @@ limitations under the License.
</div>
</div>
<div class="row interpreter">
- <div ng-show="_.isEmpty(setting.properties) || valueform.$hidden" class="col-md-12 empty-properties-message">
- <em>Currently there are no properties set for this interpreter</em>
+ <div ng-show="_.isEmpty(setting.properties) && _.isEmpty(setting.dependencies) || valueform.$hidden" class="col-md-12 gray40-message">
+ <em>Currently there are no properties and dependencies set for this interpreter</em>
</div>
<div class="col-md-12" ng-show="!_.isEmpty(setting.properties) || valueform.$visible">
<h5>Properties</h5>
@@ -96,8 +139,7 @@ limitations under the License.
<td>
<input ng-model="setting.propertyKey"
pu-elastic-input
- pu-elastic-input-minwidth="180px">
- </input>
+ pu-elastic-input-minwidth="180px" />
</td>
<td>
<textarea msd-elastic ng-model="setting.propertyValue"></textarea>
@@ -109,9 +151,66 @@ limitations under the License.
</td>
</tr>
</table>
- <form editable-form name="valueform" onaftersave="updateInterpreterSetting(setting.id)" ng-show="valueform.$visible">
- <button type="submit" class="btn btn-primary"
- ng-disabled="valueform.$waiting">
+ </div>
+
+ <div class="col-md-12" ng-show="!_.isEmpty(setting.dependencies) || valueform.$visible">
+ <h5>Dependencies</h5>
+ <p class="gray40-message" style="font-size:12px" ng-show="valueform.$visible">
+ These dependencies will be added to classpath when interpreter process starts.</p>
+ <table class="table table-striped">
+ <thead>
+ <tr>
+ <th style="width:40%">artifact</th>
+ <th>exclude</th>
+ <th ng-if="valueform.$visible">action</th>
+ </tr>
+ </thead>
+ <tr ng-repeat="dep in setting.dependencies">
+ <td>
+ <span editable-text="dep.groupArtifactVersion" e-placeholder="groupId:artifactId:version or local file path"
+ e-form="valueform" e-msd-elastic e-style="width:100%">
+ {{dep.groupArtifactVersion | breakFilter}}
+ </span>
+ </td>
+ <td>
+ <textarea ng-if="valueform.$visible" ng-model="dep.exclusions"
+ placeholder="(Optional) comma separated groupId:artifactId list"
+ form="valueform"
+ e-msd-elastic
+ ng-list="">
+ </textarea>
+ <div ng-if="!valueform.$visible">{{dep.exclusions.join()}}</div>
+ </td>
+ <td ng-if="valueform.$visible">
+ <div class="btn btn-default btn-sm fa fa-remove"
+ ng-click="removeInterpreterDependency(dep.groupArtifactVersion, setting.id)">
+ </div>
+ </td>
+ </tr>
+ <tr ng-if="valueform.$visible">
+ <td>
+ <input ng-model="setting.depArtifact"
+ placeholder="groupId:artifactId:version or local file path"
+ style="width: 100%" />
+ </td>
+ <td>
+ <textarea ng-model="setting.depExclude"
+ placeholder="(Optional) comma separated groupId:artifactId list"
+ msd-elastic
+ ng-list="">
+ </textarea>
+ </td>
+ <td>
+ <div class="btn btn-default btn-sm fa fa-plus"
+ ng-click="addNewInterpreterDependency(setting.id)">
+ </div>
+ </td>
+ </tr>
+ </table>
+ <form editable-form name="valueform"
+ onaftersave="updateInterpreterSetting(valueform, setting.id)"
+ ng-show="valueform.$visible">
+ <button type="submit" class="btn btn-primary">
Save
</button>
<button type="button" class="btn btn-default"
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-web/src/components/repository-create/repository-dialog.html
----------------------------------------------------------------------
diff --git a/zeppelin-web/src/components/repository-create/repository-dialog.html b/zeppelin-web/src/components/repository-create/repository-dialog.html
new file mode 100644
index 0000000..98b1469
--- /dev/null
+++ b/zeppelin-web/src/components/repository-create/repository-dialog.html
@@ -0,0 +1,76 @@
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+<div id="repoModal" class="modal fade" role="dialog"
+ tabindex='-1'>
+ <div class="modal-dialog">
+
+ <!-- Modal content-->
+ <div class="modal-content">
+ <div class="modal-header">
+ <button type="button" class="close" data-dismiss="modal">×</button>
+ <h4 class="modal-title">Add New Repository</h4>
+ </div>
+
+ <form class="form-horizontal" ng-submit="addNewRepository()">
+ <div class="modal-body">
+ <div class="form-group">
+ <label class="control-label col-sm-2" for="repoId">ID</label>
+ <div class="col-sm-10">
+ <input type="text" class="form-control" id="repoId" ng-model="newRepoSetting.id"
+ placeholder="Repository id" required>
+ </div>
+ </div>
+ <div class="form-group">
+ <label class="control-label col-sm-2" for="repoUrl">URL</label>
+ <div class="col-sm-10">
+ <input type="text" pattern="(http|https|file)://.*" title="Please enter URL starts with http://, https:// or file://"
+ class="form-control" id="repoUrl" ng-model="newRepoSetting.url"
+ placeholder="Repository url" required>
+ </div>
+ </div>
+ <div class="form-group">
+ <label class="control-label col-sm-2" for="repoSnapshot">Snapshot</label>
+ <div class="col-sm-10">
+ <select class="form-control" id="repoSnapshot" ng-model="newRepoSetting.snapshot">
+ <option ng-selected="true">false</option>
+ <option>true</option>
+ </select>
+ </div>
+ </div>
+ <div class="form-group">
+ <label class="control-label col-sm-2" for="repoUsername">Username</label>
+ <div class="col-sm-10">
+ <input type="text" class="form-control" id="repoUsername" ng-model="newRepoSetting.username">
+ </div>
+ </div>
+ <div class="form-group">
+ <label class="control-label col-sm-2" for="repoPassword">Password</label>
+ <div class="col-sm-10">
+ <input type="password" class="form-control" id="repoPassword" ng-model="newRepoSetting.password">
+ </div>
+ </div>
+ </div>
+ <div class="modal-footer">
+ <button type="submit"
+ class="btn btn-default">Add
+ </button>
+ <button type="button" data-dismiss="modal"
+ class="btn btn-default" ng-click="resetNewRepositorySetting()">Cancel
+ </button>
+ </div>
+ </form>
+ </div>
+ </div>
+</div>
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-web/src/index.html
----------------------------------------------------------------------
diff --git a/zeppelin-web/src/index.html b/zeppelin-web/src/index.html
index a8e082c..1f1166b 100644
--- a/zeppelin-web/src/index.html
+++ b/zeppelin-web/src/index.html
@@ -63,8 +63,8 @@ limitations under the License.
</head>
<body ng-class="{'bodyAsIframe': asIframe}" >
<!--[if lt IE 7]>
-<p class="browsehappy">You are using an <strong>outdated</strong> browser. Please <a href="http://browsehappy.com/">upgrade your browser</a> to improve your experience.</p>
-<![endif]-->
+ <p class="browsehappy">You are using an <strong>outdated</strong> browser. Please <a href="http://browsehappy.com/">upgrade your browser</a> to improve your experience.</p>
+ <![endif]-->
<toast></toast>
<div ng-include src="'components/navbar/navbar.html'"></div>
<!-- Add your site or application content here -->
@@ -81,9 +81,9 @@ limitations under the License.
</div>
<!-- build:js(.) scripts/oldieshim.js -->
<!--[if lt IE 9]>
-<script src="bower_components/es5-shim/es5-shim.js"></script>
-<script src="bower_components/json3/lib/json3.min.js"></script>
-<![endif]-->
+ <script src="bower_components/es5-shim/es5-shim.js"></script>
+ <script src="bower_components/json3/lib/json3.min.js"></script>
+ <![endif]-->
<!-- endbuild -->
<!-- build:js(.) scripts/vendor.js -->
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index 9e606ee..421b3d4 100755
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -346,6 +346,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER);
}
+ public String getInterpreterLocalRepoPath() {
+ return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO);
+ }
+
public String getRelativeDir(ConfVars c) {
return getRelativeDir(getString(c));
}
@@ -456,6 +460,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
+ "org.apache.zeppelin.scalding.ScaldingInterpreter,"
+ "org.apache.zeppelin.jdbc.JDBCInterpreter"),
ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"),
+ ZEPPELIN_INTERPRETER_LOCALREPO("zeppelin.interpreter.localRepo", "local-repo"),
ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000),
ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10),
ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"),
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
index 3cd1257..3761709 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
@@ -19,10 +19,13 @@ package org.apache.zeppelin.interpreter;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
+
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.NullArgumentException;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
@@ -34,6 +37,9 @@ import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Job.Status;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.sonatype.aether.RepositoryException;
+import org.sonatype.aether.repository.Authentication;
+import org.sonatype.aether.repository.RemoteRepository;
import java.io.*;
import java.lang.reflect.Constructor;
@@ -60,6 +66,7 @@ public class InterpreterFactory {
new HashMap<String, InterpreterSetting>();
private Map<String, List<String>> interpreterBindings = new HashMap<String, List<String>>();
+ private List<RemoteRepository> interpreterRepositories;
private Gson gson;
@@ -68,13 +75,13 @@ public class InterpreterFactory {
AngularObjectRegistryListener angularObjectRegistryListener;
private final RemoteInterpreterProcessListener remoteInterpreterProcessListener;
- DependencyResolver depResolver;
+ private DependencyResolver depResolver;
public InterpreterFactory(ZeppelinConfiguration conf,
AngularObjectRegistryListener angularObjectRegistryListener,
RemoteInterpreterProcessListener remoteInterpreterProcessListener,
DependencyResolver depResolver)
- throws InterpreterException, IOException {
+ throws InterpreterException, IOException, RepositoryException {
this(conf, new InterpreterOption(true), angularObjectRegistryListener,
remoteInterpreterProcessListener, depResolver);
}
@@ -84,11 +91,12 @@ public class InterpreterFactory {
AngularObjectRegistryListener angularObjectRegistryListener,
RemoteInterpreterProcessListener remoteInterpreterProcessListener,
DependencyResolver depResolver)
- throws InterpreterException, IOException {
+ throws InterpreterException, IOException, RepositoryException {
this.conf = conf;
this.defaultOption = defaultOption;
this.angularObjectRegistryListener = angularObjectRegistryListener;
this.depResolver = depResolver;
+ this.interpreterRepositories = depResolver.getRepos();
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
String replsConf = conf.getString(ConfVars.ZEPPELIN_INTERPRETERS);
interpreterClassList = replsConf.split(",");
@@ -101,7 +109,7 @@ public class InterpreterFactory {
init();
}
- private void init() throws InterpreterException, IOException {
+ private void init() throws InterpreterException, IOException, RepositoryException {
ClassLoader oldcl = Thread.currentThread().getContextClassLoader();
// Load classes
@@ -172,7 +180,11 @@ public class InterpreterFactory {
if (found) {
// add all interpreters in group
- add(groupName, groupName, defaultOption, p);
+ add(groupName,
+ groupName,
+ new LinkedList<Dependency>(),
+ defaultOption,
+ p);
groupClassNameMap.remove(groupName);
break;
}
@@ -186,7 +198,7 @@ public class InterpreterFactory {
logger.info("Interpreter setting group {} : id={}, name={}",
setting.getGroup(), settingId, setting.getName());
for (Interpreter interpreter : setting.getInterpreterGroup()) {
- logger.info(" className = {}", interpreter.getClassName());
+ logger.info(" className = {}", interpreter.getClassName());
}
}
}
@@ -225,12 +237,11 @@ public class InterpreterFactory {
// previously created setting should turn this feature on here.
setting.getOption().setRemote(true);
-
-
InterpreterSetting intpSetting = new InterpreterSetting(
setting.id(),
setting.getName(),
setting.getGroup(),
+ setting.getDependencies(),
setting.getOption());
InterpreterGroup interpreterGroup = createInterpreterGroup(
@@ -244,8 +255,41 @@ public class InterpreterFactory {
}
this.interpreterBindings = info.interpreterBindings;
+
+ if (info.interpreterRepositories != null) {
+ for (RemoteRepository repo : info.interpreterRepositories) {
+ if (!depResolver.getRepos().contains(repo)) {
+ this.interpreterRepositories.add(repo);
+ }
+ }
+ }
}
+ private void loadInterpreterDependencies(InterpreterSetting intSetting)
+ throws IOException, RepositoryException {
+ // dependencies to prevent library conflict
+ File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + intSetting.id());
+ if (localRepoDir.exists()) {
+ FileUtils.cleanDirectory(localRepoDir);
+ }
+
+ // load dependencies
+ List<Dependency> deps = intSetting.getDependencies();
+ if (deps != null) {
+ for (Dependency d: deps) {
+ if (d.getExclusions() != null) {
+ depResolver.load(
+ d.getGroupArtifactVersion(),
+ d.getExclusions(),
+ conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO) + "/" + intSetting.id());
+ } else {
+ depResolver.load(
+ d.getGroupArtifactVersion(),
+ conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO) + "/" + intSetting.id());
+ }
+ }
+ }
+ }
private void saveToFile() throws IOException {
String jsonString;
@@ -254,6 +298,7 @@ public class InterpreterFactory {
InterpreterInfoSaving info = new InterpreterInfoSaving();
info.interpreterBindings = interpreterBindings;
info.interpreterSettings = interpreterSettings;
+ info.interpreterRepositories = interpreterRepositories;
jsonString = gson.toJson(info);
}
@@ -330,15 +375,21 @@ public class InterpreterFactory {
* @throws IOException
*/
public InterpreterGroup add(String name, String groupName,
+ List<Dependency> dependencies,
InterpreterOption option, Properties properties)
- throws InterpreterException, IOException {
+ throws InterpreterException, IOException, RepositoryException {
synchronized (interpreterSettings) {
InterpreterSetting intpSetting = new InterpreterSetting(
name,
groupName,
+ dependencies,
option);
+ if (dependencies.size() > 0) {
+ loadInterpreterDependencies(intpSetting);
+ }
+
InterpreterGroup interpreterGroup = createInterpreterGroup(
intpSetting.id(), groupName, option, properties);
@@ -354,13 +405,13 @@ public class InterpreterFactory {
String groupName,
InterpreterOption option,
Properties properties)
- throws InterpreterException , NullArgumentException {
+ throws InterpreterException, NullArgumentException {
//When called from REST API without option we receive NPE
- if (option == null )
+ if (option == null)
throw new NullArgumentException("option");
//When called from REST API without option we receive NPE
- if (properties == null )
+ if (properties == null)
throw new NullArgumentException("properties");
AngularObjectRegistry angularObjectRegistry;
@@ -393,7 +444,8 @@ public class InterpreterFactory {
if (option.isRemote()) {
intp = createRemoteRepl(info.getPath(),
info.getClassName(),
- properties);
+ properties,
+ interpreterGroup.id);
} else {
intp = createRepl(info.getPath(),
info.getClassName(),
@@ -428,6 +480,9 @@ public class InterpreterFactory {
saveToFile();
}
}
+
+ File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + id);
+ FileUtils.deleteDirectory(localRepoDir);
}
/**
@@ -511,8 +566,10 @@ public class InterpreterFactory {
* @param properties
* @throws IOException
*/
- public void setPropertyAndRestart(String id, InterpreterOption option,
- Properties properties) throws IOException {
+ public void setPropertyAndRestart(String id,
+ InterpreterOption option,
+ Properties properties,
+ List<Dependency> dependencies) throws IOException, RepositoryException {
synchronized (interpreterSettings) {
InterpreterSetting intpsetting = interpreterSettings.get(id);
if (intpsetting != null) {
@@ -523,11 +580,14 @@ public class InterpreterFactory {
intpsetting.getInterpreterGroup().destroy();
intpsetting.setOption(option);
+ intpsetting.setDependencies(dependencies);
InterpreterGroup interpreterGroup = createInterpreterGroup(
intpsetting.id(),
intpsetting.getGroup(), option, properties);
intpsetting.setInterpreterGroup(interpreterGroup);
+
+ loadInterpreterDependencies(intpsetting);
saveToFile();
} else {
throw new InterpreterException("Interpreter setting id " + id
@@ -661,13 +721,14 @@ public class InterpreterFactory {
private Interpreter createRemoteRepl(String interpreterPath, String className,
- Properties property) {
-
+ Properties property, String interpreterId) {
int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
+ String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + interpreterId;
int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE);
LazyOpenInterpreter intp = new LazyOpenInterpreter(new RemoteInterpreter(
property, className, conf.getInterpreterRemoteRunnerPath(),
- interpreterPath, connectTimeout, maxPoolSize, remoteInterpreterProcessListener));
+ interpreterPath, localRepoPath, connectTimeout,
+ maxPoolSize, remoteInterpreterProcessListener));
return intp;
}
@@ -690,4 +751,19 @@ public class InterpreterFactory {
return new URL[] {path.toURI().toURL()};
}
}
+
+ public List<RemoteRepository> getRepositories() {
+ return this.interpreterRepositories;
+ }
+
+ public void addRepository(String id, String url, boolean snapshot, Authentication auth)
+ throws IOException {
+ depResolver.addRepo(id, url, snapshot, auth);
+ saveToFile();
+ }
+
+ public void removeRepository(String id) throws IOException {
+ depResolver.delRepo(id);
+ saveToFile();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java
index ae507d4..786a723 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java
@@ -17,6 +17,8 @@
package org.apache.zeppelin.interpreter;
+import org.sonatype.aether.repository.RemoteRepository;
+
import java.util.List;
import java.util.Map;
@@ -26,4 +28,5 @@ import java.util.Map;
public class InterpreterInfoSaving {
public Map<String, InterpreterSetting> interpreterSettings;
public Map<String, List<String>> interpreterBindings;
+ public List<RemoteRepository> interpreterRepositories;
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
index 301ed23..e8080a2 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
@@ -17,9 +17,12 @@
package org.apache.zeppelin.interpreter;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Properties;
import java.util.Random;
+import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.notebook.utility.IdHashes;
/**
@@ -32,21 +35,26 @@ public class InterpreterSetting {
private String description;
private Properties properties;
private InterpreterGroup interpreterGroup;
+ private List<Dependency> dependencies;
private InterpreterOption option;
- public InterpreterSetting(String id, String name,
+ public InterpreterSetting(String id,
+ String name,
String group,
+ List<Dependency> dependencies,
InterpreterOption option) {
this.id = id;
this.name = name;
this.group = group;
+ this.dependencies = dependencies;
this.option = option;
}
public InterpreterSetting(String name,
String group,
+ List<Dependency> dependencies,
InterpreterOption option) {
- this(generateId(), name, group, option);
+ this(generateId(), name, group, dependencies, option);
}
public String id() {
@@ -90,6 +98,17 @@ public class InterpreterSetting {
return properties;
}
+ public List<Dependency> getDependencies() {
+ if (dependencies == null) {
+ return new LinkedList<Dependency>();
+ }
+ return dependencies;
+ }
+
+ public void setDependencies(List<Dependency> dependencies) {
+ this.dependencies = dependencies;
+ }
+
public InterpreterOption getOption() {
if (option == null) {
option = new InterpreterOption();
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
index 17d91cc..8fea693 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
@@ -24,17 +24,21 @@ import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
+import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import org.apache.commons.lang.NullArgumentException;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.dep.Dependency;
+import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.sonatype.aether.RepositoryException;
public class InterpreterFactoryTest {
@@ -42,6 +46,7 @@ public class InterpreterFactoryTest {
private File tmpDir;
private ZeppelinConfiguration conf;
private InterpreterContext context;
+ private DependencyResolver depResolver;
@Before
public void setUp() throws Exception {
@@ -55,7 +60,8 @@ public class InterpreterFactoryTest {
System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
System.setProperty(ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1,org.apache.zeppelin.interpreter.mock.MockInterpreter2");
conf = new ZeppelinConfiguration();
- factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null);
+ depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
+ factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver);
context = new InterpreterContext("note", "id", "title", "text", null, null, null, null, null);
}
@@ -98,14 +104,14 @@ public class InterpreterFactoryTest {
}
@Test
- public void testFactoryDefaultList() throws IOException {
+ public void testFactoryDefaultList() throws IOException, RepositoryException {
// get default list from default setting
List<String> all = factory.getDefaultInterpreterSettingList();
assertEquals(2, all.size());
assertEquals(factory.get(all.get(0)).getInterpreterGroup().getFirst().getClassName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1");
// add setting
- factory.add("a mock", "mock2", new InterpreterOption(false), new Properties());
+ factory.add("a mock", "mock2", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties());
all = factory.getDefaultInterpreterSettingList();
assertEquals(2, all.size());
assertEquals("mock1", factory.get(all.get(0)).getName());
@@ -113,16 +119,16 @@ public class InterpreterFactoryTest {
}
@Test
- public void testExceptions() throws IOException {
+ public void testExceptions() throws InterpreterException, IOException, RepositoryException {
List<String> all = factory.getDefaultInterpreterSettingList();
// add setting with null option & properties expected nullArgumentException.class
try {
- factory.add("a mock", "mock2", null, new Properties());
+ factory.add("a mock", "mock2", new LinkedList<Dependency>(), null, new Properties());
} catch(NullArgumentException e) {
assertEquals("Test null option" , e.getMessage(),new NullArgumentException("option").getMessage());
}
try {
- factory.add("a mock" , "mock2" , new InterpreterOption(false),null);
+ factory.add("a mock", "mock2", new LinkedList<Dependency>(), new InterpreterOption(false), null);
} catch (NullArgumentException e){
assertEquals("Test null properties" , e.getMessage(),new NullArgumentException("properties").getMessage());
}
@@ -130,17 +136,17 @@ public class InterpreterFactoryTest {
@Test
- public void testSaveLoad() throws InterpreterException, IOException {
+ public void testSaveLoad() throws IOException, RepositoryException {
// interpreter settings
assertEquals(2, factory.get().size());
// check if file saved
assertTrue(new File(conf.getInterpreterSettingPath()).exists());
- factory.add("newsetting", "mock1", new InterpreterOption(false), new Properties());
+ factory.add("newsetting", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties());
assertEquals(3, factory.get().size());
- InterpreterFactory factory2 = new InterpreterFactory(conf, null, null, null);
+ InterpreterFactory factory2 = new InterpreterFactory(conf, null, null, null, depResolver);
assertEquals(3, factory2.get().size());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java
index 4fa8ef6..04a6ceb 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterOption;
@@ -40,6 +41,7 @@ public class NoteInterpreterLoaderTest {
private File tmpDir;
private ZeppelinConfiguration conf;
private InterpreterFactory factory;
+ private DependencyResolver depResolver;
@Before
public void setUp() throws Exception {
@@ -58,7 +60,8 @@ public class NoteInterpreterLoaderTest {
MockInterpreter11.register("mock11", "group1", "org.apache.zeppelin.interpreter.mock.MockInterpreter11");
MockInterpreter2.register("mock2", "group2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2");
- factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null);
+ depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
+ factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver);
}
@After
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
index 55cd6ba..1988f1c 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
@@ -35,6 +35,7 @@ import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterOption;
@@ -55,6 +56,7 @@ import org.junit.Test;
import org.quartz.SchedulerException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.sonatype.aether.RepositoryException;
public class NotebookTest implements JobListenerFactory{
private static final Logger logger = LoggerFactory.getLogger(NotebookTest.class);
@@ -66,6 +68,7 @@ public class NotebookTest implements JobListenerFactory{
private Notebook notebook;
private NotebookRepo notebookRepo;
private InterpreterFactory factory;
+ private DependencyResolver depResolver;
@Before
public void setUp() throws Exception {
@@ -86,7 +89,8 @@ public class NotebookTest implements JobListenerFactory{
MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1");
MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2");
- factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null);
+ depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
+ factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver);
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
@@ -161,7 +165,7 @@ public class NotebookTest implements JobListenerFactory{
}
@Test
- public void testPersist() throws IOException, SchedulerException{
+ public void testPersist() throws IOException, SchedulerException, RepositoryException {
Note note = notebook.createNote();
// run with default repl
@@ -173,8 +177,8 @@ public class NotebookTest implements JobListenerFactory{
note.persist();
Notebook notebook2 = new Notebook(
- conf, notebookRepo, schedulerFactory, new InterpreterFactory(conf, null, null, null), this,
- null);
+ conf, notebookRepo, schedulerFactory,
+ new InterpreterFactory(conf, null, null, null, depResolver), this, null);
assertEquals(1, notebook2.getAllNotes().size());
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
index 31970af..753fab2 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
@@ -28,6 +28,7 @@ import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterOutput;
@@ -57,6 +58,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
private Notebook notebookSync;
private NotebookRepoSync notebookRepoSync;
private InterpreterFactory factory;
+ private DependencyResolver depResolver;
private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncTest.class);
@Before
@@ -85,7 +87,8 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1");
MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2");
- factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null);
+ depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo");
+ factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver);
SearchService search = mock(SearchService.class);
notebookRepoSync = new NotebookRepoSync(conf);
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
index 2e2801c..0d4ff86 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
@@ -27,6 +27,7 @@ import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
@@ -48,6 +49,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
private Notebook notebook;
private NotebookRepo notebookRepo;
private InterpreterFactory factory;
+ private DependencyResolver depResolver;
private File mainZepDir;
private File mainNotebookDir;
@@ -73,7 +75,8 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1");
this.schedulerFactory = new SchedulerFactory();
- factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null);
+ depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo");
+ factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver);
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
[2/2] incubator-zeppelin git commit: [Zeppelin-630] Introduce new way
of dependency loading to intepreter
Posted by mo...@apache.org.
[Zeppelin-630] Introduce new way of dependency loading to intepreter
### What is this PR for?
With this PR user will be able to set external libraries to be loaded to specific interpreter.
Note that the scope of this PR is downloading libraries to local repository, not distributing them to other nodes. Only spark interpreter distributes loaded dependencies to worker nodes at the moment.
Here is a brief explanation how the code works.
1. get rest api request for interpreter dependency setting from front-end
2. download the libraries in `ZEPPELIN_HOME/local-repo` and copy them to `ZEPPELIN_HOME/local-repo/{interpreterId}`
3. `ZEPPELIN_HOME/local-repo/{interpreterId}/*.jar` are added to interpreter classpath when interpreter process starts
### What type of PR is it?
Improvement
### Todos
* [x] Add tests
* [x] Update docs
### Is there a relevant Jira issue?
https://issues.apache.org/jira/browse/ZEPPELIN-630
And this PR will resolve [ZEPPELIN-194](https://issues.apache.org/jira/browse/ZEPPELIN-194) [ZEPPELIN-381](https://issues.apache.org/jira/browse/ZEPPELIN-381) [ZEPPELIN-609](https://issues.apache.org/jira/browse/ZEPPELIN-609)
### How should this be tested?
1. Add repository(in interpreter menu, click gear button placed top right side)
```
id: spark-packages
url: http://dl.bintray.com/spark-packages/maven
snapshot: false
```
2. Set dependency in spark interpreter(click edit button of spark interpreter setting)
```
artifact: com.databricks:spark-csv_2.10:1.3.0
```
3. Download example csv file
```
$ wget https://github.com/databricks/spark-csv/raw/master/src/test/resources/cars.csv
```
4. run below code in paragraph
```
val df = sqlContext.read
.format("com.databricks.spark.csv")
.option("header", "true") // Use first line of all files as header
.option("inferSchema", "true") // Automatically infer data types
.load("file:///your/download/path/cars.csv")
df.registerTempTable("cars")
```
```
%sql select * from cars
```
### Screenshots (if appropriate)
* Toggle repository list
<img width="1146" alt="screen shot 2016-01-25 at 12 24 44 pm" src="https://cloud.githubusercontent.com/assets/8503346/12563475/52f060ac-c35f-11e5-8621-d8eb97b4d6a1.png">
* Add new repository
<img width="1146" alt="screen shot 2016-01-25 at 12 25 23 pm" src="https://cloud.githubusercontent.com/assets/8503346/12563472/52eb545e-c35f-11e5-9050-a5306d2765f1.png">
* Show repository info
<img width="1146" alt="screen shot 2016-01-25 at 12 25 28 pm" src="https://cloud.githubusercontent.com/assets/8503346/12563473/52ebab84-c35f-11e5-9acb-3a356c855dc7.png">
* Interpreter dependency
<img width="1146" alt="screen shot 2016-01-25 at 12 27 27 pm" src="https://cloud.githubusercontent.com/assets/8503346/12563471/52eadd9e-c35f-11e5-8e1a-f583ea8800aa.png">
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions?
- For the users who use rest api for creat/update interpreter setting, `dependencies` object should be added to request payload.
- %dep interpreter is deprecated. The functionality is still there, but recommend to load third party dependency via interpreter menu.
* Does this needs documentation? Yes
Author: Mina Lee <mi...@nflabs.com>
Closes #673 from minahlee/ZEPPELIN-630 and squashes the following commits:
62a75c9 [Mina Lee] Merge branch 'master' of https://github.com/apache/incubator-zeppelin into ZEPPELIN-630
545c173 [Mina Lee] Change variable name LOCAL_REPO_DIR -> LOCAL_INTERPRETER_REPO
1e3dd47 [Mina Lee] Fix docs indentation
320f400 [Mina Lee] Add documentation
6b90c3d [Mina Lee] Fix mislocated interpreter setting save/cancel button
e161b98 [Mina Lee] Add tests and split ZeppelinRestApiTest into two files
387e21e [Mina Lee] Close input tag
ee7532b [Mina Lee] Combine catch block for readability
eb4a78f [Mina Lee] Handle url with file protocol for repository URL input field
bae0c02 [Mina Lee] * Fix DependencyResolver addRepo/delRepo method * Manage repository information in `conf/interpreter.json` * Front-end modification to manage repository list * Add RestApi for adding/deleting repository * Fix tests
fe9cb92 [Mina Lee] Enable adding interpreter dependency via GUI
d5c931b [Mina Lee] Fix test after rebase
1b6a818 [Mina Lee] Remove test with unused ZeppelinContext load() method
37005c5 [Mina Lee] Remove unused methods and add deprecated message for dep interpreter
2cd715c [Mina Lee] Add env variable/property to configuration template files
848d931 [Mina Lee] Make external libraries to be added to interpreter process classpath
Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/218a3b5b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/218a3b5b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/218a3b5b
Branch: refs/heads/master
Commit: 218a3b5bca1dcbc3746b653a6db568bebf40720e
Parents: e87f5b1
Author: Mina Lee <mi...@nflabs.com>
Authored: Sat Jan 30 08:55:58 2016 -0800
Committer: Lee moon soo <mo...@apache.org>
Committed: Mon Feb 1 11:10:43 2016 +0900
----------------------------------------------------------------------
bin/interpreter.sh | 9 +-
conf/zeppelin-env.sh.template | 5 +-
conf/zeppelin-site.xml.template | 6 +
docs/_includes/themes/zeppelin/_navigation.html | 4 +-
docs/_includes/themes/zeppelin/default.html | 4 +-
.../img/docs-img/interpreter-add-repo1.png | Bin 0 -> 305124 bytes
.../img/docs-img/interpreter-add-repo2.png | Bin 0 -> 336352 bytes
.../docs-img/interpreter-dependency-loading.png | Bin 0 -> 378169 bytes
docs/interpreter/spark.md | 96 +++----
docs/manual/dependencymanagement.md | 74 ++++++
docs/rest-api/rest-interpreter.md | 104 +++++++-
.../zeppelin/spark/PySparkInterpreter.java | 24 +-
.../apache/zeppelin/spark/SparkInterpreter.java | 38 ++-
.../apache/zeppelin/spark/ZeppelinContext.java | 122 ---------
.../spark/dep/SparkDependencyContext.java | 10 +-
.../zeppelin/spark/SparkInterpreterTest.java | 10 -
.../dep/AbstractDependencyResolver.java | 26 +-
.../apache/zeppelin/dep/DependencyContext.java | 2 +-
.../apache/zeppelin/dep/DependencyResolver.java | 17 +-
.../org/apache/zeppelin/dep/Repository.java | 10 +-
.../zeppelin/interpreter/Interpreter.java | 4 -
.../interpreter/remote/RemoteInterpreter.java | 36 +--
.../remote/RemoteInterpreterProcess.java | 22 +-
.../zeppelin/dep/DependencyResolverTest.java | 50 +++-
.../remote/RemoteAngularObjectTest.java | 17 +-
.../RemoteInterpreterOutputTestStream.java | 16 +-
.../remote/RemoteInterpreterProcessTest.java | 4 +-
.../remote/RemoteInterpreterTest.java | 34 +--
.../zeppelin/scheduler/RemoteSchedulerTest.java | 32 +--
.../zeppelin/rest/InterpreterRestApi.java | 105 +++++++-
.../message/NewInterpreterSettingRequest.java | 7 +-
.../UpdateInterpreterSettingRequest.java | 17 +-
.../apache/zeppelin/server/ZeppelinServer.java | 3 +-
.../java/org/apache/zeppelin/ZeppelinIT.java | 50 +++-
.../zeppelin/rest/AbstractTestRestApi.java | 15 +-
.../zeppelin/rest/InterpreterRestApiTest.java | 206 +++++++++++++++
.../zeppelin/rest/ZeppelinRestApiTest.java | 120 +--------
zeppelin-web/src/app/app.js | 3 +-
zeppelin-web/src/app/home/home.css | 25 --
.../interpreter-create/interpreter-create.html | 52 +++-
.../app/interpreter/interpreter.controller.js | 259 +++++++++++++++----
.../src/app/interpreter/interpreter.css | 14 +-
.../src/app/interpreter/interpreter.html | 123 ++++++++-
.../repository-create/repository-dialog.html | 76 ++++++
zeppelin-web/src/index.html | 10 +-
.../zeppelin/conf/ZeppelinConfiguration.java | 5 +
.../interpreter/InterpreterFactory.java | 112 ++++++--
.../interpreter/InterpreterInfoSaving.java | 3 +
.../interpreter/InterpreterSetting.java | 23 +-
.../interpreter/InterpreterFactoryTest.java | 24 +-
.../notebook/NoteInterpreterLoaderTest.java | 5 +-
.../apache/zeppelin/notebook/NotebookTest.java | 12 +-
.../notebook/repo/NotebookRepoSyncTest.java | 5 +-
.../notebook/repo/VFSNotebookRepoTest.java | 5 +-
54 files changed, 1482 insertions(+), 573 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/bin/interpreter.sh
----------------------------------------------------------------------
diff --git a/bin/interpreter.sh b/bin/interpreter.sh
index b5603c8..b5a6d12 100755
--- a/bin/interpreter.sh
+++ b/bin/interpreter.sh
@@ -21,10 +21,10 @@ bin=$(cd "${bin}">/dev/null; pwd)
function usage() {
- echo "usage) $0 -p <port> -d <directory to load>"
+ echo "usage) $0 -p <port> -d <interpreter dir to load> -l <local interpreter repo dir to load>"
}
-while getopts "hp:d:" o; do
+while getopts "hp:d:l:" o; do
case ${o} in
h)
usage
@@ -36,6 +36,9 @@ while getopts "hp:d:" o; do
p)
PORT=${OPTARG}
;;
+ l)
+ LOCAL_INTERPRETER_REPO=${OPTARG}
+ ;;
esac
done
@@ -128,6 +131,8 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
fi
fi
+addJarInDir "${LOCAL_INTERPRETER_REPO}"
+
CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
if [[ -n "${SPARK_SUBMIT}" ]]; then
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/conf/zeppelin-env.sh.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template
index b0b1a5b..2fa5311 100644
--- a/conf/zeppelin-env.sh.template
+++ b/conf/zeppelin-env.sh.template
@@ -29,10 +29,11 @@
# export ZEPPELIN_NOTEBOOK_DIR # Where notebook saved
# export ZEPPELIN_NOTEBOOK_HOMESCREEN # Id of notebook to be displayed in homescreen. ex) 2A94M5J1Z
# export ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE # hide homescreen notebook from list when this value set to "true". default "false"
-# export ZEPPELIN_NOTEBOOK_S3_BUCKET # Bucket where notebook saved
-# export ZEPPELIN_NOTEBOOK_S3_USER # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json
+# export ZEPPELIN_NOTEBOOK_S3_BUCKET # Bucket where notebook saved
+# export ZEPPELIN_NOTEBOOK_S3_USER # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json
# export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default.
# export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0.
+# export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading
#### Spark interpreter configuration ####
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/conf/zeppelin-site.xml.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template
index 9ca740d..1d085ec 100755
--- a/conf/zeppelin-site.xml.template
+++ b/conf/zeppelin-site.xml.template
@@ -104,6 +104,12 @@
</property>
<property>
+ <name>zeppelin.interpreter.localRepo</name>
+ <value>local-repo</value>
+ <description>Local repository for interpreter's additional dependency loading</description>
+</property>
+
+<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.tachyon.TachyonIn
terpreter</value>
<description>Comma separated interpreter configurations. First interpreter become a default</description>
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/docs/_includes/themes/zeppelin/_navigation.html
----------------------------------------------------------------------
diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html
index d0581b1..7a73365 100644
--- a/docs/_includes/themes/zeppelin/_navigation.html
+++ b/docs/_includes/themes/zeppelin/_navigation.html
@@ -37,7 +37,6 @@
<a href="#" data-toggle="dropdown" class="dropdown-toggle">Interpreter <b class="caret"></b></a>
<ul class="dropdown-menu">
<li><a href="{{BASE_PATH}}/manual/interpreters.html">Overview</a></li>
- <li><a href="{{BASE_PATH}}/manual/dynamicinterpreterload.html">Dynamic Interpreter Loading</a></li>
<li role="separator" class="divider"></li>
<li><a href="{{BASE_PATH}}/interpreter/cassandra.html">Cassandra</a></li>
<li><a href="{{BASE_PATH}}/interpreter/elasticsearch.html">Elasticsearch</a></li>
@@ -53,6 +52,9 @@
<li><a href="{{BASE_PATH}}/interpreter/spark.html">Spark</a></li>
<li><a href="{{BASE_PATH}}/interpreter/tachyon.html">Tachyon</a></li>
<li><a href="{{BASE_PATH}}/pleasecontribute.html">Tajo</a></li>
+ <li role="separator" class="divider"></li>
+ <li><a href="{{BASE_PATH}}/manual/dynamicinterpreterload.html">Dynamic Interpreter Loading</a></li>
+ <li><a href="{{BASE_PATH}}/manual/dependencymanagement.html">Interpreter Dependency Management</a></li>
</ul>
</li>
<li>
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/docs/_includes/themes/zeppelin/default.html
----------------------------------------------------------------------
diff --git a/docs/_includes/themes/zeppelin/default.html b/docs/_includes/themes/zeppelin/default.html
index 3940414..9f1d6d5 100644
--- a/docs/_includes/themes/zeppelin/default.html
+++ b/docs/_includes/themes/zeppelin/default.html
@@ -11,9 +11,11 @@
<!-- Le HTML5 shim, for IE6-8 support of HTML elements -->
<!--[if lt IE 9]>
- <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
+ <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
<![endif]-->
+ <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css" rel="stylesheet">
+
<!-- Le styles -->
<link href="{{ ASSET_PATH }}/bootstrap/css/bootstrap.css" rel="stylesheet">
<link href="{{ ASSET_PATH }}/css/style.css?body=1" rel="stylesheet" type="text/css">
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/docs/assets/themes/zeppelin/img/docs-img/interpreter-add-repo1.png
----------------------------------------------------------------------
diff --git a/docs/assets/themes/zeppelin/img/docs-img/interpreter-add-repo1.png b/docs/assets/themes/zeppelin/img/docs-img/interpreter-add-repo1.png
new file mode 100644
index 0000000..d992e23
Binary files /dev/null and b/docs/assets/themes/zeppelin/img/docs-img/interpreter-add-repo1.png differ
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/docs/assets/themes/zeppelin/img/docs-img/interpreter-add-repo2.png
----------------------------------------------------------------------
diff --git a/docs/assets/themes/zeppelin/img/docs-img/interpreter-add-repo2.png b/docs/assets/themes/zeppelin/img/docs-img/interpreter-add-repo2.png
new file mode 100644
index 0000000..2f117c1
Binary files /dev/null and b/docs/assets/themes/zeppelin/img/docs-img/interpreter-add-repo2.png differ
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/docs/assets/themes/zeppelin/img/docs-img/interpreter-dependency-loading.png
----------------------------------------------------------------------
diff --git a/docs/assets/themes/zeppelin/img/docs-img/interpreter-dependency-loading.png b/docs/assets/themes/zeppelin/img/docs-img/interpreter-dependency-loading.png
new file mode 100644
index 0000000..749d391
Binary files /dev/null and b/docs/assets/themes/zeppelin/img/docs-img/interpreter-dependency-loading.png differ
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/docs/interpreter/spark.md
----------------------------------------------------------------------
diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md
index 6141619..83eb355 100644
--- a/docs/interpreter/spark.md
+++ b/docs/interpreter/spark.md
@@ -80,9 +80,58 @@ SparkContext, SQLContext, ZeppelinContext are automatically created and exposed
<a name="dependencyloading"> </a>
## Dependency Management
-There are two ways to load external library in spark interpreter. First is using Zeppelin's `%dep` interpreter and second is loading Spark properties.
+There are two ways to load external library in spark interpreter. First is using Interpreter setting menu and second is loading Spark properties.
+
+### 1. Setting Dependencies via Interpreter Setting
+Please see [Dependency Management](../manual/dependencymanagement.html) for the details.
+
+### 2. Loading Spark Properties
+Once `SPARK_HOME` is set in `conf/zeppelin-env.sh`, Zeppelin uses `spark-submit` as spark interpreter runner. `spark-submit` supports two ways to load configurations. The first is command line options such as --master and Zeppelin can pass these options to `spark-submit` by exporting `SPARK_SUBMIT_OPTIONS` in conf/zeppelin-env.sh. Second is reading configuration options from `SPARK_HOME/conf/spark-defaults.conf`. Spark properites that user can set to distribute libraries are:
+
+<table class="table-configuration">
+ <tr>
+ <th>spark-defaults.conf</th>
+ <th>SPARK_SUBMIT_OPTIONS</th>
+ <th>Applicable Interpreter</th>
+ <th>Description</th>
+ </tr>
+ <tr>
+ <td>spark.jars</td>
+ <td>--jars</td>
+ <td>%spark</td>
+ <td>Comma-separated list of local jars to include on the driver and executor classpaths.</td>
+ </tr>
+ <tr>
+ <td>spark.jars.packages</td>
+ <td>--packages</td>
+ <td>%spark</td>
+ <td>Comma-separated list of maven coordinates of jars to include on the driver and executor classpaths. Will search the local maven repo, then maven central and any additional remote repositories given by --repositories. The format for the coordinates should be groupId:artifactId:version.</td>
+ </tr>
+ <tr>
+ <td>spark.files</td>
+ <td>--files</td>
+ <td>%pyspark</td>
+ <td>Comma-separated list of files to be placed in the working directory of each executor.</td>
+ </tr>
+</table>
+> Note that adding jar to pyspark is only availabe via `%dep` interpreter at the moment.
+
+Here are few examples:
+
+* SPARK\_SUBMIT\_OPTIONS in conf/zeppelin-env.sh
+
+ export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0 --jars /path/mylib1.jar,/path/mylib2.jar --files /path/mylib1.py,/path/mylib2.zip,/path/mylib3.egg"
+
+* SPARK_HOME/conf/spark-defaults.conf
+
+ spark.jars /path/mylib1.jar,/path/mylib2.jar
+ spark.jars.packages com.databricks:spark-csv_2.10:1.2.0
+ spark.files /path/mylib1.py,/path/mylib2.egg,/path/mylib3.zip
+
+### 3. Dynamic Dependency Loading via %dep interpreter
+> Note: `%dep` interpreter is deprecated since v0.6.0-incubating.
+`%dep` interpreter load libraries to `%spark` and `%pyspark` but not to `%spark.sql` interpreter so we recommend you to use first option instead.
-### 1. Dynamic Dependency Loading via %dep interpreter
When your code requires external library, instead of doing download/copy/restart Zeppelin, you can easily do following jobs using `%dep` interpreter.
* Load libraries recursively from Maven repository
@@ -129,49 +178,6 @@ z.load("groupId:artifactId:version").exclude("groupId:*")
z.load("groupId:artifactId:version").local()
```
-### 2. Loading Spark Properties
-Once `SPARK_HOME` is set in `conf/zeppelin-env.sh`, Zeppelin uses `spark-submit` as spark interpreter runner. `spark-submit` supports two ways to load configurations. The first is command line options such as --master and Zeppelin can pass these options to `spark-submit` by exporting `SPARK_SUBMIT_OPTIONS` in conf/zeppelin-env.sh. Second is reading configuration options from `SPARK_HOME/conf/spark-defaults.conf`. Spark properites that user can set to distribute libraries are:
-
-<table class="table-configuration">
- <tr>
- <th>spark-defaults.conf</th>
- <th>SPARK_SUBMIT_OPTIONS</th>
- <th>Applicable Interpreter</th>
- <th>Description</th>
- </tr>
- <tr>
- <td>spark.jars</td>
- <td>--jars</td>
- <td>%spark</td>
- <td>Comma-separated list of local jars to include on the driver and executor classpaths.</td>
- </tr>
- <tr>
- <td>spark.jars.packages</td>
- <td>--packages</td>
- <td>%spark</td>
- <td>Comma-separated list of maven coordinates of jars to include on the driver and executor classpaths. Will search the local maven repo, then maven central and any additional remote repositories given by --repositories. The format for the coordinates should be groupId:artifactId:version.</td>
- </tr>
- <tr>
- <td>spark.files</td>
- <td>--files</td>
- <td>%pyspark</td>
- <td>Comma-separated list of files to be placed in the working directory of each executor.</td>
- </tr>
-</table>
-> Note that adding jar to pyspark is only availabe via `%dep` interpreter at the moment.
-
-Here are few examples:
-
-* SPARK\_SUBMIT\_OPTIONS in conf/zeppelin-env.sh
-
- export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0 --jars /path/mylib1.jar,/path/mylib2.jar --files /path/mylib1.py,/path/mylib2.zip,/path/mylib3.egg"
-
-* SPARK_HOME/conf/spark-defaults.conf
-
- spark.jars /path/mylib1.jar,/path/mylib2.jar
- spark.jars.packages com.databricks:spark-csv_2.10:1.2.0
- spark.files /path/mylib1.py,/path/mylib2.egg,/path/mylib3.zip
-
## ZeppelinContext
Zeppelin automatically injects ZeppelinContext as variable 'z' in your scala/python environment. ZeppelinContext provides some additional functions and utility.
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/docs/manual/dependencymanagement.md
----------------------------------------------------------------------
diff --git a/docs/manual/dependencymanagement.md b/docs/manual/dependencymanagement.md
new file mode 100644
index 0000000..612901e
--- /dev/null
+++ b/docs/manual/dependencymanagement.md
@@ -0,0 +1,74 @@
+---
+layout: page
+title: "Dependnecy Management"
+description: ""
+group: manual
+---
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+{% include JB/setup %}
+
+## Dependency Management for Interpreter
+
+You can include external libraries to interpreter by setting dependencies in interpreter menu.
+
+When your code requires external library, instead of doing download/copy/restart Zeppelin, you can easily do following jobs in this menu.
+
+ * Load libraries recursively from Maven repository
+ * Load libraries from local filesystem
+ * Add additional maven repository
+ * Automatically add libraries to SparkCluster
+
+<hr>
+<div class="row">
+ <div class="col-md-6">
+ <a data-lightbox="compiler" href="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/interpreter-dependency-loading.png">
+ <img class="img-responsive" src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/interpreter-dependency-loading.png" />
+ </a>
+ </div>
+ <div class="col-md-6" style="padding-top:30px">
+ <b> Load Dependencies to Interpreter </b>
+ <br /><br />
+ <ol>
+ <li> Click 'Interpreter' menu in navigation bar. </li>
+ <li> Click 'edit' button of the interpreter which you want to load dependencies to. </li>
+ <li> Fill artifact and exclude field to your needs.
+ You can enter not only groupId:artifactId:version but also local file in artifact field. </li>
+ <li> Press 'Save' to restart the interpreter with loaded libraries. </li>
+ </ol>
+ </div>
+</div>
+<hr>
+<div class="row">
+ <div class="col-md-6">
+ <a data-lightbox="compiler" href="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/interpreter-add-repo1.png">
+ <img class="img-responsive" src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/interpreter-add-repo1.png" />
+ </a>
+ <a data-lightbox="compiler" href="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/interpreter-add-repo2.png">
+ <img class="img-responsive" src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/interpreter-add-repo2.png" />
+ </a>
+ </div>
+ <div class="col-md-6" style="padding-top:30px">
+ <b> Add repository for dependency resolving </b>
+ <br /><br />
+ <ol>
+ <li> Press <i class="fa fa-cog"></i> icon in 'Interpreter' menu on the top right side.
+ It will show you available repository lists.</li>
+ <li> If you need to resolve dependencies from other than central maven repository or
+ local ~/.m2 repository, hit <i class="fa fa-plus"></i> icon next to repository lists. </li>
+ <li> Fill out the form and click 'Add' button, then you will be able to see that new repository is added. </li>
+ </ol>
+ </div>
+</div>
+
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/docs/rest-api/rest-interpreter.md
----------------------------------------------------------------------
diff --git a/docs/rest-api/rest-interpreter.md b/docs/rest-api/rest-interpreter.md
index 0db8470..a2053eb 100644
--- a/docs/rest-api/rest-interpreter.md
+++ b/docs/rest-api/rest-interpreter.md
@@ -151,7 +151,8 @@ limitations under the License.
"class": "org.apache.zeppelin.markdown.Markdown",
"name": "md"
}
- ]
+ ],
+ "dependencies": []
},
{
"id": "2AY6GV7Q3",
@@ -170,6 +171,11 @@ limitations under the License.
"class": "org.apache.zeppelin.spark.SparkSqlInterpreter",
"name": "sql"
}
+ ],
+ "dependencies": [
+ {
+ "groupArtifactVersion": "com.databricks:spark-csv_2.10:1.3.0"
+ }
]
}
]
@@ -219,6 +225,12 @@ limitations under the License.
"class": "org.apache.zeppelin.markdown.Markdown",
"name": "md"
}
+ ],
+ "dependencies": [
+ {
+ "groupArtifactVersion": "groupId:artifactId:version",
+ "exclusions": "groupId:artifactId"
+ }
]
}
</pre>
@@ -243,6 +255,12 @@ limitations under the License.
"class": "org.apache.zeppelin.markdown.Markdown",
"name": "md"
}
+ ],
+ "dependencies": [
+ {
+ "groupArtifactVersion": "groupId:artifactId:version",
+ "exclusions": "groupId:artifactId"
+ }
]
}
}
@@ -292,6 +310,12 @@ limitations under the License.
"class": "org.apache.zeppelin.markdown.Markdown",
"name": "md"
}
+ ],
+ "dependencies": [
+ {
+ "groupArtifactVersion": "groupId:artifactId:version",
+ "exclusions": "groupId:artifactId"
+ }
]
}
</pre>
@@ -316,6 +340,12 @@ limitations under the License.
"class": "org.apache.zeppelin.markdown.Markdown",
"name": "md"
}
+ ],
+ "dependencies": [
+ {
+ "groupArtifactVersion": "groupId:artifactId:version",
+ "exclusions": "groupId:artifactId"
+ }
]
}
}
@@ -391,3 +421,75 @@ limitations under the License.
</td>
</tr>
</table>
+
+<br/>
+### 6. Add repository for dependency resolving
+
+ <table class="table-configuration">
+ <col width="200">
+ <tr>
+ <th>Add new repository for dependency loader</th>
+ <th></th>
+ </tr>
+ <tr>
+ <td>Description</td>
+ <td>This ```POST``` method adds new repository.</td>
+ </tr>
+ <tr>
+ <td>URL</td>
+ <td>```http://[zeppelin-server]:[zeppelin-port]/api/interpreter/repository```</td>
+ </tr>
+ <tr>
+ <td>Success code</td>
+ <td>201</td>
+ </tr>
+ <tr>
+ <td>Fail code</td>
+ <td> 500 </td>
+ </tr>
+ <tr>
+ <td>Sample JSON input</td>
+ <td>
+ <pre>
+{
+ "id": "securecentral",
+ "url": "https://repo1.maven.org/maven2",
+ "snapshot": false
+}
+ </pre>
+ </td>
+ </tr>
+ <tr>
+ <td>Sample JSON response</td>
+ <td>
+ <code>{"status":"OK"}</code>
+ </td>
+ </tr>
+ </table>
+
+<br/>
+### 7. Delete repository for dependency resolving
+
+ <table class="table-configuration">
+ <col width="200">
+ <tr>
+ <th>Delete repository for dependency loader</th>
+ <th></th>
+ </tr>
+ <tr>
+ <td>Description</td>
+ <td>This ```DELETE``` method delete repository with given id.</td>
+ </tr>
+ <tr>
+ <td>URL</td>
+ <td>```http://[zeppelin-server]:[zeppelin-port]/api/interpreter/repository/[repository ID]```</td>
+ </tr>
+ <tr>
+ <td>Success code</td>
+ <td>200</td>
+ </tr>
+ <tr>
+ <td>Fail code</td>
+ <td> 500 </td>
+ </tr>
+ </table>
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
index c5441ab..2d4728d 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
@@ -59,9 +59,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonParser;
import py4j.GatewayServer;
@@ -124,12 +121,12 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
// load libraries from Dependency Interpreter
URL [] urls = new URL[0];
+ List<URL> urlList = new LinkedList<URL>();
if (depInterpreter != null) {
SparkDependencyContext depc = depInterpreter.getDependencyContext();
if (depc != null) {
List<File> files = depc.getFiles();
- List<URL> urlList = new LinkedList<URL>();
if (files != null) {
for (File f : files) {
try {
@@ -138,12 +135,29 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
logger.error("Error", e);
}
}
+ }
+ }
+ }
- urls = urlList.toArray(urls);
+ String localRepo = getProperty("zeppelin.interpreter.localRepo");
+ if (localRepo != null) {
+ File localRepoDir = new File(localRepo);
+ if (localRepoDir.exists()) {
+ File[] files = localRepoDir.listFiles();
+ if (files != null) {
+ for (File f : files) {
+ try {
+ urlList.add(f.toURI().toURL());
+ } catch (MalformedURLException e) {
+ logger.error("Error", e);
+ }
+ }
}
}
}
+ urls = urlList.toArray(urls);
+
ClassLoader oldCl = Thread.currentThread().getContextClassLoader();
try {
URLClassLoader newCl = new URLClassLoader(urls, oldCl);
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
index 7ee6d7c..d8e0f81 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
@@ -438,6 +438,23 @@ public class SparkInterpreter extends Interpreter {
}
}
+ // add dependency from local repo
+ String localRepo = getProperty("zeppelin.interpreter.localRepo");
+ if (localRepo != null) {
+ File localRepoDir = new File(localRepo);
+ if (localRepoDir.exists()) {
+ File[] files = localRepoDir.listFiles();
+ if (files != null) {
+ for (File f : files) {
+ if (classpath.length() > 0) {
+ classpath += File.pathSeparator;
+ }
+ classpath += f.getAbsolutePath();
+ }
+ }
+ }
+ }
+
pathSettings.v_$eq(classpath);
settings.scala$tools$nsc$settings$ScalaSettings$_setter_$classpath_$eq(pathSettings);
@@ -529,7 +546,7 @@ public class SparkInterpreter extends Interpreter {
throw new InterpreterException(e);
}
- // add jar
+ // add jar from DepInterpreter
if (depInterpreter != null) {
SparkDependencyContext depc = depInterpreter.getDependencyContext();
if (depc != null) {
@@ -547,6 +564,25 @@ public class SparkInterpreter extends Interpreter {
}
}
}
+
+ // add jar from local repo
+ if (localRepo != null) {
+ File localRepoDir = new File(localRepo);
+ if (localRepoDir.exists()) {
+ File[] files = localRepoDir.listFiles();
+ if (files != null) {
+ for (File f : files) {
+ if (f.getName().toLowerCase().endsWith(".jar")) {
+ sc.addJar(f.getAbsolutePath());
+ logger.info("sc.addJar(" + f.getAbsolutePath() + ")");
+ } else {
+ sc.addFile(f.getAbsolutePath());
+ logger.info("sc.addFile(" + f.getAbsolutePath() + ")");
+ }
+ }
+ }
+ }
+ }
}
private List<File> currentClassPath() {
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java
index 926f3e7..0201188 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java
@@ -33,7 +33,6 @@ import java.util.List;
import org.apache.spark.SparkContext;
import org.apache.spark.sql.SQLContext;
-import org.apache.spark.sql.SQLContext.QueryExecution;
import org.apache.spark.sql.catalyst.expressions.Attribute;
import org.apache.spark.sql.hive.HiveContext;
import org.apache.zeppelin.display.AngularObject;
@@ -74,127 +73,6 @@ public class ZeppelinContext extends HashMap<String, Object> {
public HiveContext hiveContext;
private GUI gui;
- /**
- * Load dependency for interpreter and runtime (driver).
- * And distribute them to spark cluster (sc.add())
- *
- * @param artifact "group:artifact:version" or file path like "/somepath/your.jar"
- * @return
- * @throws Exception
- */
- public Iterable<String> load(String artifact) throws Exception {
- return collectionAsScalaIterable(dep.load(artifact, true));
- }
-
- /**
- * Load dependency and it's transitive dependencies for interpreter and runtime (driver).
- * And distribute them to spark cluster (sc.add())
- *
- * @param artifact "groupId:artifactId:version" or file path like "/somepath/your.jar"
- * @param excludes exclusion list of transitive dependency. list of "groupId:artifactId" string.
- * @return
- * @throws Exception
- */
- public Iterable<String> load(String artifact, scala.collection.Iterable<String> excludes)
- throws Exception {
- return collectionAsScalaIterable(
- dep.load(artifact,
- asJavaCollection(excludes),
- true));
- }
-
- /**
- * Load dependency and it's transitive dependencies for interpreter and runtime (driver).
- * And distribute them to spark cluster (sc.add())
- *
- * @param artifact "groupId:artifactId:version" or file path like "/somepath/your.jar"
- * @param excludes exclusion list of transitive dependency. list of "groupId:artifactId" string.
- * @return
- * @throws Exception
- */
- public Iterable<String> load(String artifact, Collection<String> excludes) throws Exception {
- return collectionAsScalaIterable(dep.load(artifact, excludes, true));
- }
-
- /**
- * Load dependency for interpreter and runtime, and then add to sparkContext.
- * But not adding them to spark cluster
- *
- * @param artifact "groupId:artifactId:version" or file path like "/somepath/your.jar"
- * @return
- * @throws Exception
- */
- public Iterable<String> loadLocal(String artifact) throws Exception {
- return collectionAsScalaIterable(dep.load(artifact, false));
- }
-
-
- /**
- * Load dependency and it's transitive dependencies and then add to sparkContext.
- * But not adding them to spark cluster
- *
- * @param artifact "groupId:artifactId:version" or file path like "/somepath/your.jar"
- * @param excludes exclusion list of transitive dependency. list of "groupId:artifactId" string.
- * @return
- * @throws Exception
- */
- public Iterable<String> loadLocal(String artifact,
- scala.collection.Iterable<String> excludes) throws Exception {
- return collectionAsScalaIterable(dep.load(artifact,
- asJavaCollection(excludes), false));
- }
-
- /**
- * Load dependency and it's transitive dependencies and then add to sparkContext.
- * But not adding them to spark cluster
- *
- * @param artifact "groupId:artifactId:version" or file path like "/somepath/your.jar"
- * @param excludes exclusion list of transitive dependency. list of "groupId:artifactId" string.
- * @return
- * @throws Exception
- */
- public Iterable<String> loadLocal(String artifact, Collection<String> excludes)
- throws Exception {
- return collectionAsScalaIterable(dep.load(artifact, excludes, false));
- }
-
-
- /**
- * Add maven repository
- *
- * @param id id of repository ex) oss, local, snapshot
- * @param url url of repository. supported protocol : file, http, https
- */
- public void addRepo(String id, String url) {
- addRepo(id, url, false);
- }
-
- /**
- * Add maven repository
- *
- * @param id id of repository
- * @param url url of repository. supported protocol : file, http, https
- * @param snapshot true if it is snapshot repository
- */
- public void addRepo(String id, String url, boolean snapshot) {
- dep.addRepo(id, url, snapshot);
- }
-
- /**
- * Remove maven repository by id
- * @param id id of repository
- */
- public void removeRepo(String id){
- dep.delRepo(id);
- }
-
- /**
- * Load dependency only interpreter.
- *
- * @param name
- * @return
- */
-
public Object input(String name) {
return input(name, "");
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java b/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java
index 1b20b0f..7e73c98 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java
@@ -42,6 +42,8 @@ import org.sonatype.aether.util.artifact.JavaScopes;
import org.sonatype.aether.util.filter.DependencyFilterUtils;
import org.sonatype.aether.util.filter.PatternExclusionsDependencyFilter;
+import scala.Console;
+
/**
*
@@ -64,6 +66,8 @@ public class SparkDependencyContext {
}
public Dependency load(String lib) {
+ Console.println("DepInterpreter(%dep) deprecated. "
+ + "Load dependency through GUI interpreter menu instead.");
Dependency dep = new Dependency(lib);
if (dependencies.contains(dep)) {
@@ -74,12 +78,16 @@ public class SparkDependencyContext {
}
public Repository addRepo(String name) {
+ Console.println("DepInterpreter(%dep) deprecated. "
+ + "Add repository through GUI interpreter menu instead.");
Repository rep = new Repository(name);
repositories.add(rep);
return rep;
}
public void reset() {
+ Console.println("DepInterpreter(%dep) deprecated. "
+ + "Remove dependencies and repositories through GUI interpreter menu instead.");
dependencies = new LinkedList<Dependency>();
repositories = new LinkedList<Repository>();
@@ -156,7 +164,7 @@ public class SparkDependencyContext {
collectRequest.addRepository(repo);
}
for (Repository repo : repositories) {
- RemoteRepository rr = new RemoteRepository(repo.getName(), "default", repo.getUrl());
+ RemoteRepository rr = new RemoteRepository(repo.getId(), "default", repo.getUrl());
rr.setPolicy(repo.isSnapshot(), null);
Authentication auth = repo.getAuthentication();
if (auth != null) {
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
----------------------------------------------------------------------
diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
index 778966f..7064e73 100644
--- a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
+++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
@@ -173,16 +173,6 @@ public class SparkInterpreterTest {
}
@Test
- public void testZContextDependencyLoading() {
- // try to import library does not exist on classpath. it'll fail
- assertEquals(InterpreterResult.Code.ERROR, repl.interpret("import org.apache.commons.csv.CSVFormat", context).code());
-
- // load library from maven repository and try to import again
- repl.interpret("z.load(\"org.apache.commons:commons-csv:1.1\")", context);
- assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("import org.apache.commons.csv.CSVFormat", context).code());
- }
-
- @Test
public void emptyConfigurationVariablesOnlyForNonSparkProperties() {
Properties intpProperty = repl.getProperty();
SparkConf sparkConf = repl.getSparkContext().getConf();
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java
index ba8ee16..f2f3baa 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java
@@ -24,7 +24,9 @@ import java.util.List;
import org.sonatype.aether.RepositorySystem;
import org.sonatype.aether.RepositorySystemSession;
+import org.sonatype.aether.repository.Authentication;
import org.sonatype.aether.repository.RemoteRepository;
+import org.sonatype.aether.repository.RepositoryPolicy;
import org.sonatype.aether.resolution.ArtifactResult;
/**
@@ -41,12 +43,32 @@ public abstract class AbstractDependencyResolver {
repos.add(Booter.newCentralRepository()); // add maven central
repos.add(Booter.newLocalRepository());
}
+
+ public List<RemoteRepository> getRepos() {
+ return this.repos;
+ }
public void addRepo(String id, String url, boolean snapshot) {
synchronized (repos) {
delRepo(id);
RemoteRepository rr = new RemoteRepository(id, "default", url);
- rr.setPolicy(snapshot, null);
+ rr.setPolicy(true, new RepositoryPolicy(
+ snapshot,
+ RepositoryPolicy.UPDATE_POLICY_DAILY,
+ RepositoryPolicy.CHECKSUM_POLICY_WARN));
+ repos.add(rr);
+ }
+ }
+
+ public void addRepo(String id, String url, boolean snapshot, Authentication auth) {
+ synchronized (repos) {
+ delRepo(id);
+ RemoteRepository rr = new RemoteRepository(id, "default", url);
+ rr.setPolicy(true, new RepositoryPolicy(
+ snapshot,
+ RepositoryPolicy.UPDATE_POLICY_DAILY,
+ RepositoryPolicy.CHECKSUM_POLICY_WARN));
+ rr.setAuthentication(auth);
repos.add(rr);
}
}
@@ -54,7 +76,7 @@ public abstract class AbstractDependencyResolver {
public RemoteRepository delRepo(String id) {
synchronized (repos) {
Iterator<RemoteRepository> it = repos.iterator();
- if (it.hasNext()) {
+ while (it.hasNext()) {
RemoteRepository repo = it.next();
if (repo.getId().equals(id)) {
it.remove();
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyContext.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyContext.java
index ab4da28..a320810 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyContext.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyContext.java
@@ -127,7 +127,7 @@ public class DependencyContext {
collectRequest.addRepository(mavenCentral);
collectRequest.addRepository(mavenLocal);
for (Repository repo : repositories) {
- RemoteRepository rr = new RemoteRepository(repo.getName(), "default", repo.getUrl());
+ RemoteRepository rr = new RemoteRepository(repo.getId(), "default", repo.getUrl());
rr.setPolicy(repo.isSnapshot(), null);
collectRequest.addRepository(rr);
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
index cbe88bc..21e1cbb 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
@@ -18,6 +18,7 @@
package org.apache.zeppelin.dep;
import java.io.File;
+import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
@@ -28,6 +29,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.sonatype.aether.RepositoryException;
import org.sonatype.aether.artifact.Artifact;
import org.sonatype.aether.collection.CollectRequest;
import org.sonatype.aether.graph.Dependency;
@@ -56,16 +58,18 @@ public class DependencyResolver extends AbstractDependencyResolver {
super(localRepoPath);
}
- public List<File> load(String artifact) throws Exception {
+ public List<File> load(String artifact)
+ throws RepositoryException, IOException {
return load(artifact, new LinkedList<String>());
}
- public List<File> load(String artifact, String destPath) throws Exception {
+ public List<File> load(String artifact, String destPath)
+ throws RepositoryException, IOException {
return load(artifact, new LinkedList<String>(), destPath);
}
public synchronized List<File> load(String artifact, Collection<String> excludes)
- throws Exception {
+ throws RepositoryException, IOException {
if (StringUtils.isBlank(artifact)) {
// Should throw here
throw new RuntimeException("Invalid artifact to load");
@@ -83,7 +87,7 @@ public class DependencyResolver extends AbstractDependencyResolver {
}
public List<File> load(String artifact, Collection<String> excludes, String destPath)
- throws Exception {
+ throws RepositoryException, IOException {
List<File> libs = load(artifact, excludes);
// find home dir
@@ -105,7 +109,8 @@ public class DependencyResolver extends AbstractDependencyResolver {
return libs;
}
- private List<File> loadFromMvn(String artifact, Collection<String> excludes) throws Exception {
+ private List<File> loadFromMvn(String artifact, Collection<String> excludes)
+ throws RepositoryException {
Collection<String> allExclusions = new LinkedList<String>();
allExclusions.addAll(excludes);
allExclusions.addAll(Arrays.asList(exclusions));
@@ -142,7 +147,7 @@ public class DependencyResolver extends AbstractDependencyResolver {
*/
@Override
public List<ArtifactResult> getArtifactsWithDep(String dependency,
- Collection<String> excludes) throws Exception {
+ Collection<String> excludes) throws RepositoryException {
Artifact artifact = new DefaultArtifact(dependency);
DependencyFilter classpathFilter = DependencyFilterUtils.classpathFilter(JavaScopes.COMPILE);
PatternExclusionsDependencyFilter exclusionFilter =
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java
index 4c2d867..d2b0092 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java
@@ -23,13 +23,13 @@ import org.sonatype.aether.repository.Authentication;
*/
public class Repository {
private boolean snapshot = false;
- private String name;
+ private String id;
private String url;
private String username = null;
private String password = null;
- public Repository(String name){
- this.name = name;
+ public Repository(String id){
+ this.id = id;
}
public Repository url(String url) {
@@ -46,8 +46,8 @@ public class Repository {
return snapshot;
}
- public String getName() {
- return name;
+ public String getId() {
+ return id;
}
public String getUrl() {
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
index ac251d2..014b13e 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
@@ -127,10 +127,6 @@ public abstract class Interpreter {
}
}
-
-
-
-
public static Logger logger = LoggerFactory.getLogger(Interpreter.class);
private InterpreterGroup interpreterGroup;
private URL [] classloaderUrls;
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
index d8cb223..f1eec08 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
@@ -30,9 +30,7 @@ import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
-import org.apache.zeppelin.interpreter.WrappedInterpreter;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
@@ -53,6 +51,7 @@ public class RemoteInterpreter extends Interpreter {
Gson gson = new Gson();
private String interpreterRunner;
private String interpreterPath;
+ private String localRepoPath;
private String className;
FormType formType;
boolean initialized;
@@ -61,17 +60,19 @@ public class RemoteInterpreter extends Interpreter {
private int maxPoolSize;
public RemoteInterpreter(Properties property,
- String className,
- String interpreterRunner,
- String interpreterPath,
- int connectTimeout,
- int maxPoolSize,
- RemoteInterpreterProcessListener remoteInterpreterProcessListener) {
+ String className,
+ String interpreterRunner,
+ String interpreterPath,
+ String localRepoPath,
+ int connectTimeout,
+ int maxPoolSize,
+ RemoteInterpreterProcessListener remoteInterpreterProcessListener) {
super(property);
this.className = className;
initialized = false;
this.interpreterRunner = interpreterRunner;
this.interpreterPath = interpreterPath;
+ this.localRepoPath = localRepoPath;
env = new HashMap<String, String>();
this.connectTimeout = connectTimeout;
this.maxPoolSize = maxPoolSize;
@@ -79,16 +80,18 @@ public class RemoteInterpreter extends Interpreter {
}
public RemoteInterpreter(Properties property,
- String className,
- String interpreterRunner,
- String interpreterPath,
- Map<String, String> env,
- int connectTimeout,
- RemoteInterpreterProcessListener remoteInterpreterProcessListener) {
+ String className,
+ String interpreterRunner,
+ String interpreterPath,
+ String localRepoPath,
+ Map<String, String> env,
+ int connectTimeout,
+ RemoteInterpreterProcessListener remoteInterpreterProcessListener) {
super(property);
this.className = className;
this.interpreterRunner = interpreterRunner;
this.interpreterPath = interpreterPath;
+ this.localRepoPath = localRepoPath;
this.env = env;
this.connectTimeout = connectTimeout;
this.maxPoolSize = 10;
@@ -110,8 +113,8 @@ public class RemoteInterpreter extends Interpreter {
if (intpGroup.getRemoteInterpreterProcess() == null) {
// create new remote process
RemoteInterpreterProcess remoteProcess = new RemoteInterpreterProcess(
- interpreterRunner, interpreterPath, env, connectTimeout,
- remoteInterpreterProcessListener);
+ interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout,
+ remoteInterpreterProcessListener);
intpGroup.setRemoteInterpreterProcess(remoteProcess);
}
@@ -143,6 +146,7 @@ public class RemoteInterpreter extends Interpreter {
try {
for (Interpreter intp : this.getInterpreterGroup()) {
logger.info("Create remote interpreter {}", intp.getClassName());
+ property.put("zeppelin.interpreter.localRepo", localRepoPath);
client.createInterpreter(intp.getClassName(), (Map) property);
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
index 5237b0b..2c88894 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
@@ -45,6 +45,7 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
private int port = -1;
private final String interpreterRunner;
private final String interpreterDir;
+ private final String localRepoDir;
private GenericObjectPool<Client> clientPool;
private Map<String, String> env;
@@ -53,20 +54,28 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
private int connectTimeout;
public RemoteInterpreterProcess(String intpRunner,
- String intpDir,
- Map<String, String> env,
- int connectTimeout,
- RemoteInterpreterProcessListener listener) {
- this(intpRunner, intpDir, env, new RemoteInterpreterEventPoller(listener), connectTimeout);
+ String intpDir,
+ String localRepoDir,
+ Map<String, String> env,
+ int connectTimeout,
+ RemoteInterpreterProcessListener listener) {
+ this(intpRunner,
+ intpDir,
+ localRepoDir,
+ env,
+ new RemoteInterpreterEventPoller(listener),
+ connectTimeout);
}
RemoteInterpreterProcess(String intpRunner,
String intpDir,
+ String localRepoDir,
Map<String, String> env,
RemoteInterpreterEventPoller remoteInterpreterEventPoller,
int connectTimeout) {
this.interpreterRunner = intpRunner;
this.interpreterDir = intpDir;
+ this.localRepoDir = localRepoDir;
this.env = env;
this.interpreterContextRunnerPool = new InterpreterContextRunnerPool();
referenceCount = new AtomicInteger(0);
@@ -89,12 +98,13 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
throw new InterpreterException(e1);
}
-
CommandLine cmdLine = CommandLine.parse(interpreterRunner);
cmdLine.addArgument("-d", false);
cmdLine.addArgument(interpreterDir, false);
cmdLine.addArgument("-p", false);
cmdLine.addArgument(Integer.toString(port), false);
+ cmdLine.addArgument("-l", false);
+ cmdLine.addArgument(localRepoDir, false);
executor = new DefaultExecutor();
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java
index 33b7e54..a8664ef 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java
@@ -17,14 +17,18 @@
package org.apache.zeppelin.dep;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
import java.io.File;
+import java.util.Collections;
import org.apache.commons.io.FileUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.sonatype.aether.RepositoryException;
public class DependencyResolverTest {
private static DependencyResolver resolver;
@@ -50,13 +54,47 @@ public class DependencyResolverTest {
public static void tearDown() throws Exception {
FileUtils.deleteDirectory(new File(home + "/" + testPath));
FileUtils.deleteDirectory(new File(home + "/" + testCopyPath));
- }
-
+ }
+
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ @Test
+ public void testAddRepo() {
+ int reposCnt = resolver.getRepos().size();
+ resolver.addRepo("securecentral", "https://repo1.maven.org/maven2", false);
+ assertEquals(reposCnt + 1, resolver.getRepos().size());
+ }
+
+ @Test
+ public void testDelRepo() {
+ int reposCnt = resolver.getRepos().size();
+ resolver.delRepo("securecentral");
+ resolver.delRepo("badId");
+ assertEquals(reposCnt - 1, resolver.getRepos().size());
+ }
+
@Test
public void testLoad() throws Exception {
- resolver.load("org.apache.commons:commons-lang3:3.4", testCopyPath);
+ // basic load
+ resolver.load("com.databricks:spark-csv_2.10:1.3.0", testCopyPath);
+ assertEquals(new File(home + "/" + testCopyPath).list().length, 4);
+ FileUtils.cleanDirectory(new File(home + "/" + testCopyPath));
+
+ // load with exclusions parameter
+ resolver.load("com.databricks:spark-csv_2.10:1.3.0",
+ Collections.singletonList("org.scala-lang:scala-library"), testCopyPath);
+ assertEquals(new File(home + "/" + testCopyPath).list().length, 3);
+ FileUtils.cleanDirectory(new File(home + "/" + testCopyPath));
+
+ // load from added repository
+ resolver.addRepo("sonatype", "https://oss.sonatype.org/content/repositories/agimatec-releases/", false);
+ resolver.load("com.agimatec:agimatec-validation:0.9.3", testCopyPath);
+ assertEquals(new File(home + "/" + testCopyPath).list().length, 8);
- assertTrue(new File(home + "/" + testPath + "/org/apache/commons/commons-lang3/3.4/").exists());
- assertTrue(new File(home + "/" + testCopyPath + "/commons-lang3-3.4.jar").exists());
+ // load invalid artifact
+ resolver.delRepo("sonatype");
+ exception.expect(RepositoryException.class);
+ resolver.load("com.agimatec:agimatec-validation:0.9.3", testCopyPath);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
index 4cd974d..385b9d6 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
@@ -64,14 +64,15 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
Properties p = new Properties();
intp = new RemoteInterpreter(
- p,
- MockInterpreterAngular.class.getName(),
- new File("../bin/interpreter.sh").getAbsolutePath(),
- "fake",
- env,
- 10 * 1000,
- null
- );
+ p,
+ MockInterpreterAngular.class.getName(),
+ new File("../bin/interpreter.sh").getAbsolutePath(),
+ "fake",
+ "fakeRepo",
+ env,
+ 10 * 1000,
+ null
+ );
intpGroup.add(intp);
intp.setInterpreterGroup(intpGroup);
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
index 623a037..f229f6b 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
@@ -55,14 +55,14 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
private RemoteInterpreter createMockInterpreter() {
RemoteInterpreter intp = new RemoteInterpreter(
- new Properties(),
- MockInterpreterOutputStream.class.getName(),
- new File("../bin/interpreter.sh").getAbsolutePath(),
- "fake",
- env,
- 10 * 1000,
- this);
-
+ new Properties(),
+ MockInterpreterOutputStream.class.getName(),
+ new File("../bin/interpreter.sh").getAbsolutePath(),
+ "fake",
+ "fakeRepo",
+ env,
+ 10 * 1000,
+ this);
intpGroup.add(intp);
intp.setInterpreterGroup(intpGroup);
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
index abee5b8..7beaee1 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
@@ -33,7 +33,7 @@ public class RemoteInterpreterProcessTest {
public void testStartStop() {
InterpreterGroup intpGroup = new InterpreterGroup();
RemoteInterpreterProcess rip = new RemoteInterpreterProcess(
- "../bin/interpreter.sh", "nonexists", new HashMap<String, String>(),
+ "../bin/interpreter.sh", "nonexists", "fakeRepo", new HashMap<String, String>(),
10 * 1000, null);
assertFalse(rip.isRunning());
assertEquals(0, rip.referenceCount());
@@ -50,7 +50,7 @@ public class RemoteInterpreterProcessTest {
public void testClientFactory() throws Exception {
InterpreterGroup intpGroup = new InterpreterGroup();
RemoteInterpreterProcess rip = new RemoteInterpreterProcess(
- "../bin/interpreter.sh", "nonexists", new HashMap<String, String>(),
+ "../bin/interpreter.sh", "nonexists", "fakeRepo", new HashMap<String, String>(),
mock(RemoteInterpreterEventPoller.class), 10 * 1000);
rip.reference(intpGroup);
assertEquals(0, rip.getNumActiveClient());
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
index 034a676..82ca8d4 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
@@ -69,6 +69,7 @@ public class RemoteInterpreterTest {
MockInterpreterA.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
"fake",
+ "fakeRepo",
env,
10 * 1000,
null);
@@ -80,6 +81,7 @@ public class RemoteInterpreterTest {
MockInterpreterB.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
"fake",
+ "fakeRepo",
env,
10 * 1000,
null);
@@ -164,27 +166,27 @@ public class RemoteInterpreterTest {
Properties p = new Properties();
RemoteInterpreter intpA = new RemoteInterpreter(
- p,
- MockInterpreterA.class.getName(),
- new File("../bin/interpreter.sh").getAbsolutePath(),
- "fake",
- env,
- 10 * 1000,
- null
- );
+ p,
+ MockInterpreterA.class.getName(),
+ new File("../bin/interpreter.sh").getAbsolutePath(),
+ "fake",
+ "fakeRepo",
+ env,
+ 10 * 1000,
+ null);
intpGroup.add(intpA);
intpA.setInterpreterGroup(intpGroup);
RemoteInterpreter intpB = new RemoteInterpreter(
- p,
- MockInterpreterB.class.getName(),
- new File("../bin/interpreter.sh").getAbsolutePath(),
- "fake",
- env,
- 10 * 1000,
- null
- );
+ p,
+ MockInterpreterB.class.getName(),
+ new File("../bin/interpreter.sh").getAbsolutePath(),
+ "fake",
+ "fakeRepo",
+ env,
+ 10 * 1000,
+ null);
intpGroup.add(intpB);
intpB.setInterpreterGroup(intpGroup);
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
index 05bc676..d46b8cf 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
@@ -64,14 +64,14 @@ public class RemoteSchedulerTest {
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
final RemoteInterpreter intpA = new RemoteInterpreter(
- p,
- MockInterpreterA.class.getName(),
- new File("../bin/interpreter.sh").getAbsolutePath(),
- "fake",
- env,
- 10 * 1000,
- null
- );
+ p,
+ MockInterpreterA.class.getName(),
+ new File("../bin/interpreter.sh").getAbsolutePath(),
+ "fake",
+ "fakeRepo",
+ env,
+ 10 * 1000,
+ null);
intpGroup.add(intpA);
intpA.setInterpreterGroup(intpGroup);
@@ -148,14 +148,14 @@ public class RemoteSchedulerTest {
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
final RemoteInterpreter intpA = new RemoteInterpreter(
- p,
- MockInterpreterA.class.getName(),
- new File("../bin/interpreter.sh").getAbsolutePath(),
- "fake",
- env,
- 10 * 1000,
- null
- );
+ p,
+ MockInterpreterA.class.getName(),
+ new File("../bin/interpreter.sh").getAbsolutePath(),
+ "fake",
+ "fakeRepo",
+ env,
+ 10 * 1000,
+ null);
intpGroup.add(intpA);
intpA.setInterpreterGroup(intpGroup);
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
index e1691ad..50d7d64 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
@@ -33,6 +33,7 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.zeppelin.dep.Repository;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter;
import org.apache.zeppelin.rest.message.NewInterpreterSettingRequest;
@@ -42,6 +43,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
+import org.sonatype.aether.RepositoryException;
+import org.sonatype.aether.repository.RemoteRepository;
/**
* Interpreter Rest API
@@ -85,17 +88,34 @@ public class InterpreterRestApi {
*/
@POST
@Path("setting")
- public Response newSettings(String message) throws InterpreterException, IOException {
- NewInterpreterSettingRequest request = gson.fromJson(message,
- NewInterpreterSettingRequest.class);
- Properties p = new Properties();
- p.putAll(request.getProperties());
- // Option is deprecated from API, always use remote = true
- InterpreterGroup interpreterGroup = interpreterFactory.add(request.getName(),
- request.getGroup(), new InterpreterOption(true), p);
- InterpreterSetting setting = interpreterFactory.get(interpreterGroup.getId());
- logger.info("new setting created with " + setting.id());
- return new JsonResponse(Status.CREATED, "", setting ).build();
+ public Response newSettings(String message) {
+ try {
+ NewInterpreterSettingRequest request = gson.fromJson(message,
+ NewInterpreterSettingRequest.class);
+ Properties p = new Properties();
+ p.putAll(request.getProperties());
+ // Option is deprecated from API, always use remote = true
+ InterpreterGroup interpreterGroup = interpreterFactory.add(request.getName(),
+ request.getGroup(),
+ request.getDependencies(),
+ new InterpreterOption(true),
+ p);
+ InterpreterSetting setting = interpreterFactory.get(interpreterGroup.getId());
+ logger.info("new setting created with {}", setting.id());
+ return new JsonResponse(Status.CREATED, "", setting).build();
+ } catch (InterpreterException e) {
+ logger.error("Exception in InterpreterRestApi while creating ", e);
+ return new JsonResponse(
+ Status.NOT_FOUND,
+ e.getMessage(),
+ ExceptionUtils.getStackTrace(e)).build();
+ } catch (IOException | RepositoryException e) {
+ logger.error("Exception in InterpreterRestApi while creating ", e);
+ return new JsonResponse(
+ Status.INTERNAL_SERVER_ERROR,
+ e.getMessage(),
+ ExceptionUtils.getStackTrace(e)).build();
+ }
}
@PUT
@@ -104,16 +124,18 @@ public class InterpreterRestApi {
logger.info("Update interpreterSetting {}", settingId);
try {
- UpdateInterpreterSettingRequest p = gson.fromJson(message,
+ UpdateInterpreterSettingRequest request = gson.fromJson(message,
UpdateInterpreterSettingRequest.class);
// Option is deprecated from API, always use remote = true
interpreterFactory.setPropertyAndRestart(settingId,
- new InterpreterOption(true), p.getProperties());
+ new InterpreterOption(true),
+ request.getProperties(),
+ request.getDependencies());
} catch (InterpreterException e) {
logger.error("Exception in InterpreterRestApi while updateSetting ", e);
return new JsonResponse(
Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e)).build();
- } catch (IOException e) {
+ } catch (IOException | RepositoryException e) {
logger.error("Exception in InterpreterRestApi while updateSetting ", e);
return new JsonResponse(
Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build();
@@ -165,4 +187,59 @@ public class InterpreterRestApi {
Map<String, RegisteredInterpreter> m = Interpreter.registeredInterpreters;
return new JsonResponse(Status.OK, "", m).build();
}
+
+ /**
+ * List of dependency resolving repositories
+ * @return
+ */
+ @GET
+ @Path("repository")
+ public Response listRepositories() {
+ List<RemoteRepository> interpreterRepositories = null;
+ interpreterRepositories = interpreterFactory.getRepositories();
+ return new JsonResponse(Status.OK, "", interpreterRepositories).build();
+ }
+
+ /**
+ * Add new repository
+ * @param message
+ * @return
+ */
+ @POST
+ @Path("repository")
+ public Response addRepository(String message) {
+ try {
+ Repository request = gson.fromJson(message, Repository.class);
+ interpreterFactory.addRepository(
+ request.getId(),
+ request.getUrl(),
+ request.isSnapshot(),
+ request.getAuthentication());
+ logger.info("New repository {} added", request.getId());
+ } catch (Exception e) {
+ logger.error("Exception in InterpreterRestApi while adding repository ", e);
+ return new JsonResponse(
+ Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build();
+ }
+ return new JsonResponse(Status.CREATED).build();
+ }
+
+ /**
+ * Delete repository
+ * @param repoId
+ * @return
+ */
+ @DELETE
+ @Path("repository/{repoId}")
+ public Response removeRepository(@PathParam("repoId") String repoId) {
+ logger.info("Remove repository {}", repoId);
+ try {
+ interpreterFactory.removeRepository(repoId);
+ } catch (Exception e) {
+ logger.error("Exception in InterpreterRestApi while removing repository ", e);
+ return new JsonResponse(
+ Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build();
+ }
+ return new JsonResponse(Status.OK).build();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewInterpreterSettingRequest.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewInterpreterSettingRequest.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewInterpreterSettingRequest.java
index 36f80f6..22eb25f 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewInterpreterSettingRequest.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewInterpreterSettingRequest.java
@@ -17,9 +17,10 @@
package org.apache.zeppelin.rest.message;
+import java.util.List;
import java.util.Map;
-import org.apache.zeppelin.interpreter.InterpreterOption;
+import org.apache.zeppelin.dep.Dependency;
/**
* NewInterpreterSetting rest api request message
@@ -30,6 +31,7 @@ public class NewInterpreterSettingRequest {
String group;
// option was deprecated
Map<String, String> properties;
+ List<Dependency> dependencies;
public NewInterpreterSettingRequest() {
@@ -47,4 +49,7 @@ public class NewInterpreterSettingRequest {
return properties;
}
+ public List<Dependency> getDependencies() {
+ return dependencies;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/UpdateInterpreterSettingRequest.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/UpdateInterpreterSettingRequest.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/UpdateInterpreterSettingRequest.java
index f1f496a..a3f71ea 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/UpdateInterpreterSettingRequest.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/UpdateInterpreterSettingRequest.java
@@ -17,27 +17,30 @@
package org.apache.zeppelin.rest.message;
+import java.util.List;
import java.util.Properties;
-import org.apache.zeppelin.interpreter.InterpreterOption;
+import org.apache.zeppelin.dep.Dependency;
/**
- *
+ * UpdateInterpreterSetting rest api request message
*/
public class UpdateInterpreterSettingRequest {
-
// option was deprecated
Properties properties;
+ List<Dependency> dependencies;
- public UpdateInterpreterSettingRequest(InterpreterOption option,
- Properties properties) {
- super();
+ public UpdateInterpreterSettingRequest(Properties properties,
+ List<Dependency> dependencies) {
this.properties = properties;
+ this.dependencies = dependencies;
}
public Properties getProperties() {
return properties;
}
-
+ public List<Dependency> getDependencies() {
+ return dependencies;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
index 760298f..40e4d14 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
@@ -76,7 +76,8 @@ public class ZeppelinServer extends Application {
public ZeppelinServer() throws Exception {
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
- this.depResolver = new DependencyResolver(conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO));
+ this.depResolver = new DependencyResolver(
+ conf.getString(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO));
this.schedulerFactory = new SchedulerFactory();
this.replFactory = new InterpreterFactory(conf, notebookWsServer,
notebookWsServer, depResolver);
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-server/src/test/java/org/apache/zeppelin/ZeppelinIT.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/ZeppelinIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/ZeppelinIT.java
index e6591f2..e9a56cc 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/ZeppelinIT.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/ZeppelinIT.java
@@ -17,20 +17,17 @@
package org.apache.zeppelin;
-import com.google.common.base.Function;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.openqa.selenium.*;
-import org.openqa.selenium.support.ui.FluentWait;
-import org.openqa.selenium.support.ui.Wait;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
/**
* Test Zeppelin with web browser.
@@ -202,4 +199,49 @@ public class ZeppelinIT extends AbstractZeppelinIT {
}
}
+
+ @Test
+ public void testSparkInterpreterDependencyLoading() {
+ // navigate to interpreter page
+ WebElement interpreterLink = driver.findElement(By.linkText("Interpreter"));
+ interpreterLink.click();
+
+ // add new dependency to spark interpreter
+ WebElement sparkEditBtn = pollingWait(By.xpath("//div[h3[text()[contains(.,'spark')]]]//button[contains(.,'edit')]"),
+ MAX_BROWSER_TIMEOUT_SEC);
+ sparkEditBtn.click();
+ WebElement depArtifact = driver.findElement(By.xpath("//input[@ng-model='setting.depArtifact']"));
+ String artifact = "org.apache.commons:commons-csv:1.1";
+ depArtifact.sendKeys(artifact);
+ driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit();
+ driver.switchTo().alert().accept();
+
+ driver.navigate().back();
+ createNewNote();
+
+ // wait for first paragraph's " READY " status text
+ waitForParagraph(1, "READY");
+
+ WebElement paragraph1Editor = driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea"));
+
+ paragraph1Editor.sendKeys("import org.apache.commons.csv.CSVFormat");
+ paragraph1Editor.sendKeys(Keys.chord(Keys.SHIFT, Keys.ENTER));
+ waitForParagraph(1, "FINISHED");
+
+ // check expected text
+ assertTrue(waitForText("import org.apache.commons.csv.CSVFormat",
+ By.xpath(getParagraphXPath(1) + "//div[starts-with(@id, 'p') and contains(@id, 'text')]/div")));
+
+ // reset dependency
+ interpreterLink.click();
+ sparkEditBtn = pollingWait(By.xpath("//div[h3[text()[contains(.,'spark')]]]//button[contains(.,'edit')]"),
+ MAX_BROWSER_TIMEOUT_SEC);
+ sparkEditBtn.click();
+ WebElement testDepRemoveBtn = driver.findElement(By.xpath("//tr[descendant::text()[contains(.,'" +
+ artifact + "')]]/td[3]/div"));
+ sleep(5000, true);
+ testDepRemoveBtn.click();
+ driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit();
+ driver.switchTo().alert().accept();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/218a3b5b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
index 0c4199d..81f161a 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.lang.ref.WeakReference;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
@@ -35,6 +36,7 @@ import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.PutMethod;
import org.apache.commons.httpclient.methods.RequestEntity;
+import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterSetting;
@@ -48,6 +50,7 @@ import org.slf4j.LoggerFactory;
import com.google.gson.JsonElement;
import com.google.gson.JsonParseException;
import com.google.gson.JsonParser;
+import org.sonatype.aether.RepositoryException;
public abstract class AbstractTestRestApi {
@@ -363,10 +366,14 @@ public abstract class AbstractTestRestApi {
}
//Create new Setting and return Setting ID
- protected String createTempSetting(String tempName) throws IOException {
-
- InterpreterGroup interpreterGroup = ZeppelinServer.notebook.getInterpreterFactory().add(tempName,"newGroup",
- new InterpreterOption(false),new Properties());
+ protected String createTempSetting(String tempName)
+ throws IOException, RepositoryException {
+ InterpreterGroup interpreterGroup = ZeppelinServer.notebook.getInterpreterFactory()
+ .add(tempName,
+ "newGroup",
+ new LinkedList<Dependency>(),
+ new InterpreterOption(false),
+ new Properties());
return interpreterGroup.getId();
}