You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by jo...@apache.org on 2017/02/24 09:00:10 UTC
[2/3] zeppelin git commit: ZEPPELIN-2057 InterpreterSettingManager
for branch-0.7
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
new file mode 100644
index 0000000..b141e6b
--- /dev/null
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
@@ -0,0 +1,1125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.internal.StringMap;
+import com.google.gson.reflect.TypeToken;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.lang.reflect.Type;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream.Filter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import java.nio.file.attribute.PosixFilePermission;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.dep.Dependency;
+import org.apache.zeppelin.dep.DependencyResolver;
+import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter;
+import org.apache.zeppelin.scheduler.Job;
+import org.apache.zeppelin.scheduler.Job.Status;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.sonatype.aether.RepositoryException;
+import org.sonatype.aether.repository.Authentication;
+import org.sonatype.aether.repository.Proxy;
+import org.sonatype.aether.repository.RemoteRepository;
+
+/**
+ * TBD
+ */
+public class InterpreterSettingManager {
+
+ private static final Logger logger = LoggerFactory.getLogger(InterpreterSettingManager.class);
+ private static final String SHARED_SESSION = "shared_session";
+ private static final Map<String, Object> DEFAULT_EDITOR = ImmutableMap.of(
+ "language", (Object) "text",
+ "editOnDblClick", false);
+
+ private final ZeppelinConfiguration zeppelinConfiguration;
+ private final Path interpreterDirPath;
+ private final Path interpreterBindingPath;
+
+ /**
+ * This is only references with default settings, name and properties
+ * key: InterpreterSetting.name
+ */
+ private final Map<String, InterpreterSetting> interpreterSettingsRef;
+ /**
+ * This is used by creating and running Interpreters
+ * key: InterpreterSetting.id <- This is becuase backward compatibility
+ */
+ private final Map<String, InterpreterSetting> interpreterSettings;
+ private final Map<String, List<String>> interpreterBindings;
+
+ private final DependencyResolver dependencyResolver;
+ private final List<RemoteRepository> interpreterRepositories;
+
+ private final InterpreterOption defaultOption;
+
+ private final Map<String, URLClassLoader> cleanCl;
+
+ @Deprecated
+ private String[] interpreterClassList;
+ private String[] interpreterGroupOrderList;
+ private InterpreterGroupFactory interpreterGroupFactory;
+
+ private final Gson gson;
+
+ public InterpreterSettingManager(ZeppelinConfiguration zeppelinConfiguration,
+ DependencyResolver dependencyResolver, InterpreterOption interpreterOption)
+ throws IOException, RepositoryException {
+ this.zeppelinConfiguration = zeppelinConfiguration;
+ this.interpreterDirPath = Paths.get(zeppelinConfiguration.getInterpreterDir());
+ logger.debug("InterpreterRootPath: {}", interpreterDirPath);
+ this.interpreterBindingPath = Paths.get(zeppelinConfiguration.getInterpreterSettingPath());
+ logger.debug("InterpreterBindingPath: {}", interpreterBindingPath);
+
+ this.interpreterSettingsRef = Maps.newConcurrentMap();
+ this.interpreterSettings = Maps.newConcurrentMap();
+ this.interpreterBindings = Maps.newConcurrentMap();
+
+ this.dependencyResolver = dependencyResolver;
+ this.interpreterRepositories = dependencyResolver.getRepos();
+
+ this.defaultOption = interpreterOption;
+
+ this.cleanCl = Collections.synchronizedMap(new HashMap<String, URLClassLoader>());
+
+ String replsConf = zeppelinConfiguration.getString(ConfVars.ZEPPELIN_INTERPRETERS);
+ this.interpreterClassList = replsConf.split(",");
+ String groupOrder = zeppelinConfiguration.getString(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER);
+ this.interpreterGroupOrderList = groupOrder.split(",");
+
+ GsonBuilder gsonBuilder = new GsonBuilder();
+ gsonBuilder.setPrettyPrinting();
+ this.gson = gsonBuilder.create();
+
+ init();
+ }
+
+ private void loadFromFile() {
+ if (!Files.exists(interpreterBindingPath)) {
+ // nothing to read
+ return;
+ }
+ InterpreterInfoSaving infoSaving;
+ try (BufferedReader json =
+ Files.newBufferedReader(interpreterBindingPath, StandardCharsets.UTF_8)) {
+ infoSaving = gson.fromJson(json, InterpreterInfoSaving.class);
+
+ for (String k : infoSaving.interpreterSettings.keySet()) {
+ InterpreterSetting setting = infoSaving.interpreterSettings.get(k);
+ List<InterpreterInfo> infos = setting.getInterpreterInfos();
+
+ // Convert json StringMap to Properties
+ StringMap<String> p = (StringMap<String>) setting.getProperties();
+ Properties properties = new Properties();
+ for (String key : p.keySet()) {
+ properties.put(key, p.get(key));
+ }
+ setting.setProperties(properties);
+
+ // Always use separate interpreter process
+ // While we decided to turn this feature on always (without providing
+ // enable/disable option on GUI).
+ // previously created setting should turn this feature on here.
+ setting.getOption().setRemote(true);
+
+ // Update transient information from InterpreterSettingRef
+ InterpreterSetting interpreterSettingObject =
+ interpreterSettingsRef.get(setting.getGroup());
+ if (interpreterSettingObject == null) {
+ logger.warn("can't get InterpreterSetting " +
+ "Information From loaded Interpreter Setting Ref - {} ", setting.getGroup());
+ continue;
+ }
+ String depClassPath = interpreterSettingObject.getPath();
+ setting.setPath(depClassPath);
+
+ for (InterpreterInfo info : infos) {
+ if (info.getEditor() == null) {
+ Map<String, Object> editor = getEditorFromSettingByClassName(interpreterSettingObject,
+ info.getClassName());
+ info.setEditor(editor);
+ }
+ }
+
+ setting.setInterpreterGroupFactory(interpreterGroupFactory);
+
+ loadInterpreterDependencies(setting);
+ interpreterSettings.put(k, setting);
+ }
+
+ interpreterBindings.putAll(infoSaving.interpreterBindings);
+
+ if (infoSaving.interpreterRepositories != null) {
+ for (RemoteRepository repo : infoSaving.interpreterRepositories) {
+ if (!dependencyResolver.getRepos().contains(repo)) {
+ this.interpreterRepositories.add(repo);
+ }
+ }
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public void saveToFile() throws IOException {
+ String jsonString;
+
+ synchronized (interpreterSettings) {
+ InterpreterInfoSaving info = new InterpreterInfoSaving();
+ info.interpreterBindings = interpreterBindings;
+ info.interpreterSettings = interpreterSettings;
+ info.interpreterRepositories = interpreterRepositories;
+
+ jsonString = gson.toJson(info);
+ }
+
+ if (!Files.exists(interpreterBindingPath)) {
+ Files.createFile(interpreterBindingPath);
+
+ Set<PosixFilePermission> permissions = EnumSet.of(OWNER_READ, OWNER_WRITE);
+ Files.setPosixFilePermissions(interpreterBindingPath, permissions);
+ }
+
+ FileOutputStream fos = new FileOutputStream(interpreterBindingPath.toFile(), false);
+ OutputStreamWriter out = new OutputStreamWriter(fos);
+ out.append(jsonString);
+ out.close();
+ fos.close();
+ }
+
+ //TODO(jl): Fix it to remove InterpreterGroupFactory
+ public void setInterpreterGroupFactory(InterpreterGroupFactory interpreterGroupFactory) {
+ for (InterpreterSetting setting : interpreterSettings.values()) {
+ setting.setInterpreterGroupFactory(interpreterGroupFactory);
+ }
+ this.interpreterGroupFactory = interpreterGroupFactory;
+ }
+
+ private void init() throws InterpreterException, IOException, RepositoryException {
+ String interpreterJson = zeppelinConfiguration.getInterpreterJson();
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+
+ if (Files.exists(interpreterDirPath)) {
+ for (Path interpreterDir : Files
+ .newDirectoryStream(interpreterDirPath, new Filter<Path>() {
+ @Override
+ public boolean accept(Path entry) throws IOException {
+ return Files.exists(entry) && Files.isDirectory(entry);
+ }
+ })) {
+ String interpreterDirString = interpreterDir.toString();
+
+ /**
+ * Register interpreter by the following ordering
+ * 1. Register it from path {ZEPPELIN_HOME}/interpreter/{interpreter_name}/
+ * interpreter-setting.json
+ * 2. Register it from interpreter-setting.json in classpath
+ * {ZEPPELIN_HOME}/interpreter/{interpreter_name}
+ * 3. Register it by Interpreter.register
+ */
+ if (!registerInterpreterFromPath(interpreterDirString, interpreterJson)) {
+ if (!registerInterpreterFromResource(cl, interpreterDirString, interpreterJson)) {
+ /*
+ * TODO(jongyoul)
+ * - Remove these codes below because of legacy code
+ * - Support ThreadInterpreter
+ */
+ URLClassLoader ccl = new URLClassLoader(
+ recursiveBuildLibList(interpreterDir.toFile()), cl);
+ for (String className : interpreterClassList) {
+ try {
+ // Load classes
+ Class.forName(className, true, ccl);
+ Set<String> interpreterKeys = Interpreter.registeredInterpreters.keySet();
+ for (String interpreterKey : interpreterKeys) {
+ if (className
+ .equals(Interpreter.registeredInterpreters.get(interpreterKey)
+ .getClassName())) {
+ Interpreter.registeredInterpreters.get(interpreterKey)
+ .setPath(interpreterDirString);
+ logger.info("Interpreter " + interpreterKey + " found. class=" + className);
+ cleanCl.put(interpreterDirString, ccl);
+ }
+ }
+ } catch (Throwable t) {
+ // nothing to do
+ }
+ }
+ }
+ }
+ }
+ }
+
+ for (RegisteredInterpreter registeredInterpreter : Interpreter.registeredInterpreters
+ .values()) {
+ logger
+ .debug("Registered: {} -> {}. Properties: {}", registeredInterpreter.getInterpreterKey(),
+ registeredInterpreter.getClassName(), registeredInterpreter.getProperties());
+ }
+
+ // RegisteredInterpreters -> interpreterSettingRef
+ InterpreterInfo interpreterInfo;
+ for (RegisteredInterpreter r : Interpreter.registeredInterpreters.values()) {
+ interpreterInfo =
+ new InterpreterInfo(r.getClassName(), r.getName(), r.isDefaultInterpreter(),
+ r.getEditor());
+ add(r.getGroup(), interpreterInfo, r.getProperties(), defaultOption, r.getPath(),
+ r.getRunner());
+ }
+
+ for (String settingId : interpreterSettingsRef.keySet()) {
+ InterpreterSetting setting = interpreterSettingsRef.get(settingId);
+ logger.info("InterpreterSettingRef name {}", setting.getName());
+ }
+
+ loadFromFile();
+
+ // if no interpreter settings are loaded, create default set
+ if (0 == interpreterSettings.size()) {
+ Map<String, InterpreterSetting> temp = new HashMap<>();
+ InterpreterSetting interpreterSetting;
+ for (InterpreterSetting setting : interpreterSettingsRef.values()) {
+ interpreterSetting = createFromInterpreterSettingRef(setting);
+ temp.put(setting.getName(), interpreterSetting);
+ }
+
+ for (String group : interpreterGroupOrderList) {
+ if (null != (interpreterSetting = temp.remove(group))) {
+ interpreterSettings.put(interpreterSetting.getId(), interpreterSetting);
+ }
+ }
+
+ for (InterpreterSetting setting : temp.values()) {
+ interpreterSettings.put(setting.getId(), setting);
+ }
+
+ saveToFile();
+ }
+
+ for (String settingId : interpreterSettings.keySet()) {
+ InterpreterSetting setting = interpreterSettings.get(settingId);
+ logger.info("InterpreterSetting group {} : id={}, name={}", setting.getGroup(), settingId,
+ setting.getName());
+ }
+ }
+
+ private boolean registerInterpreterFromResource(ClassLoader cl, String interpreterDir,
+ String interpreterJson) throws IOException, RepositoryException {
+ URL[] urls = recursiveBuildLibList(new File(interpreterDir));
+ ClassLoader tempClassLoader = new URLClassLoader(urls, cl);
+
+ Enumeration<URL> interpreterSettings = tempClassLoader.getResources(interpreterJson);
+ if (!interpreterSettings.hasMoreElements()) {
+ return false;
+ }
+ for (URL url : Collections.list(interpreterSettings)) {
+ try (InputStream inputStream = url.openStream()) {
+ logger.debug("Reading {} from {}", interpreterJson, url);
+ List<RegisteredInterpreter> registeredInterpreterList =
+ getInterpreterListFromJson(inputStream);
+ registerInterpreters(registeredInterpreterList, interpreterDir);
+ }
+ }
+ return true;
+ }
+
+ private boolean registerInterpreterFromPath(String interpreterDir, String interpreterJson)
+ throws IOException, RepositoryException {
+
+ Path interpreterJsonPath = Paths.get(interpreterDir, interpreterJson);
+ if (Files.exists(interpreterJsonPath)) {
+ logger.debug("Reading {}", interpreterJsonPath);
+ List<RegisteredInterpreter> registeredInterpreterList =
+ getInterpreterListFromJson(interpreterJsonPath);
+ registerInterpreters(registeredInterpreterList, interpreterDir);
+ return true;
+ }
+ return false;
+ }
+
+ private List<RegisteredInterpreter> getInterpreterListFromJson(Path filename)
+ throws FileNotFoundException {
+ return getInterpreterListFromJson(new FileInputStream(filename.toFile()));
+ }
+
+ private List<RegisteredInterpreter> getInterpreterListFromJson(InputStream stream) {
+ Type registeredInterpreterListType = new TypeToken<List<RegisteredInterpreter>>() {
+ }.getType();
+ return gson.fromJson(new InputStreamReader(stream), registeredInterpreterListType);
+ }
+
+ private void registerInterpreters(List<RegisteredInterpreter> registeredInterpreters,
+ String absolutePath) throws IOException, RepositoryException {
+
+ for (RegisteredInterpreter registeredInterpreter : registeredInterpreters) {
+ InterpreterInfo interpreterInfo =
+ new InterpreterInfo(registeredInterpreter.getClassName(), registeredInterpreter.getName(),
+ registeredInterpreter.isDefaultInterpreter(), registeredInterpreter.getEditor());
+ // use defaultOption if it is not specified in interpreter-setting.json
+ InterpreterOption option = registeredInterpreter.getOption() == null ? defaultOption :
+ registeredInterpreter.getOption();
+ add(registeredInterpreter.getGroup(), interpreterInfo, registeredInterpreter.getProperties(),
+ option, absolutePath, registeredInterpreter.getRunner());
+ }
+
+ }
+
+ public InterpreterSetting getDefaultInterpreterSetting(List<InterpreterSetting> settings) {
+ if (settings == null || settings.isEmpty()) {
+ return null;
+ }
+ return settings.get(0);
+ }
+
+ public InterpreterSetting getDefaultInterpreterSetting(String noteId) {
+ return getDefaultInterpreterSetting(getInterpreterSettings(noteId));
+ }
+
+ public List<InterpreterSetting> getInterpreterSettings(String noteId) {
+ List<String> interpreterSettingIds = getNoteInterpreterSettingBinding(noteId);
+ LinkedList<InterpreterSetting> settings = new LinkedList<>();
+
+ Iterator<String> iter = interpreterSettingIds.iterator();
+ while (iter.hasNext()) {
+ String id = iter.next();
+ InterpreterSetting setting = get(id);
+ if (setting == null) {
+ // interpreter setting is removed from factory. remove id from here, too
+ iter.remove();
+ } else {
+ settings.add(setting);
+ }
+ }
+ return settings;
+ }
+
+ private List<String> getNoteInterpreterSettingBinding(String noteId) {
+ LinkedList<String> bindings = new LinkedList<>();
+ synchronized (interpreterSettings) {
+ List<String> settingIds = interpreterBindings.get(noteId);
+ if (settingIds != null) {
+ bindings.addAll(settingIds);
+ }
+ }
+ return bindings;
+ }
+
+ private InterpreterSetting createFromInterpreterSettingRef(String name) {
+ Preconditions.checkNotNull(name, "reference name should be not null");
+ InterpreterSetting settingRef = interpreterSettingsRef.get(name);
+ return createFromInterpreterSettingRef(settingRef);
+ }
+
+ private InterpreterSetting createFromInterpreterSettingRef(InterpreterSetting o) {
+ // should return immutable objects
+ List<InterpreterInfo> infos = (null == o.getInterpreterInfos()) ?
+ new ArrayList<InterpreterInfo>() : new ArrayList<>(o.getInterpreterInfos());
+ List<Dependency> deps = (null == o.getDependencies()) ?
+ new ArrayList<Dependency>() : new ArrayList<>(o.getDependencies());
+ Properties props =
+ convertInterpreterProperties((Map<String, InterpreterProperty>) o.getProperties());
+ InterpreterOption option = InterpreterOption.fromInterpreterOption(o.getOption());
+
+ InterpreterSetting setting = new InterpreterSetting(o.getName(), o.getName(),
+ infos, props, deps, option, o.getPath(), o.getInterpreterRunner());
+ setting.setInterpreterGroupFactory(interpreterGroupFactory);
+ return setting;
+ }
+
+ private Properties convertInterpreterProperties(Map<String, InterpreterProperty> p) {
+ Properties properties = new Properties();
+ for (String key : p.keySet()) {
+ properties.put(key, p.get(key).getValue());
+ }
+ return properties;
+ }
+
+ public Map<String, Object> getEditorSetting(Interpreter interpreter, String user, String noteId,
+ String replName) {
+ Map<String, Object> editor = DEFAULT_EDITOR;
+ String group = StringUtils.EMPTY;
+ try {
+ String defaultSettingName = getDefaultInterpreterSetting(noteId).getName();
+ List<InterpreterSetting> intpSettings = getInterpreterSettings(noteId);
+ for (InterpreterSetting intpSetting : intpSettings) {
+ String[] replNameSplit = replName.split("\\.");
+ if (replNameSplit.length == 2) {
+ group = replNameSplit[0];
+ }
+ // when replName is 'name' of interpreter
+ if (defaultSettingName.equals(intpSetting.getName())) {
+ editor = getEditorFromSettingByClassName(intpSetting, interpreter.getClassName());
+ }
+ // when replName is 'alias name' of interpreter or 'group' of interpreter
+ if (replName.equals(intpSetting.getName()) || group.equals(intpSetting.getName())) {
+ editor = getEditorFromSettingByClassName(intpSetting, interpreter.getClassName());
+ break;
+ }
+ }
+ } catch (NullPointerException e) {
+ logger.warn("Couldn't get interpreter editor setting");
+ }
+ return editor;
+ }
+
+ public Map<String, Object> getEditorFromSettingByClassName(InterpreterSetting intpSetting,
+ String className) {
+ List<InterpreterInfo> intpInfos = intpSetting.getInterpreterInfos();
+ for (InterpreterInfo intpInfo : intpInfos) {
+
+ if (className.equals(intpInfo.getClassName())) {
+ if (intpInfo.getEditor() == null) {
+ break;
+ }
+ return intpInfo.getEditor();
+ }
+ }
+ return DEFAULT_EDITOR;
+ }
+
+ private void loadInterpreterDependencies(final InterpreterSetting setting) {
+ setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
+ setting.setErrorReason(null);
+ interpreterSettings.put(setting.getId(), setting);
+ synchronized (interpreterSettings) {
+ final Thread t = new Thread() {
+ public void run() {
+ try {
+ // dependencies to prevent library conflict
+ File localRepoDir = new File(zeppelinConfiguration.getInterpreterLocalRepoPath() + "/" +
+ setting.getId());
+ if (localRepoDir.exists()) {
+ try {
+ FileUtils.cleanDirectory(localRepoDir);
+ } catch (FileNotFoundException e) {
+ logger.info("A file that does not exist cannot be deleted, nothing to worry", e);
+ }
+ }
+
+ // load dependencies
+ List<Dependency> deps = setting.getDependencies();
+ if (deps != null) {
+ for (Dependency d : deps) {
+ File destDir = new File(
+ zeppelinConfiguration.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
+
+ if (d.getExclusions() != null) {
+ dependencyResolver.load(d.getGroupArtifactVersion(), d.getExclusions(),
+ new File(destDir, setting.getId()));
+ } else {
+ dependencyResolver
+ .load(d.getGroupArtifactVersion(), new File(destDir, setting.getId()));
+ }
+ }
+ }
+
+ setting.setStatus(InterpreterSetting.Status.READY);
+ setting.setErrorReason(null);
+ } catch (Exception e) {
+ logger.error(String.format("Error while downloading repos for interpreter group : %s," +
+ " go to interpreter setting page click on edit and save it again to make " +
+ "this interpreter work properly. : %s",
+ setting.getGroup(), e.getLocalizedMessage()), e);
+ setting.setErrorReason(e.getLocalizedMessage());
+ setting.setStatus(InterpreterSetting.Status.ERROR);
+ } finally {
+ interpreterSettings.put(setting.getId(), setting);
+ }
+ }
+ };
+ t.start();
+ }
+ }
+
+ /**
+ * Overwrite dependency jar under local-repo/{interpreterId}
+ * if jar file in original path is changed
+ */
+ private void copyDependenciesFromLocalPath(final InterpreterSetting setting) {
+ setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
+ interpreterSettings.put(setting.getId(), setting);
+ synchronized (interpreterSettings) {
+ final Thread t = new Thread() {
+ public void run() {
+ try {
+ List<Dependency> deps = setting.getDependencies();
+ if (deps != null) {
+ for (Dependency d : deps) {
+ File destDir = new File(
+ zeppelinConfiguration.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
+
+ int numSplits = d.getGroupArtifactVersion().split(":").length;
+ if (!(numSplits >= 3 && numSplits <= 6)) {
+ dependencyResolver.copyLocalDependency(d.getGroupArtifactVersion(),
+ new File(destDir, setting.getId()));
+ }
+ }
+ }
+ setting.setStatus(InterpreterSetting.Status.READY);
+ } catch (Exception e) {
+ logger.error(String.format("Error while copying deps for interpreter group : %s," +
+ " go to interpreter setting page click on edit and save it again to make " +
+ "this interpreter work properly.",
+ setting.getGroup()), e);
+ setting.setErrorReason(e.getLocalizedMessage());
+ setting.setStatus(InterpreterSetting.Status.ERROR);
+ } finally {
+ interpreterSettings.put(setting.getId(), setting);
+ }
+ }
+ };
+ t.start();
+ }
+ }
+
+ /**
+ * Return ordered interpreter setting list.
+ * The list does not contain more than one setting from the same interpreter class.
+ * Order by InterpreterClass (order defined by ZEPPELIN_INTERPRETERS), Interpreter setting name
+ */
+ public List<String> getDefaultInterpreterSettingList() {
+ // this list will contain default interpreter setting list
+ List<String> defaultSettings = new LinkedList<>();
+
+ // to ignore the same interpreter group
+ Map<String, Boolean> interpreterGroupCheck = new HashMap<>();
+
+ List<InterpreterSetting> sortedSettings = get();
+
+ for (InterpreterSetting setting : sortedSettings) {
+ if (defaultSettings.contains(setting.getId())) {
+ continue;
+ }
+
+ if (!interpreterGroupCheck.containsKey(setting.getName())) {
+ defaultSettings.add(setting.getId());
+ interpreterGroupCheck.put(setting.getName(), true);
+ }
+ }
+ return defaultSettings;
+ }
+
+ List<RegisteredInterpreter> getRegisteredInterpreterList() {
+ return new ArrayList<>(Interpreter.registeredInterpreters.values());
+ }
+
+
+ private boolean findDefaultInterpreter(List<InterpreterInfo> infos) {
+ for (InterpreterInfo interpreterInfo : infos) {
+ if (interpreterInfo.isDefaultInterpreter()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public InterpreterSetting createNewSetting(String name, String group,
+ List<Dependency> dependencies, InterpreterOption option, Properties p) throws IOException {
+ if (name.indexOf(".") >= 0) {
+ throw new IOException("'.' is invalid for InterpreterSetting name.");
+ }
+ InterpreterSetting setting = createFromInterpreterSettingRef(group);
+ setting.setName(name);
+ setting.setGroup(group);
+ setting.appendDependencies(dependencies);
+ setting.setInterpreterOption(option);
+ setting.setProperties(p);
+ setting.setInterpreterGroupFactory(interpreterGroupFactory);
+ interpreterSettings.put(setting.getId(), setting);
+ loadInterpreterDependencies(setting);
+ saveToFile();
+ return setting;
+ }
+
+ private InterpreterSetting add(String group, InterpreterInfo interpreterInfo,
+ Map<String, InterpreterProperty> interpreterProperties, InterpreterOption option, String path,
+ InterpreterRunner runner)
+ throws InterpreterException, IOException, RepositoryException {
+ ArrayList<InterpreterInfo> infos = new ArrayList<>();
+ infos.add(interpreterInfo);
+ return add(group, infos, new ArrayList<Dependency>(), option, interpreterProperties, path,
+ runner);
+ }
+
+ /**
+ * @param group InterpreterSetting reference name
+ */
+ public InterpreterSetting add(String group, ArrayList<InterpreterInfo> interpreterInfos,
+ List<Dependency> dependencies, InterpreterOption option,
+ Map<String, InterpreterProperty> interpreterProperties, String path,
+ InterpreterRunner runner) {
+ Preconditions.checkNotNull(group, "name should not be null");
+ Preconditions.checkNotNull(interpreterInfos, "interpreterInfos should not be null");
+ Preconditions.checkNotNull(dependencies, "dependencies should not be null");
+ Preconditions.checkNotNull(option, "option should not be null");
+ Preconditions.checkNotNull(interpreterProperties, "properties should not be null");
+
+ InterpreterSetting interpreterSetting;
+
+ synchronized (interpreterSettingsRef) {
+ if (interpreterSettingsRef.containsKey(group)) {
+ interpreterSetting = interpreterSettingsRef.get(group);
+
+ // Append InterpreterInfo
+ List<InterpreterInfo> infos = interpreterSetting.getInterpreterInfos();
+ boolean hasDefaultInterpreter = findDefaultInterpreter(infos);
+ for (InterpreterInfo interpreterInfo : interpreterInfos) {
+ if (!infos.contains(interpreterInfo)) {
+ if (!hasDefaultInterpreter && interpreterInfo.isDefaultInterpreter()) {
+ hasDefaultInterpreter = true;
+ infos.add(0, interpreterInfo);
+ } else {
+ infos.add(interpreterInfo);
+ }
+ }
+ }
+
+ // Append dependencies
+ List<Dependency> dependencyList = interpreterSetting.getDependencies();
+ for (Dependency dependency : dependencies) {
+ if (!dependencyList.contains(dependency)) {
+ dependencyList.add(dependency);
+ }
+ }
+
+ // Append properties
+ Map<String, InterpreterProperty> properties =
+ (Map<String, InterpreterProperty>) interpreterSetting.getProperties();
+ for (String key : interpreterProperties.keySet()) {
+ if (!properties.containsKey(key)) {
+ properties.put(key, interpreterProperties.get(key));
+ }
+ }
+
+ } else {
+ interpreterSetting =
+ new InterpreterSetting(group, null, interpreterInfos, interpreterProperties,
+ dependencies, option, path, runner);
+ interpreterSettingsRef.put(group, interpreterSetting);
+ }
+ }
+
+ if (dependencies.size() > 0) {
+ loadInterpreterDependencies(interpreterSetting);
+ }
+
+ interpreterSetting.setInterpreterGroupFactory(interpreterGroupFactory);
+ return interpreterSetting;
+ }
+
+ /**
+ * map interpreter ids into noteId
+ *
+ * @param noteId note id
+ * @param ids InterpreterSetting id list
+ */
+ public void setInterpreters(String user, String noteId, List<String> ids) throws IOException {
+ putNoteInterpreterSettingBinding(user, noteId, ids);
+ }
+
+ private void putNoteInterpreterSettingBinding(String user, String noteId,
+ List<String> settingList) throws IOException {
+ List<String> unBindedSettings = new LinkedList<>();
+
+ synchronized (interpreterSettings) {
+ List<String> oldSettings = interpreterBindings.get(noteId);
+ if (oldSettings != null) {
+ for (String oldSettingId : oldSettings) {
+ if (!settingList.contains(oldSettingId)) {
+ unBindedSettings.add(oldSettingId);
+ }
+ }
+ }
+ interpreterBindings.put(noteId, settingList);
+ saveToFile();
+
+ for (String settingId : unBindedSettings) {
+ InterpreterSetting setting = get(settingId);
+ removeInterpretersForNote(setting, user, noteId);
+ }
+ }
+ }
+
+ public void removeInterpretersForNote(InterpreterSetting interpreterSetting, String user,
+ String noteId) {
+ InterpreterOption option = interpreterSetting.getOption();
+ if (option.isProcess()) {
+ interpreterSetting.closeAndRemoveInterpreterGroupByNoteId(noteId);
+ } else if (option.isSession()) {
+ InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId);
+ String key = getInterpreterSessionKey(user, noteId, interpreterSetting);
+ interpreterGroup.close(key);
+ synchronized (interpreterGroup) {
+ interpreterGroup.remove(key);
+ interpreterGroup.notifyAll(); // notify createInterpreterForNote()
+ }
+ logger.info("Interpreter instance {} for note {} is removed", interpreterSetting.getName(),
+ noteId);
+ }
+ }
+
+ public String getInterpreterSessionKey(String user, String noteId, InterpreterSetting setting) {
+ InterpreterOption option = setting.getOption();
+ String key;
+ if (option.isExistingProcess()) {
+ key = Constants.EXISTING_PROCESS;
+ } else if (option.perNoteScoped() && option.perUserScoped()) {
+ key = user + ":" + noteId;
+ } else if (option.perUserScoped()) {
+ key = user;
+ } else if (option.perNoteScoped()) {
+ key = noteId;
+ } else {
+ key = SHARED_SESSION;
+ }
+
+ logger.debug("Interpreter session key: {}, for note: {}, user: {}, InterpreterSetting Name: " +
+ "{}", key, noteId, user, setting.getName());
+ return key;
+ }
+
+
+ public List<String> getInterpreters(String noteId) {
+ return getNoteInterpreterSettingBinding(noteId);
+ }
+
+ public void closeNote(String user, String noteId) {
+ // close interpreters in this note session
+ List<InterpreterSetting> settings = getInterpreterSettings(noteId);
+ if (settings == null || settings.size() == 0) {
+ return;
+ }
+
+ logger.info("closeNote: {}", noteId);
+ for (InterpreterSetting setting : settings) {
+ removeInterpretersForNote(setting, user, noteId);
+ }
+ }
+
+ public Map<String, InterpreterSetting> getAvailableInterpreterSettings() {
+ return interpreterSettingsRef;
+ }
+
+ private URL[] recursiveBuildLibList(File path) throws MalformedURLException {
+ URL[] urls = new URL[0];
+ if (path == null || !path.exists()) {
+ return urls;
+ } else if (path.getName().startsWith(".")) {
+ return urls;
+ } else if (path.isDirectory()) {
+ File[] files = path.listFiles();
+ if (files != null) {
+ for (File f : files) {
+ urls = (URL[]) ArrayUtils.addAll(urls, recursiveBuildLibList(f));
+ }
+ }
+ return urls;
+ } else {
+ return new URL[]{path.toURI().toURL()};
+ }
+ }
+
+ public List<RemoteRepository> getRepositories() {
+ return this.interpreterRepositories;
+ }
+
+ public void addRepository(String id, String url, boolean snapshot, Authentication auth,
+ Proxy proxy) throws IOException {
+ dependencyResolver.addRepo(id, url, snapshot, auth, proxy);
+ saveToFile();
+ }
+
+ public void removeRepository(String id) throws IOException {
+ dependencyResolver.delRepo(id);
+ saveToFile();
+ }
+
+ public void removeNoteInterpreterSettingBinding(String user, String noteId) {
+ synchronized (interpreterSettings) {
+ List<String> settingIds = (interpreterBindings.containsKey(noteId) ?
+ interpreterBindings.remove(noteId) :
+ Collections.<String>emptyList());
+ for (String settingId : settingIds) {
+ this.removeInterpretersForNote(get(settingId), user, noteId);
+ }
+ }
+ }
+
+ /**
+ * Change interpreter property and restart
+ */
+ public void setPropertyAndRestart(String id, InterpreterOption option, Properties properties,
+ List<Dependency> dependencies) throws IOException {
+ synchronized (interpreterSettings) {
+ InterpreterSetting intpSetting = interpreterSettings.get(id);
+ if (intpSetting != null) {
+ try {
+ stopJobAllInterpreter(intpSetting);
+
+ intpSetting.closeAndRemoveAllInterpreterGroups();
+ intpSetting.setOption(option);
+ intpSetting.setProperties(properties);
+ intpSetting.setDependencies(dependencies);
+ loadInterpreterDependencies(intpSetting);
+
+ saveToFile();
+ } catch (Exception e) {
+ throw e;
+ } finally {
+ loadFromFile();
+ }
+ } else {
+ throw new InterpreterException("Interpreter setting id " + id + " not found");
+ }
+ }
+ }
+
+ public void restart(String settingId, String noteId, String user) {
+ InterpreterSetting intpSetting = interpreterSettings.get(settingId);
+ Preconditions.checkNotNull(intpSetting);
+
+ // restart interpreter setting in note page
+ if (noteIdIsExist(noteId) && intpSetting.getOption().isProcess()) {
+ intpSetting.closeAndRemoveInterpreterGroupByNoteId(noteId);
+ return;
+ } else {
+ // restart interpreter setting in interpreter setting page
+ restart(settingId, user);
+ }
+
+ }
+
+ private boolean noteIdIsExist(String noteId) {
+ return noteId == null ? false : true;
+ }
+
+ public void restart(String id, String user) {
+ synchronized (interpreterSettings) {
+ InterpreterSetting intpSetting = interpreterSettings.get(id);
+ // Check if dependency in specified path is changed
+ // If it did, overwrite old dependency jar with new one
+ if (intpSetting != null) {
+ //clean up metaInfos
+ intpSetting.setInfos(null);
+ copyDependenciesFromLocalPath(intpSetting);
+
+ stopJobAllInterpreter(intpSetting);
+ if (user.equals("anonymous")) {
+ intpSetting.closeAndRemoveAllInterpreterGroups();
+ } else {
+ intpSetting.closeAndRemoveInterpreterGroupByUser(user);
+ }
+
+ } else {
+ throw new InterpreterException("Interpreter setting id " + id + " not found");
+ }
+ }
+ }
+
+ public void restart(String id) {
+ restart(id, "anonymous");
+ }
+
+ private void stopJobAllInterpreter(InterpreterSetting intpSetting) {
+ if (intpSetting != null) {
+ for (InterpreterGroup intpGroup : intpSetting.getAllInterpreterGroups()) {
+ for (List<Interpreter> interpreters : intpGroup.values()) {
+ for (Interpreter intp : interpreters) {
+ for (Job job : intp.getScheduler().getJobsRunning()) {
+ job.abort();
+ job.setStatus(Status.ABORT);
+ logger.info("Job " + job.getJobName() + " aborted ");
+ }
+ for (Job job : intp.getScheduler().getJobsWaiting()) {
+ job.abort();
+ job.setStatus(Status.ABORT);
+ logger.info("Job " + job.getJobName() + " aborted ");
+ }
+ }
+ }
+ }
+ }
+ }
+
+ public InterpreterSetting get(String name) {
+ synchronized (interpreterSettings) {
+ return interpreterSettings.get(name);
+ }
+ }
+
+ public void remove(String id) throws IOException {
+ synchronized (interpreterSettings) {
+ if (interpreterSettings.containsKey(id)) {
+ InterpreterSetting intp = interpreterSettings.get(id);
+ intp.closeAndRemoveAllInterpreterGroups();
+
+ interpreterSettings.remove(id);
+ for (List<String> settings : interpreterBindings.values()) {
+ Iterator<String> it = settings.iterator();
+ while (it.hasNext()) {
+ String settingId = it.next();
+ if (settingId.equals(id)) {
+ it.remove();
+ }
+ }
+ }
+ saveToFile();
+ }
+ }
+
+ File localRepoDir = new File(zeppelinConfiguration.getInterpreterLocalRepoPath() + "/" + id);
+ FileUtils.deleteDirectory(localRepoDir);
+ }
+
+ /**
+ * Get interpreter settings
+ */
+ public List<InterpreterSetting> get() {
+ synchronized (interpreterSettings) {
+ List<InterpreterSetting> orderedSettings = new LinkedList<>();
+
+ Map<String, List<InterpreterSetting>> nameInterpreterSettingMap = new HashMap<>();
+ for (InterpreterSetting interpreterSetting : interpreterSettings.values()) {
+ String group = interpreterSetting.getGroup();
+ if (!nameInterpreterSettingMap.containsKey(group)) {
+ nameInterpreterSettingMap.put(group, new ArrayList<InterpreterSetting>());
+ }
+ nameInterpreterSettingMap.get(group).add(interpreterSetting);
+ }
+
+ for (String groupName : interpreterGroupOrderList) {
+ List<InterpreterSetting> interpreterSettingList =
+ nameInterpreterSettingMap.remove(groupName);
+ if (null != interpreterSettingList) {
+ for (InterpreterSetting interpreterSetting : interpreterSettingList) {
+ orderedSettings.add(interpreterSetting);
+ }
+ }
+ }
+
+ List<InterpreterSetting> settings = new ArrayList<>();
+
+ for (List<InterpreterSetting> interpreterSettingList : nameInterpreterSettingMap.values()) {
+ for (InterpreterSetting interpreterSetting : interpreterSettingList) {
+ settings.add(interpreterSetting);
+ }
+ }
+
+ Collections.sort(settings, new Comparator<InterpreterSetting>() {
+ @Override
+ public int compare(InterpreterSetting o1, InterpreterSetting o2) {
+ return o1.getName().compareTo(o2.getName());
+ }
+ });
+
+ orderedSettings.addAll(settings);
+
+ return orderedSettings;
+ }
+ }
+
+ public void close() {
+ List<Thread> closeThreads = new LinkedList<>();
+ synchronized (interpreterSettings) {
+ Collection<InterpreterSetting> intpSettings = interpreterSettings.values();
+ for (final InterpreterSetting intpSetting : intpSettings) {
+ Thread t = new Thread() {
+ public void run() {
+ intpSetting.closeAndRemoveAllInterpreterGroups();
+ }
+ };
+ t.start();
+ closeThreads.add(t);
+ }
+ }
+
+ for (Thread t : closeThreads) {
+ try {
+ t.join();
+ } catch (InterruptedException e) {
+ logger.error("Can't close interpreterGroup", e);
+ }
+ }
+ }
+
+ public void shutdown() {
+ List<Thread> closeThreads = new LinkedList<>();
+ synchronized (interpreterSettings) {
+ Collection<InterpreterSetting> intpSettings = interpreterSettings.values();
+ for (final InterpreterSetting intpSetting : intpSettings) {
+ Thread t = new Thread() {
+ public void run() {
+ intpSetting.shutdownAndRemoveAllInterpreterGroups();
+ }
+ };
+ t.start();
+ closeThreads.add(t);
+ }
+ }
+
+ for (Thread t : closeThreads) {
+ try {
+ t.join();
+ } catch (InterruptedException e) {
+ logger.error("Can't close interpreterGroup", e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
index 73279ff..35f32f3 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
@@ -77,6 +77,7 @@ public class Note implements Serializable, ParagraphJobListener {
private Map<String, List<AngularObject>> angularObjects = new HashMap<>();
private transient InterpreterFactory factory;
+ private transient InterpreterSettingManager interpreterSettingManager;
private transient JobListenerFactory jobListenerFactory;
private transient NotebookRepo repo;
private transient SearchService index;
@@ -101,10 +102,12 @@ public class Note implements Serializable, ParagraphJobListener {
public Note() {
}
- public Note(NotebookRepo repo, InterpreterFactory factory, JobListenerFactory jlFactory,
+ public Note(NotebookRepo repo, InterpreterFactory factory,
+ InterpreterSettingManager interpreterSettingManager, JobListenerFactory jlFactory,
SearchService noteIndex, Credentials credentials, NoteEventListener noteEventListener) {
this.repo = repo;
this.factory = factory;
+ this.interpreterSettingManager = interpreterSettingManager;
this.jobListenerFactory = jlFactory;
this.index = noteIndex;
this.noteEventListener = noteEventListener;
@@ -117,7 +120,7 @@ public class Note implements Serializable, ParagraphJobListener {
}
private String getDefaultInterpreterName() {
- InterpreterSetting setting = factory.getDefaultInterpreterSetting(getId());
+ InterpreterSetting setting = interpreterSettingManager.getDefaultInterpreterSetting(getId());
return null != setting ? setting.getName() : StringUtils.EMPTY;
}
@@ -220,6 +223,15 @@ public class Note implements Serializable, ParagraphJobListener {
}
}
+ void setInterpreterSettingManager(InterpreterSettingManager interpreterSettingManager) {
+ this.interpreterSettingManager = interpreterSettingManager;
+ synchronized (paragraphs) {
+ for (Paragraph p : paragraphs) {
+ p.setInterpreterSettingManager(interpreterSettingManager);
+ }
+ }
+ }
+
public void initializeJobListenerForParagraph(Paragraph paragraph) {
final Note paragraphNote = paragraph.getNote();
if (!paragraphNote.getId().equals(this.getId())) {
@@ -272,7 +284,7 @@ public class Note implements Serializable, ParagraphJobListener {
* Add paragraph last.
*/
public Paragraph addParagraph(AuthenticationInfo authenticationInfo) {
- Paragraph p = new Paragraph(this, this, factory);
+ Paragraph p = new Paragraph(this, this, factory, interpreterSettingManager);
p.setAuthenticationInfo(authenticationInfo);
setParagraphMagic(p, paragraphs.size());
synchronized (paragraphs) {
@@ -292,7 +304,8 @@ public class Note implements Serializable, ParagraphJobListener {
void addCloneParagraph(Paragraph srcParagraph) {
// Keep paragraph original ID
- final Paragraph newParagraph = new Paragraph(srcParagraph.getId(), this, this, factory);
+ final Paragraph newParagraph = new Paragraph(srcParagraph.getId(), this, this, factory,
+ interpreterSettingManager);
Map<String, Object> config = new HashMap<>(srcParagraph.getConfig());
Map<String, Object> param = new HashMap<>(srcParagraph.settings.getParams());
@@ -329,7 +342,7 @@ public class Note implements Serializable, ParagraphJobListener {
* @param index index of paragraphs
*/
public Paragraph insertParagraph(int index, AuthenticationInfo authenticationInfo) {
- Paragraph p = new Paragraph(this, this, factory);
+ Paragraph p = new Paragraph(this, this, factory, interpreterSettingManager);
p.setAuthenticationInfo(authenticationInfo);
setParagraphMagic(p, index);
synchronized (paragraphs) {
@@ -622,7 +635,7 @@ public class Note implements Serializable, ParagraphJobListener {
private void snapshotAngularObjectRegistry(String user) {
angularObjects = new HashMap<>();
- List<InterpreterSetting> settings = factory.getInterpreterSettings(getId());
+ List<InterpreterSetting> settings = interpreterSettingManager.getInterpreterSettings(getId());
if (settings == null || settings.size() == 0) {
return;
}
@@ -637,7 +650,7 @@ public class Note implements Serializable, ParagraphJobListener {
private void removeAllAngularObjectInParagraph(String user, String paragraphId) {
angularObjects = new HashMap<>();
- List<InterpreterSetting> settings = factory.getInterpreterSettings(getId());
+ List<InterpreterSetting> settings = interpreterSettingManager.getInterpreterSettings(getId());
if (settings == null || settings.size() == 0) {
return;
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
index 8b946f2..474f98c 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
@@ -77,6 +77,7 @@ public class Notebook implements NoteEventListener {
private SchedulerFactory schedulerFactory;
private InterpreterFactory replFactory;
+ private InterpreterSettingManager interpreterSettingManager;
/**
* Keep the order.
*/
@@ -102,13 +103,14 @@ public class Notebook implements NoteEventListener {
*/
public Notebook(ZeppelinConfiguration conf, NotebookRepo notebookRepo,
SchedulerFactory schedulerFactory, InterpreterFactory replFactory,
- JobListenerFactory jobListenerFactory, SearchService noteSearchService,
- NotebookAuthorization notebookAuthorization, Credentials credentials)
- throws IOException, SchedulerException {
+ InterpreterSettingManager interpreterSettingManager, JobListenerFactory jobListenerFactory,
+ SearchService noteSearchService, NotebookAuthorization notebookAuthorization,
+ Credentials credentials) throws IOException, SchedulerException {
this.conf = conf;
this.notebookRepo = notebookRepo;
this.schedulerFactory = schedulerFactory;
this.replFactory = replFactory;
+ this.interpreterSettingManager = interpreterSettingManager;
this.jobListenerFactory = jobListenerFactory;
this.noteSearchService = noteSearchService;
this.notebookAuthorization = notebookAuthorization;
@@ -138,7 +140,7 @@ public class Notebook implements NoteEventListener {
Preconditions.checkNotNull(subject, "AuthenticationInfo should not be null");
Note note;
if (conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING)) {
- note = createNote(replFactory.getDefaultInterpreterSettingList(), subject);
+ note = createNote(interpreterSettingManager.getDefaultInterpreterSettingList(), subject);
} else {
note = createNote(null, subject);
}
@@ -154,7 +156,7 @@ public class Notebook implements NoteEventListener {
public Note createNote(List<String> interpreterIds, AuthenticationInfo subject)
throws IOException {
Note note =
- new Note(notebookRepo, replFactory, jobListenerFactory,
+ new Note(notebookRepo, replFactory, interpreterSettingManager, jobListenerFactory,
noteSearchService, credentials, this);
note.setNoteNameListener(folders);
@@ -270,14 +272,15 @@ public class Notebook implements NoteEventListener {
throws IOException {
Note note = getNote(id);
if (note != null) {
- List<InterpreterSetting> currentBindings = replFactory.getInterpreterSettings(id);
+ List<InterpreterSetting> currentBindings =
+ interpreterSettingManager.getInterpreterSettings(id);
for (InterpreterSetting setting : currentBindings) {
if (!interpreterSettingIds.contains(setting.getId())) {
fireUnbindInterpreter(note, setting);
}
}
- replFactory.setInterpreters(user, note.getId(), interpreterSettingIds);
+ interpreterSettingManager.setInterpreters(user, note.getId(), interpreterSettingIds);
// comment out while note.getNoteReplLoader().setInterpreters(...) do the same
// replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds);
}
@@ -286,7 +289,7 @@ public class Notebook implements NoteEventListener {
List<String> getBindedInterpreterSettingsIds(String id) {
Note note = getNote(id);
if (note != null) {
- return getInterpreterFactory().getInterpreters(note.getId());
+ return interpreterSettingManager.getInterpreters(note.getId());
} else {
return new LinkedList<>();
}
@@ -295,7 +298,7 @@ public class Notebook implements NoteEventListener {
public List<InterpreterSetting> getBindedInterpreterSettings(String id) {
Note note = getNote(id);
if (note != null) {
- return replFactory.getInterpreterSettings(note.getId());
+ return interpreterSettingManager.getInterpreterSettings(note.getId());
} else {
return new LinkedList<>();
}
@@ -328,12 +331,12 @@ public class Notebook implements NoteEventListener {
note = notes.remove(id);
folders.removeNote(note);
}
- replFactory.removeNoteInterpreterSettingBinding(subject.getUser(), id);
+ interpreterSettingManager.removeNoteInterpreterSettingBinding(subject.getUser(), id);
noteSearchService.deleteIndexDocs(note);
notebookAuthorization.removeNote(id);
// remove from all interpreter instance's angular object registry
- for (InterpreterSetting settings : replFactory.get()) {
+ for (InterpreterSetting settings : interpreterSettingManager.get()) {
AngularObjectRegistry registry =
settings.getInterpreterGroup(subject.getUser(), id).getAngularObjectRegistry();
if (registry instanceof RemoteAngularObjectRegistry) {
@@ -467,6 +470,7 @@ public class Notebook implements NoteEventListener {
note.setCredentials(this.credentials);
note.setInterpreterFactory(replFactory);
+ note.setInterpreterSettingManager(interpreterSettingManager);
note.setJobListenerFactory(jobListenerFactory);
note.setNotebookRepo(notebookRepo);
@@ -509,7 +513,7 @@ public class Notebook implements NoteEventListener {
for (String name : angularObjectSnapshot.keySet()) {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
- List<InterpreterSetting> settings = replFactory.get();
+ List<InterpreterSetting> settings = interpreterSettingManager.get();
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(subject.getUser(), note.getId());
if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
@@ -753,9 +757,10 @@ public class Notebook implements NoteEventListener {
// set interpreter bind type
String interpreterGroupName = null;
- if (replFactory.getInterpreterSettings(jobNote.getId()) != null
- && replFactory.getInterpreterSettings(jobNote.getId()).size() >= 1) {
- interpreterGroupName = replFactory.getInterpreterSettings(jobNote.getId()).get(0).getName();
+ if (interpreterSettingManager.getInterpreterSettings(jobNote.getId()) != null
+ && interpreterSettingManager.getInterpreterSettings(jobNote.getId()).size() >= 1) {
+ interpreterGroupName =
+ interpreterSettingManager.getInterpreterSettings(jobNote.getId()).get(0).getName();
}
// note json object root information.
@@ -829,9 +834,10 @@ public class Notebook implements NoteEventListener {
// set interpreter bind type
String interpreterGroupName = null;
- if (replFactory.getInterpreterSettings(note.getId()) != null
- && replFactory.getInterpreterSettings(note.getId()).size() >= 1) {
- interpreterGroupName = replFactory.getInterpreterSettings(note.getId()).get(0).getName();
+ if (interpreterSettingManager.getInterpreterSettings(note.getId()) != null
+ && interpreterSettingManager.getInterpreterSettings(note.getId()).size() >= 1) {
+ interpreterGroupName =
+ interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getName();
}
// not update and not running -> pass
@@ -881,9 +887,9 @@ public class Notebook implements NoteEventListener {
logger.error(e.getMessage(), e);
}
if (releaseResource) {
- for (InterpreterSetting setting : notebook.getInterpreterFactory()
+ for (InterpreterSetting setting : notebook.getInterpreterSettingManager()
.getInterpreterSettings(note.getId())) {
- notebook.getInterpreterFactory().restart(setting.getId());
+ notebook.getInterpreterSettingManager().restart(setting.getId());
}
}
}
@@ -948,6 +954,10 @@ public class Notebook implements NoteEventListener {
return replFactory;
}
+ public InterpreterSettingManager getInterpreterSettingManager() {
+ return interpreterSettingManager;
+ }
+
public NotebookAuthorization getNotebookAuthorization() {
return notebookAuthorization;
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
index 28b6ab3..f609ecb 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
@@ -53,6 +53,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
private static Logger logger = LoggerFactory.getLogger(Paragraph.class);
private transient InterpreterFactory factory;
+ private transient InterpreterSettingManager interpreterSettingManager;
private transient Note note;
private transient AuthenticationInfo authenticationInfo;
private transient Map<String, Paragraph> userParagraphMap = Maps.newHashMap(); // personalized
@@ -84,10 +85,11 @@ public class Paragraph extends Job implements Serializable, Cloneable {
}
public Paragraph(String paragraphId, Note note, JobListener listener,
- InterpreterFactory factory) {
+ InterpreterFactory factory, InterpreterSettingManager interpreterSettingManager) {
super(paragraphId, generateId(), listener);
this.note = note;
this.factory = factory;
+ this.interpreterSettingManager = interpreterSettingManager;
title = null;
text = null;
authenticationInfo = null;
@@ -97,10 +99,12 @@ public class Paragraph extends Job implements Serializable, Cloneable {
config = new HashMap<>();
}
- public Paragraph(Note note, JobListener listener, InterpreterFactory factory) {
+ public Paragraph(Note note, JobListener listener, InterpreterFactory factory,
+ InterpreterSettingManager interpreterSettingManager) {
super(generateId(), listener);
this.note = note;
this.factory = factory;
+ this.interpreterSettingManager = interpreterSettingManager;
title = null;
text = null;
authenticationInfo = null;
@@ -249,7 +253,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
public List<InterpreterCompletion> getInterpreterCompletion() {
List<InterpreterCompletion> completion = new LinkedList();
- for (InterpreterSetting intp : factory.getInterpreterSettings(note.getId())) {
+ for (InterpreterSetting intp : interpreterSettingManager.getInterpreterSettings(note.getId())) {
List<InterpreterInfo> intInfo = intp.getInterpreterInfos();
if (intInfo.size() > 1) {
for (InterpreterInfo info : intInfo) {
@@ -292,6 +296,10 @@ public class Paragraph extends Job implements Serializable, Cloneable {
this.factory = factory;
}
+ public void setInterpreterSettingManager(InterpreterSettingManager interpreterSettingManager) {
+ this.interpreterSettingManager = interpreterSettingManager;
+ }
+
public InterpreterResult getResult() {
return (InterpreterResult) getReturn();
}
@@ -416,7 +424,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
}
private boolean noteHasInterpreters() {
- return !factory.getInterpreterSettings(note.getId()).isEmpty();
+ return !interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty();
}
private boolean interpreterHasUser(InterpreterSetting intp) {
@@ -430,7 +438,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
private InterpreterSetting getInterpreterSettingById(String id) {
InterpreterSetting setting = null;
- for (InterpreterSetting i : factory.getInterpreterSettings(note.getId())) {
+ for (InterpreterSetting i : interpreterSettingManager.getInterpreterSettings(note.getId())) {
if (id.startsWith(i.getId())) {
setting = i;
break;
@@ -504,8 +512,9 @@ public class Paragraph extends Job implements Serializable, Cloneable {
AngularObjectRegistry registry = null;
ResourcePool resourcePool = null;
- if (!factory.getInterpreterSettings(note.getId()).isEmpty()) {
- InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0);
+ if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) {
+ InterpreterSetting intpGroup =
+ interpreterSettingManager.getInterpreterSettings(note.getId()).get(0);
registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
}
@@ -532,8 +541,9 @@ public class Paragraph extends Job implements Serializable, Cloneable {
AngularObjectRegistry registry = null;
ResourcePool resourcePool = null;
- if (!factory.getInterpreterSettings(note.getId()).isEmpty()) {
- InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0);
+ if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) {
+ InterpreterSetting intpGroup =
+ interpreterSettingManager.getInterpreterSettings(note.getId()).get(0);
registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
index 2588c4c..3940fc3 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
@@ -49,6 +49,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
private SchedulerFactory schedulerFactory;
private DependencyResolver depResolver;
private InterpreterFactory factory;
+ private InterpreterSettingManager interpreterSettingManager;
private VFSNotebookRepo notebookRepo;
private Notebook notebook;
private HeliumApplicationFactory heliumAppFactory;
@@ -84,8 +85,8 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
heliumAppFactory = new HeliumApplicationFactory();
depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
- factory = new InterpreterFactory(conf,
- new InterpreterOption(true), null, null, heliumAppFactory, depResolver, false);
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
+ factory = new InterpreterFactory(conf, null, null, heliumAppFactory, depResolver, false, interpreterSettingManager);
HashMap<String, String> env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
factory.setEnv(env);
@@ -98,6 +99,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
notebookRepo,
schedulerFactory,
factory,
+ interpreterSettingManager,
this,
search,
notebookAuthorization,
@@ -112,7 +114,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
@After
public void tearDown() throws Exception {
- List<InterpreterSetting> settings = factory.get();
+ List<InterpreterSetting> settings = interpreterSettingManager.get();
for (InterpreterSetting setting : settings) {
for (InterpreterGroup intpGroup : setting.getAllInterpreterGroups()) {
intpGroup.close();
@@ -138,7 +140,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
- factory.setInterpreters("user", note1.getId(),factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters("user", note1.getId(),interpreterSettingManager.getDefaultInterpreterSettingList());
Paragraph p1 = note1.addParagraph(AuthenticationInfo.ANONYMOUS);
@@ -184,7 +186,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
- factory.setInterpreters("user", note1.getId(), factory.getDefaultInterpreterSettingList());
+ interpreterSettingManager.setInterpreters("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
Paragraph p1 = note1.addParagraph(AuthenticationInfo.ANONYMOUS);
@@ -224,7 +226,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
- notebook.bindInterpretersToNote("user", note1.getId(), factory.getDefaultInterpreterSettingList());
+ notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
Paragraph p1 = note1.addParagraph(AuthenticationInfo.ANONYMOUS);
@@ -285,7 +287,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
- notebook.bindInterpretersToNote("user", note1.getId(), factory.getDefaultInterpreterSettingList());
+ notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
String mock1IntpSettingId = null;
for (InterpreterSetting setting : notebook.getBindedInterpreterSettings(note1.getId())) {
if (setting.getName().equals("mock1")) {
@@ -312,7 +314,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
Thread.yield();
}
// when restart interpreter
- factory.restart(mock1IntpSettingId);
+ interpreterSettingManager.restart(mock1IntpSettingId);
while (app.getStatus() == ApplicationState.Status.LOADED) {
Thread.yield();
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
index 7522366..a8da7d3 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
@@ -68,6 +68,7 @@ import org.mockito.Mock;
public class InterpreterFactoryTest {
private InterpreterFactory factory;
+ private InterpreterSettingManager interpreterSettingManager;
private File tmpDir;
private ZeppelinConfiguration conf;
private InterpreterContext context;
@@ -102,13 +103,14 @@ public class InterpreterFactoryTest {
conf = new ZeppelinConfiguration();
schedulerFactory = new SchedulerFactory();
depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
- factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false);
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
+ factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
context = new InterpreterContext("note", "id", null, "title", "text", null, null, null, null, null, null, null);
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
- notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, jobListenerFactory, search,
+ notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, jobListenerFactory, search,
notebookAuthorization, null);
}
@@ -119,7 +121,7 @@ public class InterpreterFactoryTest {
@Test
public void testBasic() {
- List<InterpreterSetting> all = factory.get();
+ List<InterpreterSetting> all = interpreterSettingManager.get();
InterpreterSetting mock1Setting = null;
for (InterpreterSetting setting : all) {
if (setting.getName().equals("mock1")) {
@@ -137,17 +139,18 @@ public class InterpreterFactoryTest {
assertNotNull("get Interpreter", interpreterGroup.get("session").get(0));
// try to get unavailable interpreter
- assertNull(factory.get("unknown"));
+ assertNull(interpreterSettingManager.get("unknown"));
// restart interpreter
- factory.restart(mock1Setting.getId());
+ interpreterSettingManager.restart(mock1Setting.getId());
assertNull(mock1Setting.getInterpreterGroup("user", "sharedProcess").get("session"));
}
@Test
public void testRemoteRepl() throws Exception {
- factory = new InterpreterFactory(conf, new InterpreterOption(true), null, null, null, depResolver, false);
- List<InterpreterSetting> all = factory.get();
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
+ factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
+ List<InterpreterSetting> all = interpreterSettingManager.get();
InterpreterSetting mock1Setting = null;
for (InterpreterSetting setting : all) {
if (setting.getName().equals("mock1")) {
@@ -174,8 +177,9 @@ public class InterpreterFactoryTest {
*/
@Test
public void testRestartInterpreterInScopedMode() throws Exception {
- factory = new InterpreterFactory(conf, new InterpreterOption(true), null, null, null, depResolver, false);
- List<InterpreterSetting> all = factory.get();
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
+ factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
+ List<InterpreterSetting> all = interpreterSettingManager.get();
InterpreterSetting mock1Setting = null;
for (InterpreterSetting setting : all) {
if (setting.getName().equals("mock1")) {
@@ -210,8 +214,9 @@ public class InterpreterFactoryTest {
*/
@Test
public void testRestartInterpreterInIsolatedMode() throws Exception {
- factory = new InterpreterFactory(conf, new InterpreterOption(true), null, null, null, depResolver, false);
- List<InterpreterSetting> all = factory.get();
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
+ factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
+ List<InterpreterSetting> all = interpreterSettingManager.get();
InterpreterSetting mock1Setting = null;
for (InterpreterSetting setting : all) {
if (setting.getName().equals("mock1")) {
@@ -243,21 +248,21 @@ public class InterpreterFactoryTest {
@Test
public void testFactoryDefaultList() throws IOException, RepositoryException {
// get default settings
- List<String> all = factory.getDefaultInterpreterSettingList();
- assertTrue(factory.get().size() >= all.size());
+ List<String> all = interpreterSettingManager.getDefaultInterpreterSettingList();
+ assertTrue(interpreterSettingManager.get().size() >= all.size());
}
@Test
public void testExceptions() throws InterpreterException, IOException, RepositoryException {
- List<String> all = factory.getDefaultInterpreterSettingList();
+ List<String> all = interpreterSettingManager.getDefaultInterpreterSettingList();
// add setting with null option & properties expected nullArgumentException.class
try {
- factory.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null);
+ interpreterSettingManager.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null);
} catch(NullArgumentException e) {
assertEquals("Test null option" , e.getMessage(),new NullArgumentException("option").getMessage());
}
try {
- factory.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null);
+ interpreterSettingManager.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null);
} catch (NullArgumentException e){
assertEquals("Test null properties" , e.getMessage(),new NullArgumentException("properties").getMessage());
}
@@ -267,22 +272,23 @@ public class InterpreterFactoryTest {
@Test
public void testSaveLoad() throws IOException, RepositoryException {
// interpreter settings
- int numInterpreters = factory.get().size();
+ int numInterpreters = interpreterSettingManager.get().size();
// check if file saved
assertTrue(new File(conf.getInterpreterSettingPath()).exists());
- factory.createNewSetting("new-mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties());
- assertEquals(numInterpreters + 1, factory.get().size());
+ interpreterSettingManager.createNewSetting("new-mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties());
+ assertEquals(numInterpreters + 1, interpreterSettingManager.get().size());
- InterpreterFactory factory2 = new InterpreterFactory(conf, null, null, null, depResolver, false);
- assertEquals(numInterpreters + 1, factory2.get().size());
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
+
+ assertEquals(numInterpreters + 1, interpreterSettingManager.get().size());
}
@Test
public void testInterpreterSettingPropertyClass() throws IOException, RepositoryException {
// check if default interpreter reference's property type is map
- Map<String, InterpreterSetting> interpreterSettingRefs = factory.getAvailableInterpreterSettings();
+ Map<String, InterpreterSetting> interpreterSettingRefs = interpreterSettingManager.getAvailableInterpreterSettings();
InterpreterSetting intpSetting = interpreterSettingRefs.get("mock1");
Map<String, InterpreterProperty> intpProperties =
(Map<String, InterpreterProperty>) intpSetting.getProperties();
@@ -293,7 +299,7 @@ public class InterpreterFactoryTest {
properties.put("key1", "value1");
properties.put("key2", "value2");
- factory.createNewSetting("newMock", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), properties);
+ interpreterSettingManager.createNewSetting("newMock", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), properties);
String confFilePath = conf.getInterpreterSettingPath();
byte[] encoded = Files.readAllBytes(Paths.get(confFilePath));
@@ -312,20 +318,21 @@ public class InterpreterFactoryTest {
@Test
public void testInterpreterAliases() throws IOException, RepositoryException {
- factory = new InterpreterFactory(conf, null, null, null, depResolver, false);
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
+ factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null);
final InterpreterInfo info2 = new InterpreterInfo("className2", "name1", true, null);
- factory.add("group1", new ArrayList<InterpreterInfo>() {{
+ interpreterSettingManager.add("group1", new ArrayList<InterpreterInfo>() {{
add(info1);
}}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path1", null);
- factory.add("group2", new ArrayList<InterpreterInfo>(){{
+ interpreterSettingManager.add("group2", new ArrayList<InterpreterInfo>(){{
add(info2);
}}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path2", null);
- final InterpreterSetting setting1 = factory.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new Properties());
- final InterpreterSetting setting2 = factory.createNewSetting("test-group2", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new Properties());
+ final InterpreterSetting setting1 = interpreterSettingManager.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new Properties());
+ final InterpreterSetting setting2 = interpreterSettingManager.createNewSetting("test-group2", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new Properties());
- factory.setInterpreters("user", "note", new ArrayList<String>() {{
+ interpreterSettingManager.setInterpreters("user", "note", new ArrayList<String>() {{
add(setting1.getId());
add(setting2.getId());
}});
@@ -336,20 +343,21 @@ public class InterpreterFactoryTest {
@Test
public void testMultiUser() throws IOException, RepositoryException {
- factory = new InterpreterFactory(conf, null, null, null, depResolver, true);
+ interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
+ factory = new InterpreterFactory(conf, null, null, null, depResolver, true, interpreterSettingManager);
final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null);
- factory.add("group1", new ArrayList<InterpreterInfo>(){{
+ interpreterSettingManager.add("group1", new ArrayList<InterpreterInfo>(){{
add(info1);
}}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path1", null);
InterpreterOption perUserInterpreterOption = new InterpreterOption(true, InterpreterOption.ISOLATED, InterpreterOption.SHARED);
- final InterpreterSetting setting1 = factory.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), perUserInterpreterOption, new Properties());
+ final InterpreterSetting setting1 = interpreterSettingManager.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), perUserInterpreterOption, new Properties());
- factory.setInterpreters("user1", "note", new ArrayList<String>() {{
+ interpreterSettingManager.setInterpreters("user1", "note", new ArrayList<String>() {{
add(setting1.getId());
}});
- factory.setInterpreters("user2", "note", new ArrayList<String>() {{
+ interpreterSettingManager.setInterpreters("user2", "note", new ArrayList<String>() {{
add(setting1.getId());
}});
@@ -360,7 +368,7 @@ public class InterpreterFactoryTest {
@Test
public void testInvalidInterpreterSettingName() {
try {
- factory.createNewSetting("new.mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties());
+ interpreterSettingManager.createNewSetting("new.mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties());
fail("expect fail because of invalid InterpreterSetting Name");
} catch (IOException e) {
assertEquals("'.' is invalid for InterpreterSetting name.", e.getMessage());
@@ -371,39 +379,40 @@ public class InterpreterFactoryTest {
@Test
public void getEditorSetting() throws IOException, RepositoryException, SchedulerException {
List<String> intpIds = new ArrayList<>();
- for(InterpreterSetting intpSetting: factory.get()) {
+ for(InterpreterSetting intpSetting: interpreterSettingManager.get()) {
if (intpSetting.getName().startsWith("mock1")) {
intpIds.add(intpSetting.getId());
}
}
Note note = notebook.createNote(intpIds, new AuthenticationInfo("anonymous"));
+ Interpreter interpreter = factory.getInterpreter("user1", note.getId(), "mock11");
// get editor setting from interpreter-setting.json
- Map<String, Object> editor = factory.getEditorSetting("user1", note.getId(), "mock11");
+ Map<String, Object> editor = interpreterSettingManager.getEditorSetting(interpreter, "user1", note.getId(), "mock11");
assertEquals("java", editor.get("language"));
// when interpreter is not loaded via interpreter-setting.json
// or editor setting doesn't exit
- editor = factory.getEditorSetting("user1", note.getId(), "mock1");
+ editor = interpreterSettingManager.getEditorSetting(factory.getInterpreter("user1", note.getId(), "mock1"),"user1", note.getId(), "mock1");
assertEquals(null, editor.get("language"));
// when interpreter is not bound to note
- editor = factory.getEditorSetting("user1", note.getId(), "mock2");
+ editor = interpreterSettingManager.getEditorSetting(factory.getInterpreter("user1", note.getId(), "mock11"),"user1", note.getId(), "mock2");
assertEquals("text", editor.get("language"));
}
@Test
public void registerCustomInterpreterRunner() throws IOException {
- InterpreterFactory spyFactory = spy(factory);
+ InterpreterSettingManager spyInterpreterSettingManager = spy(interpreterSettingManager);
- doNothing().when(spyFactory).saveToFile();
+ doNothing().when(spyInterpreterSettingManager).saveToFile();
ArrayList<InterpreterInfo> interpreterInfos1 = new ArrayList<>();
interpreterInfos1.add(new InterpreterInfo("name1.class", "name1", true, Maps.<String, Object>newHashMap()));
- spyFactory.add("normalGroup1", interpreterInfos1, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, InterpreterProperty>newHashMap(), "/normalGroup1", null);
+ spyInterpreterSettingManager.add("normalGroup1", interpreterInfos1, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, InterpreterProperty>newHashMap(), "/normalGroup1", null);
- spyFactory.createNewSetting("normalGroup1", "normalGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new Properties());
+ spyInterpreterSettingManager.createNewSetting("normalGroup1", "normalGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new Properties());
ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
interpreterInfos2.add(new InterpreterInfo("name1.class", "name1", true, Maps.<String, Object>newHashMap()));
@@ -412,13 +421,13 @@ public class InterpreterFactoryTest {
when(mockInterpreterRunner.getPath()).thenReturn("custom-linux-path.sh");
- spyFactory.add("customGroup1", interpreterInfos2, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, InterpreterProperty>newHashMap(), "/customGroup1", mockInterpreterRunner);
+ spyInterpreterSettingManager.add("customGroup1", interpreterInfos2, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, InterpreterProperty>newHashMap(), "/customGroup1", mockInterpreterRunner);
- spyFactory.createNewSetting("customGroup1", "customGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new Properties());
+ spyInterpreterSettingManager.createNewSetting("customGroup1", "customGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new Properties());
- spyFactory.setInterpreters("anonymous", "noteCustome", spyFactory.getDefaultInterpreterSettingList());
+ spyInterpreterSettingManager.setInterpreters("anonymous", "noteCustome", spyInterpreterSettingManager.getDefaultInterpreterSettingList());
- spyFactory.getInterpreter("anonymous", "noteCustome", "customGroup1");
+ factory.getInterpreter("anonymous", "noteCustome", "customGroup1");
verify(mockInterpreterRunner, times(1)).getPath();
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java
index 8325e8a..27aa633 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java
@@ -19,6 +19,7 @@ package org.apache.zeppelin.notebook;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterFactory;
+import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.search.SearchService;
@@ -59,6 +60,9 @@ public class FolderTest {
@Mock
InterpreterFactory interpreterFactory;
+ @Mock
+ InterpreterSettingManager interpreterSettingManager;
+
Folder folder;
Note note1;
@@ -67,13 +71,13 @@ public class FolderTest {
@Before
public void createFolderAndNotes() {
- note1 = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
+ note1 = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener);
note1.setName("this/is/a/folder/note1");
- note2 = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
+ note2 = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener);
note2.setName("this/is/a/folder/note2");
- note3 = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
+ note3 = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener);
note3.setName("this/is/a/folder/note3");
folder = new Folder("this/is/a/folder");
@@ -114,7 +118,7 @@ public class FolderTest {
@Test
public void addNoteTest() {
- Note note4 = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
+ Note note4 = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener);
note4.setName("this/is/a/folder/note4");
folder.addNote(note4);