You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2012/12/19 22:03:52 UTC
svn commit: r1424106 - in /accumulo/trunk:
core/src/main/java/org/apache/accumulo/core/client/
core/src/main/java/org/apache/accumulo/core/conf/
core/src/main/java/org/apache/accumulo/core/iterators/
server/src/main/java/org/apache/accumulo/server/tabl...
Author: kturner
Date: Wed Dec 19 21:03:52 2012
New Revision: 1424106
URL: http://svn.apache.org/viewvc?rev=1424106&view=rev
Log:
ACCUMULO-867 initial checkin of pertable classpath
Added:
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
- copied, changed from r1424085, accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoaderTest.java
Removed:
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoader.java
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoaderTest.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java?rev=1424106&r1=1424105&r2=1424106&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java Wed Dec 19 21:03:52 2012
@@ -188,7 +188,7 @@ public class ClientSideIteratorScanner e
@Override
public void registerSideChannel(final SortedKeyValueIterator<Key,Value> iter) {}
- }, false);
+ }, false, null);
} catch (IOException e) {
throw new RuntimeException(e);
}
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1424106&r1=1424105&r2=1424106&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java Wed Dec 19 21:03:52 2012
@@ -291,11 +291,11 @@ public enum Property {
"The Formatter class to apply on results in the shell"),
TABLE_INTERPRETER_CLASS("table.interepreter", DefaultScanInterpreter.class.getName(), PropertyType.STRING,
"The ScanInterpreter class to apply on scan arguments in the shell"),
+ TABLE_CLASSPATH("table.classpath.context", "", PropertyType.STRING, "Per table classpath"),
//VFS ClassLoader properties
VFS_CLASSLOADER_PREFIX("classloader.vfs", null, PropertyType.PREFIX, "Properties in this category affect the VFS ClassLoader"),
- VFS_CLASSLOADER_ENABLED("classloader.vfs.enabled", "false", PropertyType.BOOLEAN, "Enable/disable VFS Classloader"),
VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY("classloader.vfs.context.classpath.system", "", PropertyType.STRING,
"Classpath for the system context"),
VFS_CLASSLOADER_CONTEXT_NAMES_PROPERTY("classloader.vfs.context.names", "", PropertyType.STRING,
@@ -303,6 +303,7 @@ public enum Property {
VFS_CONTEXT_CLASSPATH_PROPERTY("classloader.vfs.context.classpath.", null, PropertyType.PREFIX, "Classpath for this context");
+
private String key, defaultValue, description;
private PropertyType type;
@@ -364,7 +365,7 @@ public enum Property {
// white list prefixes
return key.startsWith(Property.TABLE_PREFIX.getKey()) || key.startsWith(Property.TSERV_PREFIX.getKey()) || key.startsWith(Property.LOGGER_PREFIX.getKey())
|| key.startsWith(Property.MASTER_PREFIX.getKey()) || key.startsWith(Property.GC_PREFIX.getKey())
- || key.startsWith(Property.MONITOR_PREFIX.getKey() + "banner.");
+ || key.startsWith(Property.MONITOR_PREFIX.getKey() + "banner.") || key.startsWith(VFS_CONTEXT_CLASSPATH_PROPERTY.getKey());
}
public static Property getPropertyByKey(String key) {
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java?rev=1424106&r1=1424105&r2=1424106&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java Wed Dec 19 21:03:52 2012
@@ -210,12 +210,13 @@ public class IteratorUtil {
}
}
- return loadIterators(source, iters, allOptions, env, useAccumuloClassLoader);
+ return loadIterators(source, iters, allOptions, env, useAccumuloClassLoader, conf.get(Property.TABLE_CLASSPATH));
}
@SuppressWarnings("unchecked")
public static <K extends WritableComparable<?>,V extends Writable> SortedKeyValueIterator<K,V> loadIterators(SortedKeyValueIterator<K,V> source,
- Collection<IterInfo> iters, Map<String,Map<String,String>> iterOpts, IteratorEnvironment env, boolean useAccumuloClassLoader) throws IOException {
+ Collection<IterInfo> iters, Map<String,Map<String,String>> iterOpts, IteratorEnvironment env, boolean useAccumuloClassLoader, String context)
+ throws IOException {
// wrap the source in a SynchronizedIterator in case any of the additional configured iterators want to use threading
SortedKeyValueIterator<K,V> prev = new SynchronizedIterator<K,V>(source);
@@ -224,7 +225,11 @@ public class IteratorUtil {
Class<? extends SortedKeyValueIterator<K,V>> clazz;
if (useAccumuloClassLoader){
- clazz = (Class<? extends SortedKeyValueIterator<K,V>>) AccumuloVFSClassLoader.loadClass(iterInfo.className, SortedKeyValueIterator.class);
+ if (context != null && !context.equals(""))
+ clazz = (Class<? extends SortedKeyValueIterator<K,V>>) AccumuloVFSClassLoader.getContextManager().loadClass(context, iterInfo.className,
+ SortedKeyValueIterator.class);
+ else
+ clazz = (Class<? extends SortedKeyValueIterator<K,V>>) AccumuloVFSClassLoader.loadClass(iterInfo.className, SortedKeyValueIterator.class);
}else{
clazz = (Class<? extends SortedKeyValueIterator<K,V>>) Class.forName(iterInfo.className).asSubclass(SortedKeyValueIterator.class);
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java?rev=1424106&r1=1424105&r2=1424106&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java Wed Dec 19 21:03:52 2012
@@ -122,6 +122,7 @@ import org.apache.accumulo.server.util.M
import org.apache.accumulo.server.util.MetadataTable.LogEntry;
import org.apache.accumulo.server.util.TabletOperations;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
import org.apache.hadoop.conf.Configuration;
@@ -1469,6 +1470,13 @@ public class Tablet {
+ " entries created)");
}
+ String contextName = acuTableConf.get(Property.TABLE_CLASSPATH);
+ if (contextName != null && !contextName.equals("")) {
+ // initialize context classloader, instead of possibly waiting for it to initialize for a scan
+ // TODO this could hang causing other tablets to fail to load
+ AccumuloVFSClassLoader.getContextManager().getClassLoader(contextName);
+ }
+
// do this last after tablet is completely setup because it
// could cause major compaction to start
datafileManager = new DatafileManager(datafiles);
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1424106&r1=1424105&r2=1424106&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java Wed Dec 19 21:03:52 2012
@@ -198,6 +198,8 @@ import org.apache.accumulo.server.zookee
import org.apache.accumulo.server.zookeeper.ZooLock;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.accumulo.start.Platform;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.start.classloader.vfs.ContextManager;
import org.apache.commons.collections.map.LRUMap;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -3052,6 +3054,68 @@ public class TabletServer extends Abstra
}
}
+ try {
+ AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.ContextConfig() {
+
+ @Override
+ public boolean isIsolated(String context) {
+ return false;
+ }
+
+ @Override
+ public Set<String> getContextURIs(String context) {
+ String key = Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + context;
+
+ Iterator<Entry<String,String>> iter = getSystemConfiguration().iterator();
+ while (iter.hasNext()) {
+ Entry<String,String> entry = iter.next();
+ if (entry.getKey().equals(key)) {
+ return new HashSet<String>(Arrays.asList(entry.getValue().split(",")));
+ }
+ }
+
+ return null;
+ }
+ });
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ // A task that cleans up unused classloader contexts
+ TimerTask contextCleaner = new TimerTask() {
+ @Override
+ public void run() {
+ ArrayList<KeyExtent> extents;
+
+ synchronized (onlineTablets) {
+ extents = new ArrayList<KeyExtent>(onlineTablets.keySet());
+ }
+
+ Set<Text> tables = new HashSet<Text>();
+
+ for (KeyExtent keyExtent : extents) {
+ tables.add(keyExtent.getTableId());
+ }
+
+ HashSet<String> contexts = new HashSet<String>();
+
+ for (Text tableid : tables) {
+ String context = getTableConfiguration(new KeyExtent(tableid, null, null)).get(Property.TABLE_CLASSPATH);
+ if (!context.equals("")) {
+ contexts.add(context);
+ }
+ }
+
+ try {
+ AccumuloVFSClassLoader.getContextManager().removeUnusedContexts(contexts);
+ } catch (IOException e) {
+ log.warn(e.getMessage(), e);
+ }
+ }
+ };
+
+ SimpleTimer.getInstance().schedule(contextCleaner, 60000, 60000);
+
FileSystemMonitor.start(getSystemConfiguration(), Property.TSERV_MONITOR_FS);
TimerTask gcDebugTask = new TimerTask() {
Modified: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java?rev=1424106&r1=1424105&r2=1424106&view=diff
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java (original)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java Wed Dec 19 21:03:52 2012
@@ -20,7 +20,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.security.SecureClassLoader;
-import java.util.Arrays;
import java.util.Enumeration;
import org.apache.commons.vfs2.FileChangeEvent;
@@ -154,7 +153,17 @@ public class AccumuloReloadingVFSClassLo
@Override
public String toString() {
- return Arrays.toString(this.files);
+ StringBuilder buf = new StringBuilder();
+
+ for (FileObject f : files) {
+ try {
+ buf.append("\t").append(f.getURL().toString()).append("\n");
+ } catch (FileSystemException e) {
+ log.error("Error getting URL for file", e);
+ }
+ }
+
+ return buf.toString();
}
}
Modified: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java?rev=1424106&r1=1424105&r2=1424106&view=diff
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java (original)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java Wed Dec 19 21:03:52 2012
@@ -74,8 +74,6 @@ public class AccumuloVFSClassLoader {
private static final Logger log = Logger.getLogger(AccumuloVFSClassLoader.class);
- public static final String VFS_CLASSLOADER_ENABLED = "classloader.vfs.enabled";
-
public static final String VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY = "classloader.vfs.context.classpath.system";
public static final String VFS_CLASSLOADER_CONTEXT_NAMES_PROPERTY = "classloader.vfs.context.names";
@@ -84,10 +82,12 @@ public class AccumuloVFSClassLoader {
private static DefaultFileSystemManager vfs = null;
private static ClassLoader parent = null;
- private static volatile AccumuloContextClassLoader loader = null;
+ private static volatile AccumuloReloadingVFSClassLoader loader = null;
private static final Object lock = new Object();
private static final Configuration ACC_CONF = new Configuration();
private static final String SITE_CONF;
+ private static ContextManager contextManager;
+
static {
String configFile = System.getProperty("org.apache.accumulo.config.file", "accumulo-site.xml");
if (System.getenv("ACCUMULO_HOME") != null) {
@@ -230,11 +230,6 @@ public class AccumuloVFSClassLoader {
synchronized (lock) {
if (null == loader) {
- if (ACC_CONF.getBoolean(VFS_CLASSLOADER_ENABLED, false) == false) {
- localLoader = AccumuloClassLoader.getClassLoader();
- return localLoader;
- }
-
if (null == vfs) {
vfs = new DefaultFileSystemManager();
//TODO: Might be able to use a different cache impl or specify cache directory in configuration.
@@ -276,18 +271,18 @@ public class AccumuloVFSClassLoader {
vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE);
vfs.init();
}
-
- //Set up the 2nd tier class loader
+
+ // Set up the 2nd tier class loader
if (null == parent)
parent = getAccumuloClassLoader();
- //Get the default context classpaths from the configuration
+ // Get the default context classpaths from the configuration
String[] defaultPaths = ACC_CONF.getStrings(VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY);
if (null == defaultPaths || defaultPaths.length == 0) {
- log.info("Default context not configured.");
localLoader = parent;
return localLoader;
}
+
ArrayList<FileObject> defaultClassPath = new ArrayList<FileObject>();
for (String path : defaultPaths) {
FileObject fo = vfs.resolveFile(path);
@@ -301,30 +296,7 @@ public class AccumuloVFSClassLoader {
}
//Create the Accumulo Context ClassLoader using the DEFAULT_CONTEXT
- loader = new AccumuloContextClassLoader(defaultClassPath.toArray(new FileObject[defaultClassPath.size()]), vfs, parent);
-
- //Add the other contexts
- String[] contexts = ACC_CONF.getStrings(VFS_CLASSLOADER_CONTEXT_NAMES_PROPERTY);
- if (null != contexts) {
- for (String context : contexts) {
- String[] contextPaths = ACC_CONF.getStrings(VFS_CONTEXT_CLASSPATH_PROPERTY + context);
- if (null != contextPaths) {
- ArrayList<FileObject> contextClassPath = new ArrayList<FileObject>();
- for (String cp : contextPaths) {
- FileObject fo = vfs.resolveFile(cp);
- if (fo.getType().equals(FileType.FILE)) {
- contextClassPath.add(fo);
- } else {
- for (FileObject child : fo.getChildren()) {
- contextClassPath.add(child);
- }
- }
- }
- log.debug("Creating Context ClassLoader for context: " + context + " using paths: " + contextClassPath.toString());
- loader.addContext(context, contextClassPath.toArray(new FileObject[contextClassPath.size()]));
- }
- }
- }
+ loader = new AccumuloReloadingVFSClassLoader(defaultClassPath.toArray(new FileObject[defaultClassPath.size()]), vfs, parent);
}
}
localLoader = loader;
@@ -335,18 +307,20 @@ public class AccumuloVFSClassLoader {
public static void printClassPath() {
try {
ClassLoader cl = getClassLoader();
- if (ACC_CONF.getBoolean(VFS_CLASSLOADER_ENABLED, false) == false) {
- //If using older classloader, then use its printClassPath method
- AccumuloClassLoader.printClassPath();
- } else if (cl instanceof URLClassLoader) {
+ if (cl instanceof URLClassLoader) {
//If VFS class loader enabled, but no contexts defined.
URLClassLoader ucl = (URLClassLoader) cl;
System.out.println("URL classpath items are: \n");
for (URL u : ucl.getURLs()) {
- System.out.println(u.toExternalForm());
+ System.out.println("\t" + u.toExternalForm());
}
- } else if (cl instanceof AccumuloContextClassLoader) {
+ } else if (cl instanceof AccumuloReloadingVFSClassLoader) {
//If VFS class loader enabled and contexts are defined
+ System.out.println("URL classpath items are: \n");
+ URLClassLoader ucl = (URLClassLoader) cl.getParent();
+ for (URL u : ucl.getURLs()) {
+ System.out.println("\t" + u.toExternalForm());
+ }
System.out.println("VFS classpaths items are:\n" + getClassLoader().toString());
} else {
System.out.println("Unknown classloader configuration");
@@ -357,9 +331,18 @@ public class AccumuloVFSClassLoader {
}
+ public static synchronized ContextManager getContextManager() throws IOException {
+ // TODO is there problem with using this lck?
+
+ if (contextManager == null) {
+ contextManager = new ContextManager(vfs, getClassLoader());
+ }
+
+ return contextManager;
+ }
+
public static void close() {
if (null != vfs)
vfs.close();
}
-
}
Added: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java?rev=1424106&view=auto
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java (added)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java Wed Dec 19 21:03:52 2012
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader.vfs;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.commons.vfs2.FileType;
+
+public class ContextManager {
+
+ // there is a lock per context so that one context can initialize w/o blocking another context
+ private class Context {
+ AccumuloReloadingVFSClassLoader loader;
+ Set<String> uris;
+ boolean closed = false;
+
+ Context(Set<String> uris) {
+ this.uris = new HashSet<String>(uris);
+ }
+
+ synchronized AccumuloReloadingVFSClassLoader getClassLoader() throws FileSystemException {
+ if (closed)
+ return null;
+
+ if (loader == null) {
+ ArrayList<FileObject> defaultClassPath = new ArrayList<FileObject>();
+ for (String path : uris) {
+ FileObject fo = vfs.resolveFile(path);
+ if (fo.getType().equals(FileType.FILE)) {
+ defaultClassPath.add(fo);
+ } else {
+ for (FileObject child : fo.getChildren()) {
+ defaultClassPath.add(child);
+ }
+ }
+ }
+
+ loader = new AccumuloReloadingVFSClassLoader(defaultClassPath.toArray(new FileObject[0]), vfs, parent);
+ }
+
+ return loader;
+ }
+
+ synchronized void close() {
+ closed = true;
+ loader.close();
+ loader = null;
+ }
+ }
+
+ private Map<String,Context> contexts = new HashMap<String,Context>();
+
+ private volatile ContextConfig config;
+ private FileSystemManager vfs;
+ private ClassLoader parent;
+
+ ContextManager(FileSystemManager vfs, ClassLoader parent) {
+ this.vfs = vfs;
+ this.parent = parent;
+ }
+
+ public interface ContextConfig {
+ Set<String> getContextURIs(String context);
+
+ boolean isIsolated(String context);
+ }
+
+ /**
+ * configuration must be injected for ContextManager to work
+ *
+ * @param config
+ */
+ public synchronized void setContextConfig(ContextConfig config) {
+ if (this.config != null)
+ throw new IllegalStateException("Context manager config already set");
+ this.config = config;
+ }
+
+ public ClassLoader getClassLoader(String contextName) throws FileSystemException {
+
+ Set<String> uris = config.getContextURIs(contextName);
+
+ if (uris == null)
+ throw new IllegalArgumentException("Unknown context " + contextName);
+
+ Context context = null;
+ Context contextToClose = null;
+
+ synchronized (this) {
+ // only manipulate internal data structs in this sync block... avoid creating or closing classloader, reading config, etc... basically avoid operations
+ // that may block
+ context = contexts.get(context);
+
+ if (context == null) {
+ context = new Context(uris);
+ contexts.put(contextName, context);
+ } else if (!context.uris.equals(uris)) {
+ contextToClose = context;
+ context = new Context(uris);
+ contexts.put(contextName, context);
+ }
+ }
+
+ if (contextToClose != null)
+ contextToClose.close();
+
+ AccumuloReloadingVFSClassLoader loader = context.getClassLoader();
+ if (loader == null) {
+ // ooppss, context was closed by another thread, try again
+ return getClassLoader(contextName);
+ }
+
+ return loader;
+
+ }
+
+ public <U> Class<? extends U> loadClass(String context, String classname, Class<U> extension) throws ClassNotFoundException {
+ try {
+ return (Class<? extends U>) getClassLoader(context).loadClass(classname).asSubclass(extension);
+ } catch (IOException e) {
+ throw new ClassNotFoundException("IO Error loading class " + classname, e);
+ }
+ }
+
+ public void removeUnusedContexts(Set<String> inUse) {
+
+ Map<String,Context> unused;
+
+ synchronized (this) {
+ unused = new HashMap<String,Context>(contexts);
+ unused.keySet().removeAll(inUse);
+ contexts.keySet().removeAll(unused.keySet());
+ }
+
+ for (Context context : unused.values()) {
+ // close outside of lock
+ context.close();
+ }
+ }
+}
Modified: accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java?rev=1424106&r1=1424105&r2=1424106&view=diff
==============================================================================
--- accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java (original)
+++ accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java Wed Dec 19 21:03:52 2012
@@ -51,7 +51,7 @@ public class AccumuloVFSClassLoaderTest
Whitebox.setInternalState(AccumuloVFSClassLoader.class, "log", Logger.getLogger(AccumuloVFSClassLoader.class));
ClassLoader acl = AccumuloVFSClassLoader.getClassLoader();
Assert.assertTrue((acl instanceof URLClassLoader));
- Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloContextClassLoader) null);
+ Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloReloadingVFSClassLoader) null);
}
/*
@@ -67,7 +67,7 @@ public class AccumuloVFSClassLoaderTest
Whitebox.setInternalState(AccumuloVFSClassLoader.class, "log", Logger.getLogger(AccumuloVFSClassLoader.class));
ClassLoader acl = AccumuloVFSClassLoader.getClassLoader();
Assert.assertTrue((acl instanceof URLClassLoader));
- Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloContextClassLoader) null);
+ Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloReloadingVFSClassLoader) null);
// URLClassLoader ucl = (URLClassLoader) acl;
// URL[] classpath = ucl.getURLs();
// System.out.println(Arrays.toString(classpath));
@@ -97,71 +97,19 @@ public class AccumuloVFSClassLoaderTest
Whitebox.setInternalState(AccumuloVFSClassLoader.class, "lock", new Object());
Whitebox.setInternalState(AccumuloVFSClassLoader.class, "log", Logger.getLogger(AccumuloVFSClassLoader.class));
ClassLoader acl = AccumuloVFSClassLoader.getClassLoader();
- Assert.assertTrue((acl instanceof AccumuloContextClassLoader));
- AccumuloContextClassLoader accl = (AccumuloContextClassLoader) acl;
- AccumuloReloadingVFSClassLoader arvcl = accl.getClassLoader(AccumuloContextClassLoader.DEFAULT_CONTEXT);
+ Assert.assertTrue((acl instanceof AccumuloReloadingVFSClassLoader));
+ AccumuloReloadingVFSClassLoader arvcl = (AccumuloReloadingVFSClassLoader) acl;
Assert.assertEquals(1, arvcl.getFiles().length);
Assert.assertTrue(arvcl.getFiles()[0].getURL().toString().equals(AccumuloDFSBase.HDFS_URI + "/accumulo/classpath/HelloWorld.jar"));
Class<?> clazz1 = arvcl.loadClass("test.HelloWorld");
Object o1 = clazz1.newInstance();
Assert.assertEquals("Hello World!", o1.toString());
- Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloContextClassLoader) null);
+ Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloReloadingVFSClassLoader) null);
hdfs.delete(DEFAULT, true);
}
-
- @Test
- public void testAdditionalContextConfigured() throws Exception {
-
- // Create default and application1 context directory
- FileSystem hdfs = cluster.getFileSystem();
- Path DEFAULT = new Path("/accumulo/classpath");
- hdfs.mkdirs(DEFAULT);
- Path APPLICATION = new Path("/application1/classpath");
- hdfs.mkdirs(APPLICATION);
-
- // Copy jar file to DEFAULT and APPLICATION directories
- URL jarPath = this.getClass().getResource("/HelloWorld.jar");
- Path src = new Path(jarPath.toURI().toString());
- Path dst = new Path(DEFAULT, src.getName());
- hdfs.copyFromLocalFile(src, dst);
- dst = new Path(APPLICATION, src.getName());
- hdfs.copyFromLocalFile(src, dst);
-
- URL defaultDir = this.getClass().getResource("/application1");
- Configuration conf = new Configuration(hdfs.getConf());
- conf.addResource(new File(defaultDir.getPath() + "/conf/accumulo-site.xml").toURI().toURL());
- Whitebox.setInternalState(AccumuloVFSClassLoader.class, "ACC_CONF", conf);
- Whitebox.setInternalState(AccumuloVFSClassLoader.class, "lock", new Object());
- Whitebox.setInternalState(AccumuloVFSClassLoader.class, "log", Logger.getLogger(AccumuloVFSClassLoader.class));
-
- ClassLoader acl = AccumuloVFSClassLoader.getClassLoader();
- Assert.assertTrue((acl instanceof AccumuloContextClassLoader));
- AccumuloContextClassLoader accl = (AccumuloContextClassLoader) acl;
- // DEFAULT CONTEXT
- AccumuloReloadingVFSClassLoader arvcl = accl.getClassLoader(AccumuloContextClassLoader.DEFAULT_CONTEXT);
- Assert.assertEquals(1, arvcl.getFiles().length);
- Assert.assertTrue(arvcl.getFiles()[0].getURL().toString().equals(AccumuloDFSBase.HDFS_URI + "/accumulo/classpath/HelloWorld.jar"));
- Class<?> clazz1 = arvcl.loadClass("test.HelloWorld");
- Object o1 = clazz1.newInstance();
- Assert.assertEquals("Hello World!", o1.toString());
-
- // APPLICATION CONTEXT
- AccumuloReloadingVFSClassLoader arvcl2 = accl.getClassLoader("application1");
- Assert.assertEquals(1, arvcl2.getFiles().length);
- Assert.assertTrue(arvcl2.getFiles()[0].getURL().toString().equals(AccumuloDFSBase.HDFS_URI + "/application1/classpath/HelloWorld.jar"));
- Class<?> clazz2 = arvcl2.loadClass("test.HelloWorld");
- Object o2 = clazz2.newInstance();
- Assert.assertEquals("Hello World!", o2.toString());
-
- Assert.assertTrue(!o1.equals(o2));
- Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloContextClassLoader) null);
-
- hdfs.delete(DEFAULT, true);
- hdfs.delete(APPLICATION, true);
-
- }
+
@Test
public void testLoadClass() throws Exception {
@@ -191,7 +139,7 @@ public class AccumuloVFSClassLoaderTest
Object o1 = clazz1.newInstance();
Assert.assertEquals("Hello World!", o1.toString());
- Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloContextClassLoader) null);
+ Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloReloadingVFSClassLoader) null);
hdfs.delete(DEFAULT, true);
hdfs.delete(APPLICATION, true);
Copied: accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java (from r1424085, accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoaderTest.java)
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java?p2=accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java&p1=accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoaderTest.java&r1=1424085&r2=1424106&rev=1424106&view=diff
==============================================================================
--- accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoaderTest.java (original)
+++ accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java Wed Dec 19 21:03:52 2012
@@ -17,7 +17,11 @@
package org.apache.accumulo.start.classloader.vfs;
import java.net.URL;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.accumulo.start.classloader.vfs.ContextManager.ContextConfig;
import org.apache.accumulo.test.AccumuloDFSBase;
import org.apache.commons.vfs2.FileObject;
import org.apache.hadoop.fs.FileSystem;
@@ -27,13 +31,12 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-public class AccumuloContextClassLoaderTest extends AccumuloDFSBase {
+public class ContextManagerTest extends AccumuloDFSBase {
private static final Path TEST_DIR = new Path(HDFS_URI + "/test-dir");
private static final Path TEST_DIR2 = new Path(HDFS_URI + "/test-dir2");
private FileSystem hdfs = null;
- private AccumuloContextClassLoader cl = null;
@Before
public void setup() throws Exception {
@@ -54,33 +57,54 @@ public class AccumuloContextClassLoaderT
@Test
public void differentContexts() throws Exception {
+
+ ContextManager cm = new ContextManager(vfs, ClassLoader.getSystemClassLoader());
+ cm.setContextConfig(new ContextConfig() {
+ @Override
+ public Set<String> getContextURIs(String context) {
+ if (context.equals("CX1")) {
+ return new HashSet<String>(Arrays.asList(new Path(TEST_DIR, "HelloWorld.jar").toUri().toString()));
+ } else if (context.equals("CX2")) {
+ return new HashSet<String>(Arrays.asList(new Path(TEST_DIR2, "HelloWorld.jar").toUri().toString()));
+ }
+ return null;
+ }
+
+ @Override
+ public boolean isIsolated(String context) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+ });
+
FileObject testDir = vfs.resolveFile(TEST_DIR.toUri().toString());
FileObject[] dirContents = testDir.getChildren();
- cl = new AccumuloContextClassLoader(dirContents, vfs, ClassLoader.getSystemClassLoader());
- FileObject[] files = cl.getClassLoader(AccumuloContextClassLoader.DEFAULT_CONTEXT).getFiles();
+ ClassLoader cl1 = cm.getClassLoader("CX1");
+ FileObject[] files = ((AccumuloReloadingVFSClassLoader) cl1).getFiles();
Assert.assertArrayEquals(dirContents, files);
+
FileObject testDir2 = vfs.resolveFile(TEST_DIR2.toUri().toString());
FileObject[] dirContents2 = testDir2.getChildren();
- cl.addContext("MYCONTEXT", dirContents2);
- FileObject[] files2 = cl.getClassLoader("MYCONTEXT").getFiles();
+ ClassLoader cl2 = cm.getClassLoader("CX2");
+ FileObject[] files2 = ((AccumuloReloadingVFSClassLoader) cl2).getFiles();
Assert.assertArrayEquals(dirContents2, files2);
- Class<?> defaultContextClass = cl.loadClass("test.HelloWorld");
+ Class<?> defaultContextClass = cl1.loadClass("test.HelloWorld");
Object o1 = defaultContextClass.newInstance();
Assert.assertEquals("Hello World!", o1.toString());
- Class<?> myContextClass = cl.loadClass("MYCONTEXT", "test.HelloWorld");
+ Class<?> myContextClass = cl2.loadClass("test.HelloWorld");
Object o2 = myContextClass.newInstance();
Assert.assertEquals("Hello World!", o2.toString());
Assert.assertFalse(defaultContextClass.equals(myContextClass));
+ cm.removeUnusedContexts(new HashSet<String>());
}
@After
public void tearDown() throws Exception {
- cl.close();
this.hdfs.delete(TEST_DIR, true);
}