You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2013/05/01 21:48:30 UTC
svn commit: r1478165 - in /accumulo/trunk: ./ assemble/ core/ examples/
fate/src/main/java/org/apache/accumulo/fate/
fate/src/main/java/org/apache/accumulo/fate/zookeeper/ server/ src/
start/src/main/java/org/apache/accumulo/start/classloader/vfs/ star...
Author: kturner
Date: Wed May 1 19:48:23 2013
New Revision: 1478165
URL: http://svn.apache.org/r1478165
Log:
ACCUMULO-1321 Created new file vfs file replicator avoid collisions, added classloader unit test that uses two jars, added code to delete replicated files and close vfs instances
Added:
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/UniqueFileReplicator.java
- copied unchanged from r1478154, accumulo/branches/1.5/start/src/main/java/org/apache/accumulo/start/classloader/vfs/UniqueFileReplicator.java
accumulo/trunk/start/src/test/resources/HelloWorld2.jar
- copied unchanged from r1478154, accumulo/branches/1.5/start/src/test/resources/HelloWorld2.jar
Modified:
accumulo/trunk/ (props changed)
accumulo/trunk/assemble/ (props changed)
accumulo/trunk/core/ (props changed)
accumulo/trunk/examples/ (props changed)
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java (props changed)
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java (props changed)
accumulo/trunk/pom.xml (props changed)
accumulo/trunk/server/ (props changed)
accumulo/trunk/src/ (props changed)
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
Propchange: accumulo/trunk/
------------------------------------------------------------------------------
Merged /accumulo/branches/1.5:r1478154
Propchange: accumulo/trunk/assemble/
------------------------------------------------------------------------------
Merged /accumulo/branches/1.5/assemble:r1478154
Propchange: accumulo/trunk/core/
------------------------------------------------------------------------------
Merged /accumulo/branches/1.5/core:r1478154
Propchange: accumulo/trunk/examples/
------------------------------------------------------------------------------
Merged /accumulo/branches/1.5/examples:r1478154
Propchange: accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
------------------------------------------------------------------------------
Merged /accumulo/branches/1.5/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java:r1478154
Propchange: accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
------------------------------------------------------------------------------
Merged /accumulo/branches/1.5/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java:r1478154
Propchange: accumulo/trunk/pom.xml
------------------------------------------------------------------------------
Merged /accumulo/branches/1.5/pom.xml:r1478154
Propchange: accumulo/trunk/server/
------------------------------------------------------------------------------
Merged /accumulo/branches/1.5/server:r1478154
Propchange: accumulo/trunk/src/
------------------------------------------------------------------------------
Merged /accumulo/branches/1.5/src:r1478154
Modified: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java?rev=1478165&r1=1478164&r2=1478165&view=diff
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java (original)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java Wed May 1 19:48:23 2013
@@ -64,7 +64,7 @@ public class AccumuloReloadingVFSClassLo
public synchronized ClassLoader getClassLoader() {
if (cl == null || cl.getParent() != parent.getClassLoader()) {
try {
- vfs = AccumuloVFSClassLoader.generateVfs(true);
+ vfs = AccumuloVFSClassLoader.generateVfs();
files = AccumuloVFSClassLoader.resolve(vfs, uris);
log.debug("Rebuilding dynamic classloader using files- " + stringify(files));
Modified: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java?rev=1478165&r1=1478164&r2=1478165&view=diff
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java (original)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java Wed May 1 19:48:23 2013
@@ -18,10 +18,12 @@ package org.apache.accumulo.start.classl
import java.io.File;
import java.io.IOException;
+import java.lang.ref.WeakReference;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.List;
import org.apache.accumulo.start.classloader.AccumuloClassLoader;
import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider;
@@ -31,7 +33,6 @@ import org.apache.commons.vfs2.FileSyste
import org.apache.commons.vfs2.FileSystemManager;
import org.apache.commons.vfs2.FileType;
import org.apache.commons.vfs2.cache.SoftRefFilesCache;
-import org.apache.commons.vfs2.impl.DefaultFileReplicator;
import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory;
import org.apache.commons.vfs2.impl.VFSClassLoader;
@@ -70,6 +71,9 @@ public class AccumuloVFSClassLoader {
}
+ private static List<WeakReference<DefaultFileSystemManager>> vfsInstances = Collections
+ .synchronizedList(new ArrayList<WeakReference<DefaultFileSystemManager>>());
+
public static final String DYNAMIC_CLASSPATH_PROPERTY_NAME = "general.dynamic.classpaths";
public static final String DEFAULT_DYNAMIC_CLASSPATH_VALUE = "$ACCUMULO_HOME/lib/ext/[^.].*.jar\n";
@@ -80,7 +84,6 @@ public class AccumuloVFSClassLoader {
public static final String VFS_CACHE_DIR = "general.vfs.cache.dir";
- private static DefaultFileSystemManager vfs = null;
private static ClassLoader parent = null;
private static volatile ReloadingClassLoader loader = null;
private static final Object lock = new Object();
@@ -186,7 +189,7 @@ public class AccumuloVFSClassLoader {
return wrapper;
// TODO monitor time for lib/ext was 1 sec... should this be configurable? - ACCUMULO-1301
- return new AccumuloReloadingVFSClassLoader(dynamicCPath, vfs, wrapper, 1000, true);
+ return new AccumuloReloadingVFSClassLoader(dynamicCPath, generateVfs(), wrapper, 1000, true);
}
public static ClassLoader getClassLoader() throws IOException {
@@ -195,10 +198,9 @@ public class AccumuloVFSClassLoader {
synchronized (lock) {
if (null == loader) {
- if (null == vfs) {
- vfs = generateVfs(false);
- }
+ FileSystemManager vfs = generateVfs();
+
// Set up the 2nd tier class loader
if (null == parent) {
parent = AccumuloClassLoader.getClassLoader();
@@ -213,7 +215,7 @@ public class AccumuloVFSClassLoader {
}
// Create the Accumulo Context ClassLoader using the DEFAULT_CONTEXT
- localLoader = createDynamicClassloader(new VFSClassLoader(vfsCP, generateVfs(true), parent));
+ localLoader = createDynamicClassloader(new VFSClassLoader(vfsCP, vfs, parent));
loader = localLoader;
}
}
@@ -222,8 +224,8 @@ public class AccumuloVFSClassLoader {
return localLoader.getClassLoader();
}
- public static FinalCloseDefaultFileSystemManager generateVfs(boolean reloading) throws FileSystemException {
- FinalCloseDefaultFileSystemManager vfs = new FinalCloseDefaultFileSystemManager();
+ public static FileSystemManager generateVfs() throws FileSystemException {
+ DefaultFileSystemManager vfs = new FinalCloseDefaultFileSystemManager();
vfs.addProvider("res", new org.apache.commons.vfs2.provider.res.ResourceFileProvider());
vfs.addProvider("zip", new org.apache.commons.vfs2.provider.zip.ZipFileProvider());
vfs.addProvider("gz", new org.apache.commons.vfs2.provider.gzip.GzipFileProvider());
@@ -258,12 +260,13 @@ public class AccumuloVFSClassLoader {
vfs.setFileContentInfoFactory(new FileContentInfoFilenameFactory());
vfs.setFilesCache(new SoftRefFilesCache());
String cacheDirPath = AccumuloClassLoader.getAccumuloString(VFS_CACHE_DIR, "");
- File cacheDir = new File(System.getProperty("java.io.tmpdir"), (reloading? "accumulo-vfs-reloading-cache":"accumulo-vfs-cache-") + System.getProperty("user.name", "nouser"));
+ File cacheDir = new File(System.getProperty("java.io.tmpdir"), "accumulo-vfs-cache-" + System.getProperty("user.name", "nouser"));
if (!("".equals(cacheDirPath)))
cacheDir = new File(cacheDirPath);
- vfs.setReplicator(new DefaultFileReplicator(cacheDir));
+ vfs.setReplicator(new UniqueFileReplicator(cacheDir));
vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE);
vfs.init();
+ vfsInstances.add(new WeakReference<DefaultFileSystemManager>(vfs));
return vfs;
}
@@ -327,7 +330,7 @@ public class AccumuloVFSClassLoader {
public static synchronized ContextManager getContextManager() throws IOException {
if (contextManager == null) {
getClassLoader();
- contextManager = new ContextManager(vfs, new ReloadingClassLoader() {
+ contextManager = new ContextManager(generateVfs(), new ReloadingClassLoader() {
@Override
public ClassLoader getClassLoader() {
try {
@@ -343,7 +346,10 @@ public class AccumuloVFSClassLoader {
}
public static void close() {
- if (null != vfs)
- vfs.close();
+ for (WeakReference<DefaultFileSystemManager> vfsInstance : vfsInstances) {
+ DefaultFileSystemManager ref = vfsInstance.get();
+ if (ref != null)
+ ref.close();
+ }
}
}
Modified: accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java?rev=1478165&r1=1478164&r2=1478165&view=diff
==============================================================================
--- accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java (original)
+++ accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java Wed May 1 19:48:23 2013
@@ -21,7 +21,7 @@ import java.io.File;
import org.apache.commons.io.FileUtils;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
-import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
+import org.apache.commons.vfs2.FileSystemManager;
import org.apache.commons.vfs2.impl.VFSClassLoader;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -34,7 +34,7 @@ import org.junit.rules.TemporaryFolder;
public class AccumuloReloadingVFSClassLoaderTest {
private TemporaryFolder folder1 = new TemporaryFolder();
- private DefaultFileSystemManager vfs;
+ private FileSystemManager vfs;
@Before
public void setup() throws Exception {
@@ -120,6 +120,60 @@ public class AccumuloReloadingVFSClassLo
arvcl.close();
}
+
+ @Test
+ public void testModifiedClass() throws Exception {
+
+ FileObject testDir = vfs.resolveFile(folder1.getRoot().toURI().toString());
+ FileObject[] dirContents = testDir.getChildren();
+
+ AccumuloReloadingVFSClassLoader arvcl = new AccumuloReloadingVFSClassLoader(folder1.getRoot().toURI().toString(), vfs, new ReloadingClassLoader() {
+ @Override
+ public ClassLoader getClassLoader() {
+ return ClassLoader.getSystemClassLoader();
+ }
+ }, 1000, true);
+
+ FileObject[] files = ((VFSClassLoader) arvcl.getClassLoader()).getFileObjects();
+ Assert.assertArrayEquals(createFileSystems(dirContents), files);
+
+ ClassLoader loader1 = arvcl.getClassLoader();
+ Class<?> clazz1 = loader1.loadClass("test.HelloWorld");
+ Object o1 = clazz1.newInstance();
+ Assert.assertEquals("Hello World!", o1.toString());
+
+ // Check that the class is the same before the update
+ Class<?> clazz1_5 = arvcl.getClassLoader().loadClass("test.HelloWorld");
+ Assert.assertEquals(clazz1, clazz1_5);
+
+ // java does aggressive caching of jar files. When using java code to read jar files that are created in the same second, it will only see the first jar
+ // file
+ Thread.sleep(1000);
+
+ new File(folder1.getRoot(), "HelloWorld.jar").delete();
+
+ // Update the class
+ FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld2.jar"), folder1.newFile("HelloWorld.jar"));
+
+ // Wait for the monitor to notice
+ Thread.sleep(2000);
+
+
+ Class<?> clazz2 = arvcl.getClassLoader().loadClass("test.HelloWorld");
+ Object o2 = clazz2.newInstance();
+ Assert.assertEquals("Hallo Welt", o2.toString());
+
+ // This is false because they are loaded by a different classloader
+ Assert.assertFalse(clazz1.equals(clazz2));
+ Assert.assertFalse(o1.equals(o2));
+
+ Class<?> clazz3 = loader1.loadClass("test.HelloWorld");
+ Object o3 = clazz3.newInstance();
+ Assert.assertEquals("Hello World!", o3.toString());
+
+ arvcl.close();
+ }
+
@After
public void tearDown() throws Exception {
folder1.delete();
Modified: accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java?rev=1478165&r1=1478164&r2=1478165&view=diff
==============================================================================
--- accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java (original)
+++ accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java Wed May 1 19:48:23 2013
@@ -22,16 +22,10 @@ import java.util.HashSet;
import org.apache.accumulo.start.classloader.vfs.ContextManager.ContextConfig;
import org.apache.accumulo.start.classloader.vfs.ContextManager.ContextsConfig;
-import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.vfs2.CacheStrategy;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
-import org.apache.commons.vfs2.cache.DefaultFilesCache;
-import org.apache.commons.vfs2.cache.SoftRefFilesCache;
-import org.apache.commons.vfs2.impl.DefaultFileReplicator;
-import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
-import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory;
+import org.apache.commons.vfs2.FileSystemManager;
import org.apache.commons.vfs2.impl.VFSClassLoader;
import org.junit.After;
import org.junit.Assert;
@@ -43,55 +37,16 @@ public class ContextManagerTest {
private TemporaryFolder folder1 = new TemporaryFolder();
private TemporaryFolder folder2 = new TemporaryFolder();
- private DefaultFileSystemManager vfs;
+ private FileSystemManager vfs;
private URI uri1;
private URI uri2;
- static DefaultFileSystemManager getVFS() {
- DefaultFileSystemManager vfs = new DefaultFileSystemManager();
+ static FileSystemManager getVFS() {
try {
- vfs.setFilesCache(new DefaultFilesCache());
- vfs.addProvider("res", new org.apache.commons.vfs2.provider.res.ResourceFileProvider());
- vfs.addProvider("zip", new org.apache.commons.vfs2.provider.zip.ZipFileProvider());
- vfs.addProvider("gz", new org.apache.commons.vfs2.provider.gzip.GzipFileProvider());
- vfs.addProvider("ram", new org.apache.commons.vfs2.provider.ram.RamFileProvider());
- vfs.addProvider("file", new org.apache.commons.vfs2.provider.local.DefaultLocalFileProvider());
- vfs.addProvider("jar", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
- vfs.addProvider("http", new org.apache.commons.vfs2.provider.http.HttpFileProvider());
- vfs.addProvider("https", new org.apache.commons.vfs2.provider.https.HttpsFileProvider());
- vfs.addProvider("ftp", new org.apache.commons.vfs2.provider.ftp.FtpFileProvider());
- vfs.addProvider("ftps", new org.apache.commons.vfs2.provider.ftps.FtpsFileProvider());
- vfs.addProvider("war", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
- vfs.addProvider("par", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
- vfs.addProvider("ear", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
- vfs.addProvider("sar", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
- vfs.addProvider("ejb3", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
- vfs.addProvider("tmp", new org.apache.commons.vfs2.provider.temp.TemporaryFileProvider());
- vfs.addProvider("tar", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
- vfs.addProvider("tbz2", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
- vfs.addProvider("tgz", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
- vfs.addProvider("bz2", new org.apache.commons.vfs2.provider.bzip2.Bzip2FileProvider());
- vfs.addProvider("hdfs", new HdfsFileProvider());
- vfs.addExtensionMap("jar", "jar");
- vfs.addExtensionMap("zip", "zip");
- vfs.addExtensionMap("gz", "gz");
- vfs.addExtensionMap("tar", "tar");
- vfs.addExtensionMap("tbz2", "tar");
- vfs.addExtensionMap("tgz", "tar");
- vfs.addExtensionMap("bz2", "bz2");
- vfs.addMimeTypeMap("application/x-tar", "tar");
- vfs.addMimeTypeMap("application/x-gzip", "gz");
- vfs.addMimeTypeMap("application/zip", "zip");
- vfs.setFileContentInfoFactory(new FileContentInfoFilenameFactory());
- vfs.setFilesCache(new SoftRefFilesCache());
- vfs.setReplicator(new DefaultFileReplicator());
- vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE);
- vfs.init();
+ return AccumuloVFSClassLoader.generateVfs();
} catch (FileSystemException e) {
throw new RuntimeException("Error setting up VFS", e);
}
-
- return vfs;
}
@Before