You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@commons.apache.org by im...@apache.org on 2004/07/07 22:01:35 UTC
cvs commit: jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/cache SoftRefFilesCache.java
imario 2004/07/07 13:01:35
Modified: vfs/src/java/org/apache/commons/vfs/provider
AbstractFileObject.java AbstractFileSystem.java
vfs/src/test/org/apache/commons/vfs/test
AbstractProviderTestConfig.java
AbstractTestSuite.java NamingTests.java
vfs/src/java/org/apache/commons/vfs/impl
DefaultFileSystemManager.java
vfs/src/java/org/apache/commons/vfs FileSystemManager.java
vfs/src/java/org/apache/commons/vfs/cache
SoftRefFilesCache.java
Log:
SoftRefFilesCache
Close filesystem if there are no longer files cached for it.
This automatically closes the communication link to the underlaying filesystem.
Revision Changes Path
1.50 +61 -12 jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/provider/AbstractFileObject.java
Index: AbstractFileObject.java
===================================================================
RCS file: /home/cvs/jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/provider/AbstractFileObject.java,v
retrieving revision 1.49
retrieving revision 1.50
diff -u -r1.49 -r1.50
--- AbstractFileObject.java 4 Jul 2004 18:45:55 -0000 1.49
+++ AbstractFileObject.java 7 Jul 2004 20:01:35 -0000 1.50
@@ -58,7 +58,8 @@
public abstract class AbstractFileObject
implements FileObject
{
- private static final FileObject[] EMPTY_FILE_ARRAY = {};
+ // private static final FileObject[] EMPTY_FILE_ARRAY = {};
+ private static final FileName[] EMPTY_FILE_ARRAY = {};
private final FileName name;
private final AbstractFileSystem fs;
@@ -68,7 +69,11 @@
private boolean attached;
private FileType type;
private AbstractFileObject parent;
- private FileObject[] children;
+
+ // Changed to hold only the name of the children and let the object
+ // go into the global files cache
+ // private FileObject[] children;
+ private FileName[] children;
protected AbstractFileObject(final FileName name,
final AbstractFileSystem fs)
@@ -511,22 +516,24 @@
// Use cached info, if present
if (children != null)
{
- return children;
+ return resolveFiles(children);
}
// allow the filesystem to return resolved children. e.g. prefill type for webdav
+ FileObject[] childrenObjects;
try
{
- children = doListChildrenResolved();
+ childrenObjects = doListChildrenResolved();
+ children = extractNames(childrenObjects);
}
catch (Exception exc)
{
throw new FileSystemException("vfs.provider/list-children.error", new Object[]{name}, exc);
}
- if (children != null)
+ if (childrenObjects != null)
{
- return children;
+ return childrenObjects;
}
// List the children
@@ -548,15 +555,54 @@
else
{
// Create file objects for the children
- children = new FileObject[files.length];
+ // children = new FileObject[files.length];
+ children = new FileName[files.length];
for (int i = 0; i < files.length; i++)
{
final String file = files[i];
- children[i] = fs.resolveFile(name.resolveName(file, NameScope.CHILD));
+ // children[i] = fs.resolveFile(name.resolveName(file, NameScope.CHILD));
+ children[i] = name.resolveName(file, NameScope.CHILD);
}
}
- return children;
+ return resolveFiles(children);
+ }
+
+ private FileName[] extractNames(FileObject[] objects)
+ {
+ if (objects == null)
+ {
+ return null;
+ }
+
+ FileName names[] = new FileName[objects.length];
+ for (int iterObjects = 0; iterObjects < objects.length; iterObjects++)
+ {
+ names[iterObjects] = objects[iterObjects].getName();
+ }
+
+ return names;
+ }
+
+ private FileObject[] resolveFiles(FileName[] children) throws FileSystemException
+ {
+ if (children == null)
+ {
+ return null;
+ }
+
+ FileObject objects[] = new FileObject[children.length];
+ for (int iterChildren = 0; iterChildren < children.length; iterChildren++)
+ {
+ objects[iterChildren] = resolveFile(children[iterChildren]);
+ }
+
+ return objects;
+ }
+
+ private FileObject resolveFile(FileName child) throws FileSystemException
+ {
+ return fs.resolveFile(child);
}
/**
@@ -568,11 +614,13 @@
getChildren();
for (int i = 0; i < children.length; i++)
{
- final FileObject child = children[i];
+ // final FileObject child = children[i];
+ final FileName child = children[i];
// TODO - use a comparator to compare names
- if (child.getName().getBaseName().equals(name))
+ // if (child.getName().getBaseName().equals(name))
+ if (child.getBaseName().equals(name))
{
- return child;
+ return resolveFile(child);
}
}
return null;
@@ -1095,6 +1143,7 @@
{
attached = false;
type = null;
+ parent = null;
fs.fileDetached(this);
1.29 +6 -10 jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/provider/AbstractFileSystem.java
Index: AbstractFileSystem.java
===================================================================
RCS file: /home/cvs/jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/provider/AbstractFileSystem.java,v
retrieving revision 1.28
retrieving revision 1.29
diff -u -r1.28 -r1.29
--- AbstractFileSystem.java 4 Jul 2004 18:45:55 -0000 1.28
+++ AbstractFileSystem.java 7 Jul 2004 20:01:35 -0000 1.29
@@ -52,7 +52,7 @@
private final FileName rootName;
private FileObject parentLayer;
- private FileObject root;
+ // private FileObject root;
private final Collection caps = new HashSet();
/**
@@ -79,6 +79,7 @@
final FileObject parentLayer,
final FileSystemOptions fileSystemOptions)
{
+ // this.parentLayer = parentLayer;
this.parentLayer = parentLayer;
this.rootName = rootName;
this.fileSystemOptions = fileSystemOptions;
@@ -99,15 +100,7 @@
*/
public void close()
{
- try
- {
- closeCommunicationLink();
- }
- finally
- {
- // Clean-up
- files.clear(this);
- }
+ closeCommunicationLink();
}
/**
@@ -228,11 +221,14 @@
*/
public FileObject getRoot() throws FileSystemException
{
+ return resolveFile(rootName);
+ /*
if (root == null)
{
root = resolveFile(rootName);
}
return root;
+ */
}
/**
1.8 +4 -3 jakarta-commons-sandbox/vfs/src/test/org/apache/commons/vfs/test/AbstractProviderTestConfig.java
Index: AbstractProviderTestConfig.java
===================================================================
RCS file: /home/cvs/jakarta-commons-sandbox/vfs/src/test/org/apache/commons/vfs/test/AbstractProviderTestConfig.java,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -r1.7 -r1.8
--- AbstractProviderTestConfig.java 10 May 2004 20:09:44 -0000 1.7
+++ AbstractProviderTestConfig.java 7 Jul 2004 20:01:35 -0000 1.8
@@ -16,7 +16,7 @@
package org.apache.commons.vfs.test;
import org.apache.commons.vfs.FilesCache;
-import org.apache.commons.vfs.cache.DefaultFilesCache;
+import org.apache.commons.vfs.cache.SoftRefFilesCache;
import org.apache.commons.vfs.impl.DefaultFileSystemManager;
/**
@@ -42,7 +42,8 @@
{
if (cache == null)
{
- cache = new DefaultFilesCache();
+ // cache = new DefaultFilesCache();
+ cache = new SoftRefFilesCache();
}
return cache;
1.4 +157 -2 jakarta-commons-sandbox/vfs/src/test/org/apache/commons/vfs/test/AbstractTestSuite.java
Index: AbstractTestSuite.java
===================================================================
RCS file: /home/cvs/jakarta-commons-sandbox/vfs/src/test/org/apache/commons/vfs/test/AbstractTestSuite.java,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -r1.3 -r1.4
--- AbstractTestSuite.java 23 Jun 2004 18:17:44 -0000 1.3
+++ AbstractTestSuite.java 7 Jul 2004 20:01:35 -0000 1.4
@@ -27,9 +27,12 @@
import org.apache.commons.vfs.provider.local.DefaultLocalFileProvider;
import java.io.File;
+import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
+import java.util.ArrayList;
import java.util.Enumeration;
+import java.util.List;
/**
* The suite of tests for a file system.
@@ -43,7 +46,7 @@
{
private final ProviderTestConfig providerConfig;
private final String prefix;
- private final TestSuite testSuite;
+ private TestSuite testSuite;
private FileObject baseFolder;
private FileObject readFolder;
@@ -51,6 +54,9 @@
private DefaultFileSystemManager manager;
private File tempDir;
+ private Thread[] startThreadSnapshot;
+ private Thread[] endThreadSnapshot;
+
/**
* Adds the tests for a file system to this suite.
*/
@@ -122,6 +128,8 @@
protected void setUp() throws Exception
{
+ startThreadSnapshot = createThreadSnapshot();
+
// Locate the temp directory, and clean it up
tempDir = AbstractVfsTestCase.getTestDirectory("temp");
checkTempDir("Temp dir not empty before test");
@@ -167,6 +175,51 @@
protected void tearDown() throws Exception
{
+ readFolder.close();
+ writeFolder.close();
+ baseFolder.close();
+
+ readFolder = null;
+ writeFolder = null;
+ baseFolder = null;
+ testSuite = null;
+ fTest = null;
+
+ // force the SoftRefFilesChache to free all files
+ System.err.println(".");
+ System.gc();
+ Thread.sleep(1000);
+ System.err.println(".");
+ System.gc();
+ Thread.sleep(1000);
+ System.err.println(".");
+ System.gc();
+ Thread.sleep(1000);
+ System.err.println(".");
+ System.gc();
+ Thread.sleep(1000);
+
+ manager.freeUnusedResources();
+ endThreadSnapshot = createThreadSnapshot();
+
+ Thread[] diffThreadSnapshot = diffThreadSnapshot(startThreadSnapshot, endThreadSnapshot);
+ if (diffThreadSnapshot.length > 0)
+ {
+ String message = dumpThreadSnapshot(diffThreadSnapshot);
+ /*
+ if (providerConfig.checkCleanThreadState())
+ {
+ // close the manager to do a "not thread safe" release of all resources
+ // and allow the vm to shutdown
+ manager.close();
+ fail(message);
+ }
+ else
+ {
+ */
+ System.out.println(message);
+ // }
+ }
manager.close();
// Make sure temp directory is empty or gone
@@ -184,4 +237,106 @@
}
}
+ private String dumpThreadSnapshot(Thread[] threadSnapshot)
+ {
+ StringBuffer sb = new StringBuffer(256);
+ sb.append("created threads still running:\n");
+
+ Field threadTargetField = null;
+ try
+ {
+ threadTargetField = Thread.class.getDeclaredField("target");
+ threadTargetField.setAccessible(true);
+ }
+ catch (NoSuchFieldException e)
+ {
+ ;
+ }
+
+ for (int iter = 0; iter < threadSnapshot.length; iter++)
+ {
+ Thread thread = threadSnapshot[iter];
+ if (thread == null)
+ {
+ continue;
+ }
+
+ sb.append("#");
+ sb.append(iter + 1);
+ sb.append(": ");
+ sb.append(thread.getThreadGroup().getName());
+ sb.append("\t");
+ sb.append(thread.getName());
+ sb.append("\t");
+ if (thread.isDaemon())
+ {
+ sb.append("daemon");
+ }
+ else
+ {
+ sb.append("not_a_daemon");
+ }
+
+ if (threadTargetField != null)
+ {
+ sb.append("\t");
+ try
+ {
+ Object threadTarget = threadTargetField.get(thread);
+ if (threadTarget != null)
+ {
+ sb.append(threadTarget.getClass());
+ }
+ else
+ {
+ sb.append("null");
+ }
+ }
+ catch (IllegalAccessException e)
+ {
+ sb.append("unknown class");
+ }
+ }
+
+ sb.append("\n");
+ }
+
+ return sb.toString();
+ }
+
+ private Thread[] diffThreadSnapshot(Thread[] startThreadSnapshot, Thread[] endThreadSnapshot)
+ {
+ List diff = new ArrayList(10);
+
+ nextEnd: for (int iterEnd = 0; iterEnd < endThreadSnapshot.length; iterEnd++)
+ {
+ nextStart: for (int iterStart = 0; iterStart < startThreadSnapshot.length; iterStart++)
+ {
+ if (startThreadSnapshot[iterStart] == endThreadSnapshot[iterEnd])
+ {
+ continue nextEnd;
+ }
+ }
+
+ diff.add(endThreadSnapshot[iterEnd]);
+ }
+
+ Thread ret[] = new Thread[diff.size()];
+ diff.toArray(ret);
+ return ret;
+ }
+
+ private Thread[] createThreadSnapshot()
+ {
+ ThreadGroup tg = Thread.currentThread().getThreadGroup();
+ while (tg.getParent() != null)
+ {
+ tg = tg.getParent();
+ }
+
+ Thread snapshot[] = new Thread[200];
+ tg.enumerate(snapshot, true);
+
+ return snapshot;
+ }
}
1.7 +7 -1 jakarta-commons-sandbox/vfs/src/test/org/apache/commons/vfs/test/NamingTests.java
Index: NamingTests.java
===================================================================
RCS file: /home/cvs/jakarta-commons-sandbox/vfs/src/test/org/apache/commons/vfs/test/NamingTests.java,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -r1.6 -r1.7
--- NamingTests.java 10 May 2004 20:09:44 -0000 1.6
+++ NamingTests.java 7 Jul 2004 20:01:35 -0000 1.7
@@ -53,6 +53,9 @@
// Locate parent
file = getManager().resolveFile("..");
assertSame("file object", getReadFolder().getParent(), file);
+
+ // free basefile
+ getManager().setBaseFile((FileObject) null);
}
/**
@@ -113,6 +116,9 @@
catch (FileSystemException e)
{
}
+
+ // free basefile
+ getManager().setBaseFile((FileObject) null);
}
/**
1.35 +7 -3 jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/impl/DefaultFileSystemManager.java
Index: DefaultFileSystemManager.java
===================================================================
RCS file: /home/cvs/jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/impl/DefaultFileSystemManager.java,v
retrieving revision 1.34
retrieving revision 1.35
diff -u -r1.34 -r1.35
--- DefaultFileSystemManager.java 7 Jul 2004 12:03:32 -0000 1.34
+++ DefaultFileSystemManager.java 7 Jul 2004 20:01:35 -0000 1.35
@@ -418,6 +418,7 @@
/**
* Sets the base file to use when resolving relative URI.
*/
+ // public void setBaseFile(final FileObject baseFile)
public void setBaseFile(final FileObject baseFile)
throws FileSystemException
{
@@ -435,7 +436,7 @@
/**
* Returns the base file used to resolve relative URI.
*/
- public FileObject getBaseFile()
+ public FileObject getBaseFile() throws FileSystemException
{
return baseFile;
}
@@ -445,7 +446,8 @@
*/
public FileObject resolveFile(final String uri) throws FileSystemException
{
- return resolveFile(baseFile, uri);
+ // return resolveFile(baseFile, uri);
+ return resolveFile(getBaseFile(), uri);
}
/**
@@ -454,7 +456,8 @@
public FileObject resolveFile(final String uri, final FileSystemOptions fileSystemOptions) throws FileSystemException
{
- return resolveFile(baseFile, uri, fileSystemOptions);
+ // return resolveFile(baseFile, uri, fileSystemOptions);
+ return resolveFile(getBaseFile(), uri, fileSystemOptions);
}
/**
@@ -528,6 +531,7 @@
{
throw new FileSystemException("vfs.impl/find-rel-file.error", uri);
}
+
return baseFile.resolveFile(decodedUri);
}
1.22 +1 -1 jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/FileSystemManager.java
Index: FileSystemManager.java
===================================================================
RCS file: /home/cvs/jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/FileSystemManager.java,v
retrieving revision 1.21
retrieving revision 1.22
diff -u -r1.21 -r1.22
--- FileSystemManager.java 28 Jun 2004 19:44:57 -0000 1.21
+++ FileSystemManager.java 7 Jul 2004 20:01:35 -0000 1.22
@@ -70,7 +70,7 @@
/**
* Returns the base file used to resolve relative paths.
*/
- FileObject getBaseFile();
+ FileObject getBaseFile() throws FileSystemException;
/**
* Locates a file by name. Equivalent to calling
1.7 +65 -12 jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/cache/SoftRefFilesCache.java
Index: SoftRefFilesCache.java
===================================================================
RCS file: /home/cvs/jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/cache/SoftRefFilesCache.java,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -r1.6 -r1.7
--- SoftRefFilesCache.java 21 May 2004 20:54:31 -0000 1.6
+++ SoftRefFilesCache.java 7 Jul 2004 20:01:35 -0000 1.7
@@ -21,6 +21,7 @@
import org.apache.commons.vfs.FileObject;
import org.apache.commons.vfs.FileSystem;
import org.apache.commons.vfs.VfsLog;
+import org.apache.commons.vfs.provider.AbstractFileSystem;
import org.apache.commons.vfs.util.Messages;
import java.lang.ref.Reference;
@@ -46,7 +47,7 @@
*/
private Log log = LogFactory.getLog(SoftRefFilesCache.class);
- private final Map files = new TreeMap();
+ private final Map filesystemCache = new HashMap();
private final Map refReverseMap = new HashMap(100);
private final ReferenceQueue refqueue = new ReferenceQueue();
@@ -80,7 +81,10 @@
{
FileSystemAndNameKey key = (FileSystemAndNameKey) refReverseMap.get(ref);
- removeFile(key);
+ if (key != null)
+ {
+ removeFile(key);
+ }
}
}
}
@@ -122,14 +126,18 @@
{
synchronized (this)
{
+ Map files = getOrCreateFilesystemCache(file.getFileSystem());
+ /*
if (files.size() < 1)
{
startThread();
}
+ */
SoftReference ref = new SoftReference(file, refqueue);
FileSystemAndNameKey key = new FileSystemAndNameKey(file.getFileSystem(), file.getName());
- files.put(key, ref);
+ // files.put(key, ref);
+ files.put(file.getName(), ref);
refReverseMap.put(ref, key);
}
}
@@ -138,9 +146,11 @@
{
synchronized (this)
{
- FileSystemAndNameKey key = new FileSystemAndNameKey(filesystem, name);
+ Map files = getOrCreateFilesystemCache(filesystem);
+ // FileSystemAndNameKey key = new FileSystemAndNameKey(filesystem, name);
- SoftReference ref = (SoftReference) files.get(key);
+ // SoftReference ref = (SoftReference) files.get(key);
+ SoftReference ref = (SoftReference) files.get(name);
if (ref == null)
{
return null;
@@ -149,7 +159,8 @@
FileObject fo = (FileObject) ref.get();
if (fo == null)
{
- removeFile(key);
+ // removeFile(key);
+ removeFile(filesystem, name);
}
return fo;
}
@@ -159,25 +170,40 @@
{
synchronized (this)
{
- Iterator iterKeys = files.keySet().iterator();
+ Map files = getOrCreateFilesystemCache(filesystem);
+
+ Iterator iterKeys = refReverseMap.values().iterator();
+ // Iterator iterKeys = files.keySet().iterator();
while (iterKeys.hasNext())
{
FileSystemAndNameKey key = (FileSystemAndNameKey) iterKeys.next();
if (key.getFileSystem() == filesystem)
{
- Object ref = files.get(key);
+ // Object ref = files.get(key);
iterKeys.remove();
- refReverseMap.remove(ref);
+ files.remove(key.getFileName());
+ // refReverseMap.remove(ref);
}
}
if (files.size() < 1)
{
+ filesystemClose(filesystem);
+ }
+ /*
+ if (files.size() < 1)
+ {
endThread();
}
+ */
}
}
+ private void filesystemClose(FileSystem filesystem)
+ {
+ ((AbstractFileSystem) filesystem).close();
+ }
+
public void close()
{
super.close();
@@ -186,7 +212,8 @@
{
endThread();
- files.clear();
+ // files.clear();
+ filesystemCache.clear();
refReverseMap.clear();
}
}
@@ -204,7 +231,10 @@
{
synchronized (this)
{
- Object ref = files.remove(key);
+ Map files = getOrCreateFilesystemCache(key.getFileSystem());
+
+ // Object ref = files.remove(key);
+ Object ref = files.remove(key.getFileName());
if (ref != null)
{
refReverseMap.remove(ref);
@@ -212,9 +242,32 @@
if (files.size() < 1)
{
+ filesystemClose(key.getFileSystem());
+ }
+
+ /*
+ if (files.size() < 1)
+ {
endThread();
}
+ */
}
}
+ protected Map getOrCreateFilesystemCache(final FileSystem filesystem)
+ {
+ if (filesystemCache.size() < 1)
+ {
+ startThread();
+ }
+
+ Map files = (Map) filesystemCache.get(filesystem);
+ if (files == null)
+ {
+ files = new TreeMap();
+ filesystemCache.put(filesystem, files);
+ }
+
+ return files;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commons-dev-unsubscribe@jakarta.apache.org
For additional commands, e-mail: commons-dev-help@jakarta.apache.org