You are viewing a plain text version of this content. The canonical link for it is here.
Posted to cvs@cocoon.apache.org by di...@apache.org on 2001/07/13 13:13:44 UTC
cvs commit: xml-cocoon2/src/org/apache/cocoon/components/store FilesystemStore.java MRUMemoryStore.java
dims 01/07/13 04:13:43
Modified: src/org/apache/cocoon/components/store FilesystemStore.java
MRUMemoryStore.java
Log:
Patches from "Gerhard Froehlich" <g-...@gmx.de> for "Big Bug" in File/MRU SystemStore
Revision Changes Path
1.2 +3 -1 xml-cocoon2/src/org/apache/cocoon/components/store/FilesystemStore.java
Index: FilesystemStore.java
===================================================================
RCS file: /home/cvs/xml-cocoon2/src/org/apache/cocoon/components/store/FilesystemStore.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- FilesystemStore.java 2001/05/09 20:50:00 1.1
+++ FilesystemStore.java 2001/07/13 11:13:15 1.2
@@ -9,6 +9,7 @@
import java.io.File;
import java.io.IOException;
+import java.net.URLEncoder;
import java.util.Enumeration;
import org.apache.avalon.framework.context.Context;
import org.apache.avalon.framework.context.ContextException;
@@ -82,12 +83,13 @@
* Get the file associated with the given unique key name.
*/
public Object get(Object key) {
+ getLogger().debug("FilesystemStore get(): Get file with key: " + key.toString());
File file = fileFromKey(key);
if (file != null && file.exists()) {
+ getLogger().debug("FilesystemStore get(): Found file with key: " + key.toString());
return file;
}
-
return null;
}
1.3 +231 -70 xml-cocoon2/src/org/apache/cocoon/components/store/MRUMemoryStore.java
Index: MRUMemoryStore.java
===================================================================
RCS file: /home/cvs/xml-cocoon2/src/org/apache/cocoon/components/store/MRUMemoryStore.java,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -r1.2 -r1.3
--- MRUMemoryStore.java 2001/05/31 17:38:16 1.2
+++ MRUMemoryStore.java 2001/07/13 11:13:24 1.3
@@ -7,10 +7,21 @@
*****************************************************************************/
package org.apache.cocoon.components.store;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
+import java.io.File;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.LinkedList;
+import java.util.Stack;
+
+import org.apache.cocoon.util.ClassUtils;
+import org.apache.cocoon.util.IOUtils;
+
import org.apache.avalon.framework.component.Component;
+import org.apache.avalon.framework.component.ComponentException;
+import org.apache.avalon.framework.component.ComponentManager;
+import org.apache.avalon.framework.component.Composable;
import org.apache.avalon.framework.configuration.Configurable;
import org.apache.avalon.framework.configuration.Configuration;
import org.apache.avalon.framework.configuration.ConfigurationException;
@@ -22,11 +33,8 @@
* This class provides a cache algorithm for the requested documents.
* It combines a HashMap and a LinkedList to create a so called MRU
* (Most Recently Used) cache.
- * The cached objects also have a "lifecycle". If the "lifecycle" of a
- * object is over, it "dies" like in real life :-) and a new object will
- * be born.
- * Also could the number of objects in the cache be limited. If the Limit is
- * reache, the last object in the cache will be removed.
+ * The objects can also be stored onto the filesystem to hold them in a
+ * persitent state over jvm restarts.
*
* The idea was token from the "Writing Advanced Applikation Tutorial" from
* javasoft. Many thanx to the writers!
@@ -35,7 +43,7 @@
* @author <a href="mailto:dims@yahoo.com">Davanum Srinivas</a>
*/
-public class MRUMemoryStore extends AbstractLoggable implements Store, Configurable, ThreadSafe, Runnable {
+public class MRUMemoryStore extends AbstractLoggable implements Store, Configurable, ThreadSafe, Runnable, Composable {
/**
* Indicates how much memory should be left free in the JVM for
* normal operation.
@@ -51,12 +59,12 @@
/**
* Indicates the time in seconds to sleep between memory checks.
*/
- private long interval;
+ private long cleanupthreadinterval;
/**
* Indicates whether we use a cleanup thread or not.
*/
- private boolean useThread;
+ private boolean usecleanupthread;
/**
* Indicates the daemon thread priority.
@@ -64,14 +72,19 @@
private int priority;
/**
- * Indicates the object lifetime
+ * Indicates the max. object in the cache
*/
- private int ObjectLifeTime;
+ private int maxobjects;
/**
- * Indicates the max. object in the cache
+ * Sets the filesystem store on or off
*/
- private int maxobjects;
+ private boolean filesystem;
+
+ /**
+ * Indicates the interval of the WriterThread
+ */
+ private int writerthreadinterval;
/**
* The heart of the cache
@@ -80,11 +93,22 @@
private LinkedList mrulist;
private Runtime jvm;
+
+ private File cachefile;
+ private Store fsstore;
+ private Stack writerstack;
- public MRUMemoryStore() {
- this.jvm = Runtime.getRuntime();
- this.cache = new HashMap();
- this.mrulist = new LinkedList();
+ /** the component manager */
+ protected ComponentManager manager;
+
+ public void compose(ComponentManager manager) throws ComponentException {
+ try {
+ this.manager = manager;
+ getLogger().debug("Looking up FilesystemStore" + FilesystemStore.ROLE);
+ this.fsstore = (Store)manager.lookup(Store.ROLE + "/Filesystem");
+ } catch(ComponentException e) {
+ getLogger().error("Error in MRUMemoryStore!",e);
+ }
}
/**
@@ -93,57 +117,106 @@
* <UL>
* <LI>freememory = How much memory to keep free for normal jvm operation. (Default: 1 Mb)</LI>
* <LI>heapsize = The size of the heap before cleanup starts. (Default: 60 Mb)</LI>
- * <LI>usethread = use a cleanup daemon thread. (Default: true)</LI>
- * <LI>threadpriority = priority to run cleanup thread (1-10). (Default: 10)</LI>
- * <LI>interval = time in seconds to sleep between memory checks (Default: 10 seconds)</LI>
- * <LI>objectlifetime = Object lifetime in seconds
+ * <LI>cleanupthreadinterval = time in seconds to sleep between memory checks (Default: 10 seconds)</LI>
+ * <LI>maxobjects = how many objects will be stored in memory (Default: 10o objects)</LI>
+ * <LI>threadpriority = priority of the thread (1-10). (Default: 10)</LI>
+ * <LI>filesystem = use filesystem storage to keep object persistent (Default: false)</LI>
+ * <LI>writerthreadinterval = time in millis to sleep between writing onto the filesystem (Default: 100 millis)</LI>
+ * <LI>usecleanupthread = use a cleanup daemon thread. (Default: true)</LI>
* </UL>
*/
-
public void configure(Configuration conf) throws ConfigurationException {
- Parameters params = Parameters.fromConfiguration(conf);
-
- this.freememory = params.getParameterAsInteger("freememory",1000000);
- this.heapsize = params.getParameterAsInteger("heapsize",60000000);
- this.ObjectLifeTime = params.getParameterAsInteger("objectlifetime",300);
- this.interval = params.getParameterAsInteger("interval",10);
- this.maxobjects = params.getParameterAsInteger("maxobjects",100);
- this.priority = params.getParameterAsInteger("threadpriority",Thread.currentThread().getPriority());
-
- if ((this.priority < 1) || (this.priority > 10)) {
- throw new ConfigurationException("Thread priority must be between 1 and 10");
- }
+ this.jvm = Runtime.getRuntime();
+ this.cache = new HashMap();
+ this.mrulist = new LinkedList();
+ this.writerstack = new Stack();
+
+ Parameters params = Parameters.fromConfiguration(conf);
+ this.freememory = params.getParameterAsInteger("freememory",1000000);
+ this.heapsize = params.getParameterAsInteger("heapsize",60000000);
+ this.cleanupthreadinterval = params.getParameterAsInteger("cleanupthreadinterval",10);
+ this.maxobjects = params.getParameterAsInteger("maxobjects",100);
+ this.priority = params.getParameterAsInteger("threadpriority",Thread.currentThread().getPriority());
+ this.filesystem = params.getParameterAsBoolean("filesystem",false);
+ this.writerthreadinterval = params.getParameterAsInteger("writerthreadinterval",100);
+ if ((this.priority < 1) || (this.priority > 10)) {
+ throw new ConfigurationException("MRUMemoryStore cleanup thread priority must be between 1 and 10!");
+ }
+ if ((this.writerthreadinterval < 1)) {
+ throw new ConfigurationException("MRUMemoryStore writer thread interval must be at least 1 millis!");
+ }
+ if ((this.maxobjects < 1)) {
+ throw new ConfigurationException("MRUMemoryStore maxobjects must be at least 1 milli second!");
+ }
+ if ((this.cleanupthreadinterval < 1)) {
+ throw new ConfigurationException("MRUMemoryStore cleanup thread interval must be at least 1 second!");
+ }
- this.useThread = params.getParameter("usethread","true").equals("true");
- if (this.useThread) {
- Thread checker = new Thread(this);
- checker.setPriority(this.priority);
- checker.setDaemon(true);
- checker.start();
- }
+ this.usecleanupthread = params.getParameter("usecleanupthread","true").equals("true");
+
+ if (this.usecleanupthread) {
+ getLogger().debug("MRUMemoryStore intializing checker thread");
+ Thread checker = new Thread(this);
+ checker.setPriority(this.priority);
+ checker.setDaemon(true);
+ checker.setName("checker");
+ checker.start();
+ }
+
+ if (this.filesystem) {
+ getLogger().debug("MRUMemoryStore intializing writer thread");
+ Thread writer = new Thread(this);
+ writer.setPriority(this.priority);
+ writer.setDaemon(true);
+ writer.setName("writer");
+ writer.start();
+ }
}
/**
- * Background memory check.
- * Checks that memory is not running too low in the JVM because of the Store.
+ * Background threads.
+ * Thread checker checks that memory is not running too low in the JVM because of the Store.
* It will try to keep overall memory usage below the requested levels.
+ * Thread writer writes objects from the writer stack onto the filesystem.
*/
- public void run() {
- while (true) {
- if (this.jvm.totalMemory() > this.heapsize) {
- this.jvm.runFinalization();
- this.jvm.gc();
- synchronized (this) {
- while ((this.cache.size() > 0) && (this.jvm.freeMemory() < this.freememory)) {
- this.free();
- }
- }
- }
- try {
- Thread.currentThread().sleep(this.interval * 1000);
- } catch (InterruptedException ignore) {}
- }
- }
+ public void run() {
+ while (true) {
+ if(Thread.currentThread().getName().equals("checker")) {
+ if (this.jvm.totalMemory() > this.heapsize) {
+ this.jvm.runFinalization();
+ this.jvm.gc();
+ synchronized (this) {
+ while ((this.cache.size() > 0) && (this.jvm.freeMemory() < this.freememory)) {
+ this.free();
+ }
+ }
+ }
+ try {
+ Thread.currentThread().sleep(this.cleanupthreadinterval * 1000);
+ } catch (InterruptedException ignore) {}
+ } else if(Thread.currentThread().getName().equals("writer")) {
+ if(!writerstack.empty()) {
+ try {
+ TmpStackObject tmpstackobject = new TmpStackObject();
+ Object key = new Object();
+ Object object = new Object();
+ tmpstackobject = (TmpStackObject)this.writerstack.pop();
+ key = tmpstackobject.getKey();
+ object = tmpstackobject.getObject();
+ this.fsstore.store(URLEncoder.encode(key.toString()),object);
+ key = null;
+ object = null;
+ tmpstackobject = null;
+ } catch(java.io.IOException e) {
+ getLogger().error("Error in MRUMemoryStore",e);
+ }
+ }
+ try {
+ Thread.currentThread().sleep(this.writerthreadinterval);
+ } catch (InterruptedException ignore) {}
+ }
+ }
+ }
/**
* Store the given object in a persistent state. It is up to the
@@ -157,16 +230,33 @@
/**
* This method holds the requested object in a HashMap combined with a LinkedList to
* create the MRU.
+ * It also can store the objects onto the filesystem if configured.
*/
public void hold(Object key, Object value) {
- getLogger().debug("Holding object in memory. Key: " + key);
+ getLogger().debug("MRUMemoryStore holding object in memory. Key: " + key);
+ boolean serialisedFlag;
+
/** ...first test if the max. objects in cache is reached... */
if(this.mrulist.size() >= this.maxobjects) {
/** ...ok, heapsize is reached, remove the last element... */
this.free();
}
+
+ /** put the object on the filesystem */
+ if(this.filesystem) {
+ if(this.checkSeriazable(value)) {
+ getLogger().debug("MRUMemoryStore storing object on fs");
+ this.writerstack.push(new TmpStackObject(key,value));
+ getLogger().debug("MRUMemoryStore stack size=" + writerstack.size());
+ serialisedFlag = true;
+ } else {
+ serialisedFlag = false;
+ }
+ } else {
+ serialisedFlag = false;
+ }
/** ..put the new object in the cache, on the top of course ... */
- this.cache.put(key, new CacheObject(value,System.currentTimeMillis()));
+ this.cache.put(key, new CacheObject(value,System.currentTimeMillis(),serialisedFlag));
this.mrulist.addFirst(key);
}
@@ -174,19 +264,40 @@
* Get the object associated to the given unique key.
*/
public Object get(Object key) {
+ getLogger().debug("MRUMemoryStore getting object from memory. Key: " + key);
+ //CacheObject tmpobject = new CacheObject();
+ Object tmpobject = new Object();
+
try {
- long TimeDiff = System.currentTimeMillis() - ((CacheObject)this.cache.get(key)).getCreateTime();
- /** ...check if the object life time is reached... */
- if(TimeDiff >= (this.ObjectLifeTime * 1000)) {
- this.remove(key);
- return null;
- }
/** put the accessed key on top of the linked list */
this.mrulist.remove(key);
this.mrulist.addFirst(key);
return ((CacheObject)this.cache.get(key)).getCacheObject();
} catch(NullPointerException e) {
- return null;
+ getLogger().debug("MRUMemoryStore object not found in memory");
+ /** try to fetch from filesystem */
+ if(this.filesystem) {
+ tmpobject = this.fsstore.get(URLEncoder.encode(key.toString()));
+ if (tmpobject == null) {
+ return null;
+ } else {
+ getLogger().debug("MRUMemoryStor found object on fs");
+
+ try {
+ tmpobject = IOUtils.deserializeObject((File)tmpobject);
+ this.hold(key,tmpobject);
+ return tmpobject;
+ } catch (ClassNotFoundException ce) {
+ getLogger().error("Error in MRUMemoryStore!",e);
+ return null;
+ } catch (java.io.IOException ioe) {
+ getLogger().error("Error in MRUMemoryStore!",e);
+ return null;
+ }
+ }
+ } else {
+ return null;
+ }
}
}
@@ -195,8 +306,10 @@
* the object associated to the given key or null if not found.
*/
public void remove(Object key) {
+ getLogger().debug("MRUMemoryStore removing object from store");
this.cache.remove(key);
this.mrulist.remove(key);
+ this.fsstore.remove(URLEncoder.encode(key.toString()));
}
/**
@@ -221,9 +334,30 @@
* It removes the last element in the cache.
*/
public void free() {
+ if(this.checkSeriazable(cache.get(this.mrulist.getLast()))) {
+ this.writerstack.push(new TmpStackObject(this.mrulist.getLast(),cache.get(this.mrulist.getLast())));
+ }
this.cache.remove(this.mrulist.getLast());
this.mrulist.removeLast();
}
+
+ /**
+ * This method checks if an object is seriazable
+ */
+ private boolean checkSeriazable(Object object) {
+ try {
+ if((object.getClass().getName().equals("CachedEventObject"))
+ || (object.getClass().getName().equals("org.apache.cocoon.caching.CachedStreamObject"))
+ || (ClassUtils.implementsInterface(object.getClass().getName(),"org.apache.cocoon.caching.CacheValidity"))) {
+ return true;
+ } else {
+ return false;
+ }
+ } catch (Exception e) {
+ getLogger().error("Error in MRUMemoryStore!",e);
+ return false;
+ }
+ }
/**
* Container object for the documents.
@@ -231,18 +365,45 @@
class CacheObject {
private long time = -1;
private Object cacheObject;
-
- public CacheObject(Object ToCacheObject, long lTime) {
+ private boolean serialised;
+
+ public CacheObject(Object ToCacheObject, long lTime, boolean serialised) {
this.cacheObject = ToCacheObject;
this.time = lTime;
+ this.serialised = serialised;
}
-
+
public Object getCacheObject() {
return this.cacheObject;
}
public long getCreateTime() {
return this.time;
+ }
+
+ public boolean getSerialisedFlag() {
+ return this.serialised;
+ }
+ }
+
+ /** Temporary container object for the writerstack */
+ class TmpStackObject {
+ private Object object;
+ private Object key;
+
+ public TmpStackObject (Object key, Object object) {
+ this.object = object;
+ this.key = key;
+ }
+
+ public TmpStackObject() {}
+
+ public Object getKey() {
+ return this.key;
+ }
+
+ public Object getObject() {
+ return this.object;
}
}
}
----------------------------------------------------------------------
In case of troubles, e-mail: webmaster@xml.apache.org
To unsubscribe, e-mail: cocoon-cvs-unsubscribe@xml.apache.org
For additional commands, e-mail: cocoon-cvs-help@xml.apache.org