You are viewing a plain text version of this content. The canonical link for it is here.
Posted to cvs@avalon.apache.org by vg...@apache.org on 2003/07/31 05:13:48 UTC
cvs commit: avalon-excalibur/store/src/java/org/apache/excalibur/store/impl AbstractJispFilesystemStore.java StoreJanitorImpl.java
vgritsenko 2003/07/30 20:13:48
Modified: store/src/java/org/apache/excalibur/store/impl
AbstractJispFilesystemStore.java
StoreJanitorImpl.java
Log:
New boolean parameter for StoreJanitor: adaptivethreadinterval. Default is false.
When true, cleanupthreadinterval defines the maximum sleep time. Sleep time then
is determined depending on the memory fill rate: the faster memory is filled in,
the shorter is sleep time (this feature is ported from cocoon scratchpad).
Revision Changes Path
1.16 +2 -2 avalon-excalibur/store/src/java/org/apache/excalibur/store/impl/AbstractJispFilesystemStore.java
Index: AbstractJispFilesystemStore.java
===================================================================
RCS file: /home/cvs/avalon-excalibur/store/src/java/org/apache/excalibur/store/impl/AbstractJispFilesystemStore.java,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -r1.15 -r1.16
--- AbstractJispFilesystemStore.java 31 Jul 2003 02:51:32 -0000 1.15
+++ AbstractJispFilesystemStore.java 31 Jul 2003 03:13:47 -0000 1.16
@@ -352,7 +352,7 @@
return new JispKey().makeNullKey();
}
- class BTreeObjectEnumeration implements Enumeration
+ class BTreeObjectEnumeration implements Enumeration
{
private Object m_Next;
private BTreeIterator m_Iterator;
1.10 +160 -75 avalon-excalibur/store/src/java/org/apache/excalibur/store/impl/StoreJanitorImpl.java
Index: StoreJanitorImpl.java
===================================================================
RCS file: /home/cvs/avalon-excalibur/store/src/java/org/apache/excalibur/store/impl/StoreJanitorImpl.java,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -r1.9 -r1.10
--- StoreJanitorImpl.java 29 Jul 2003 04:43:14 -0000 1.9
+++ StoreJanitorImpl.java 31 Jul 2003 03:13:47 -0000 1.10
@@ -86,27 +86,35 @@
private static boolean doRun = false;
- private int freememory = -1;
- private int heapsize = -1;
- private int cleanupthreadinterval = -1;
+ // Configuration parameters
+ private int minFreeMemory = -1;
+ private int maxHeapSize = -1;
+ private int threadInterval = -1;
+ private boolean adaptiveThreadInterval = false;
private int priority = -1;
+ private double fraction;
+
private Runtime jvm;
private ArrayList storelist;
private int index = -1;
- private double fraction;
/**
* Initialize the StoreJanitorImpl.
* A few options can be used :
* <UL>
- * <LI>freememory = how many bytes shall be always free in the jvm</LI>
- * <LI>heapsize = max. size of jvm memory consumption</LI>
- * <LI>cleanupthreadinterval = how often (sec) shall run the cleanup thread</LI>
- * <LI>threadpriority = priority of the thread (1-10). (Default: 10)</LI>
+ * <LI><B>freememory</B>: How many bytes shall be always free in the JVM (Default: 1mb)</LI>
+ * <LI><B>heapsize</B>: Maximum possible size of the JVM memory consumption (Default: 64mb)</LI>
+ * <LI><B>cleanupthreadinterval</B>: How often (sec) shall run the cleanup thread (Default: 10s)</LI>
+ * <LI><B>adaptivethreadinterval</B> (experimental): Enable adaptive algorithm to determine thread interval
+ * (Default: false) When true, <code>cleanupthreadinterval</code> defines the maximum cleanup interval.
+ * Cleanup interval then is determined based on the memory fill rate: the faster memory is filled in,
+ * and the less free memory is left, the shorter is the cleanup time.</LI>
+ * <LI><B>threadpriority</B>: priority of the thread (1-10). (Default: 10)</LI>
+ * <LI><B>percent_to_free</B>: What fraction of the store to free when memory is low (1-100). (Default: 10%)</LI>
* </UL>
*
* @param params the Configuration of the application
- * @exception ConfigurationException
+ * @exception ParameterException
*/
public void parameterize(Parameters params) throws ParameterException {
if (getLogger().isDebugEnabled()) {
@@ -114,30 +122,32 @@
}
setJVM(Runtime.getRuntime());
- setFreememory(params.getParameterAsInteger("freememory",1000000));
- setHeapsize(params.getParameterAsInteger("heapsize",60000000));
- setCleanupthreadinterval(params.getParameterAsInteger("cleanupthreadinterval", 10));
+ setMinFreeMemory(params.getParameterAsInteger("freememory", 1024 * 1024));
+ setMaxHeapSize(params.getParameterAsInteger("heapsize", 60 * 1024 * 1024));
+ // Parameter value is in seconds, converted to millis
+ setThreadInterval(params.getParameterAsInteger("cleanupthreadinterval", 10) * 1000);
+ setAdaptiveThreadInterval(params.getParameterAsBoolean("adaptivethreadinterval", false));
setPriority(params.getParameterAsInteger("threadpriority",
Thread.currentThread().getPriority()));
int percent = params.getParameterAsInteger("percent_to_free", 10);
- if (getFreememory() < 1) {
+ if (getMinFreeMemory() < 1) {
throw new ParameterException("StoreJanitorImpl freememory parameter has to be greater then 1");
}
- if (getHeapsize() < 1) {
+ if (getMaxHeapSize() < 1) {
throw new ParameterException("StoreJanitorImpl heapsize parameter has to be greater then 1");
}
- if (getCleanupthreadinterval() < 1) {
+ if (getThreadInterval() < 1) {
throw new ParameterException("StoreJanitorImpl cleanupthreadinterval parameter has to be greater then 1");
}
- if (getPriority() < 1) {
- throw new ParameterException("StoreJanitorImpl threadpriority has to be greater then 1");
+ if (getPriority() < 1 || getPriority() > 10) {
+ throw new ParameterException("StoreJanitorImpl threadpriority has to be between 1 and 10");
}
if (percent > 100 && percent < 1) {
throw new ParameterException("StoreJanitorImpl percent_to_free, has to be between 1 and 100");
}
- this.fraction = percent / 100.0;
+ this.fraction = percent / 100.0D;
setStoreList(new ArrayList());
}
@@ -161,34 +171,64 @@
* The "checker" thread checks if memory is running low in the jvm.
*/
public void run() {
+ boolean firstRun = true;
+ long inUse = memoryInUse(); // Amount of memory in use before sleep()
+ long interval = Long.MAX_VALUE; // Sleep time in ms
+ long maxRateOfChange = 1; // Used memory change rate in bytes per second
+
while (doRun) {
- // amount of memory used is greater then heapsize
- if (memoryLow()) {
- if (getLogger().isDebugEnabled()) {
- getLogger().debug("Invoking garbage collection, total memory = "
- + getJVM().totalMemory() + ", free memory = "
- + getJVM().freeMemory());
+ if (getAdaptiveThreadInterval()) {
+ // Monitor the rate of change of heap in use.
+ long change = memoryInUse() - inUse;
+ long rateOfChange = longDiv(change * 1000, interval); // bps.
+ if (maxRateOfChange < rateOfChange) {
+ maxRateOfChange = (maxRateOfChange + rateOfChange) / 2;
}
-
- //this.freePhysicalMemory();
-
if (getLogger().isDebugEnabled()) {
- getLogger().debug("Garbage collection complete, total memory = "
- + getJVM().totalMemory() + ", free memory = "
- + getJVM().freeMemory());
+ getLogger().debug("Waking after " + interval + "ms, in use change "
+ + change + "b to " + memoryInUse() + "b, rate "
+ + rateOfChange + "b/sec, max rate " + maxRateOfChange + "b/sec");
}
+ }
+
+ // Amount of memory used is greater than heapsize
+ if (memoryLow()) {
+ // Uncomment "memoryLow() &&" if uncommenting this line
+ // this.freePhysicalMemory();
synchronized (this) {
- if (memoryLow() && getStoreList().size() > 0) {
+ if (// memoryLow() &&
+ getStoreList().size() > 0) {
freeMemory();
setIndex(getIndex() + 1);
}
}
}
+ if (getAdaptiveThreadInterval()) {
+ // Calculate sleep interval based on the change rate and free memory left
+ interval = minTimeToFill(maxRateOfChange) * 1000 / 2;
+ if (interval > this.threadInterval) {
+ interval = this.threadInterval;
+ }
+ inUse = memoryInUse();
+ } else {
+ interval = this.threadInterval;
+ }
+ if (getLogger().isDebugEnabled()) {
+ getLogger().debug("Sleeping for " + interval + "ms");
+ }
+
+ // Sleep
try {
- Thread.sleep(this.cleanupthreadinterval * 1000);
+ Thread.sleep(interval);
} catch (InterruptedException ignore) {}
+
+ // Ignore change in memory during the first run (startup)
+ if (firstRun) {
+ firstRun = false;
+ inUse = memoryInUse();
+ }
}
}
@@ -199,25 +239,50 @@
*/
private boolean memoryLow() {
if (getLogger().isDebugEnabled()) {
- getLogger().debug("JVM total Memory: " + getJVM().totalMemory());
- getLogger().debug("JVM free Memory: " + getJVM().freeMemory());
+ getLogger().debug("JVM Memory total: " + getJVM().totalMemory()
+ + ", free: " + getJVM().freeMemory());
}
- if ((getJVM().totalMemory() > getHeapsize())
- && (getJVM().freeMemory() < getFreememory())) {
+ if ((getJVM().totalMemory() >= getMaxHeapSize())
+ && (getJVM().freeMemory() < getMinFreeMemory())) {
if (getLogger().isDebugEnabled()) {
- getLogger().debug("Memory is low = true");
+ getLogger().debug("Memory is low!");
}
return true;
} else {
- if (getLogger().isDebugEnabled()) {
- getLogger().debug("Memory is low = false");
- }
return false;
}
}
/**
+ * Calculate the JVM memory in use now.
+ *
+ * @return memory in use.
+ */
+ private long memoryInUse() {
+ return jvm.totalMemory() - jvm.freeMemory();
+ }
+
+ /**
+ * Calculate amount of time needed to fill all free memory with given
+ * fill rate.
+ *
+ * @param rate memory fill rate in time per bytes
+ * @return amount of time to fill all the memory with given fill rate
+ */
+ private long minTimeToFill(long rate) {
+ return longDiv(jvm.freeMemory(), rate);
+ }
+
+ private long longDiv(long top, long bottom) {
+ try {
+ return top / bottom;
+ } catch (Exception e) {
+ return top > 0 ? Long.MAX_VALUE : Long.MIN_VALUE;
+ }
+ }
+
+ /**
* This method register the stores
*
* @param store the store to be registered
@@ -225,8 +290,7 @@
public synchronized void register(Store store) {
getStoreList().add(store);
if (getLogger().isDebugEnabled()) {
- getLogger().debug("Registering store instance");
- getLogger().debug("Size of StoreJanitor now:"
+ getLogger().debug("Registered store instance. Stores now: "
+ getStoreList().size());
}
}
@@ -239,8 +303,7 @@
public synchronized void unregister(Store store) {
getStoreList().remove(store);
if (getLogger().isDebugEnabled()) {
- getLogger().debug("Unregister store instance");
- getLogger().debug("Size of StoreJanitor now:"
+ getLogger().debug("Unregistered store instance. Stores now: "
+ getStoreList().size());
}
}
@@ -259,42 +322,44 @@
*/
public Iterator iterator() {
return getStoreList().iterator();
- }
+ }
/**
* Round Robin alghorithm for freeing the registered caches.
*/
private void freeMemory() {
+ // TODO: Alternative to RR might be to free same fraction from every storage.
try {
- //Determine elements in Store:
- if (getLogger().isDebugEnabled()) {
- getLogger().debug("StoreList size=" + getStoreList().size());
- getLogger().debug("Actual Index position: " + getIndex());
- }
+ // Determine the store.
if (getIndex() < getStoreList().size()) {
if (getIndex() == -1) {
setIndex(0);
}
} else {
+ // Store list changed (one or more store has been removed).
if (getLogger().isDebugEnabled()) {
getLogger().debug("Restarting from the beginning");
}
setIndex(0);
}
+ // Delete proportionate elements out of the store as configured.
Store store = (Store)getStoreList().get(getIndex());
+ int limit = calcToFree(store);
if (getLogger().isDebugEnabled()) {
- getLogger().debug("Freeing Store: " + getIndex());
+ getLogger().debug("Freeing " + limit + " items from store N " + getIndex());
}
-
- //delete proportionate elements out of the cache as
- //configured.
- int limit = calcToFree(store);
for (int i=0; i < limit; i++) {
- store.free();
+ try {
+ store.free();
+ } catch (OutOfMemoryError e) {
+ getLogger().error("OutOfMemoryError in freeMemory()");
+ }
}
} catch (Exception e) {
getLogger().error("Error in freeMemory()", e);
+ } catch (OutOfMemoryError e) {
+ getLogger().error("OutOfMemoryError in freeMemory()");
}
}
@@ -317,33 +382,37 @@
/**
* This method forces the garbage collector
private void freePhysicalMemory() {
+ if (getLogger().isDebugEnabled()) {
+ getLogger().debug("Invoking garbage collection. Memory total: "
+ + getJVM().totalMemory() + ", free: "
+ + getJVM().freeMemory());
+ }
+
getJVM().runFinalization();
getJVM().gc();
- }
- */
-
- private int getFreememory() {
- return freememory;
- }
- private void setFreememory(int _freememory) {
- this.freememory = _freememory;
+ if (getLogger().isDebugEnabled()) {
+ getLogger().debug("Garbage collection complete. Memory total: "
+ + getJVM().totalMemory() + ", free: "
+ + getJVM().freeMemory());
+ }
}
+ */
- private int getHeapsize() {
- return this.heapsize;
+ private int getMinFreeMemory() {
+ return this.minFreeMemory;
}
- private void setHeapsize(int _heapsize) {
- this.heapsize = _heapsize;
+ private void setMinFreeMemory(int _freememory) {
+ this.minFreeMemory = _freememory;
}
- private int getCleanupthreadinterval() {
- return this.cleanupthreadinterval;
+ private int getMaxHeapSize() {
+ return this.maxHeapSize;
}
- private void setCleanupthreadinterval(int _cleanupthreadinterval) {
- this.cleanupthreadinterval = _cleanupthreadinterval;
+ private void setMaxHeapSize(int _heapsize) {
+ this.maxHeapSize = _heapsize;
}
private int getPriority() {
@@ -354,12 +423,28 @@
this.priority = _priority;
}
+ private int getThreadInterval() {
+ return this.threadInterval;
+ }
+
+ private void setThreadInterval(int _threadInterval) {
+ this.threadInterval = _threadInterval;
+ }
+
+ private boolean getAdaptiveThreadInterval() {
+ return this.adaptiveThreadInterval;
+ }
+
+ private void setAdaptiveThreadInterval(boolean _adaptiveThreadInterval) {
+ this.adaptiveThreadInterval = _adaptiveThreadInterval;
+ }
+
private Runtime getJVM() {
return this.jvm;
}
- private void setJVM(Runtime _runtime) {
- this.jvm = _runtime;
+ private void setJVM(Runtime _jvm) {
+ this.jvm = _jvm;
}
private ArrayList getStoreList() {
---------------------------------------------------------------------
To unsubscribe, e-mail: cvs-unsubscribe@avalon.apache.org
For additional commands, e-mail: cvs-help@avalon.apache.org