You are viewing a plain text version of this content. The canonical link for it is here.
Posted to cvs@cocoon.apache.org by gi...@apache.org on 2004/11/09 20:56:24 UTC
svn commit: rev 57092 - in cocoon/branches/BRANCH_2_1_X/src/java/org/apache/cocoon: . components/store/impl
Author: giacomo
Date: Tue Nov 9 11:56:23 2004
New Revision: 57092
Added:
cocoon/branches/BRANCH_2_1_X/src/java/org/apache/cocoon/components/store/impl/CocoonStoreJanitor.java
cocoon/branches/BRANCH_2_1_X/src/java/org/apache/cocoon/components/store/impl/StoreJanitorImpl.java
Modified:
cocoon/branches/BRANCH_2_1_X/src/java/org/apache/cocoon/cocoon.roles
Log:
temporary using patched copy of excalibur StoreJanitorImpl until it is released
Modified: cocoon/branches/BRANCH_2_1_X/src/java/org/apache/cocoon/cocoon.roles
==============================================================================
--- cocoon/branches/BRANCH_2_1_X/src/java/org/apache/cocoon/cocoon.roles (original)
+++ cocoon/branches/BRANCH_2_1_X/src/java/org/apache/cocoon/cocoon.roles Tue Nov 9 11:56:23 2004
@@ -91,9 +91,18 @@
shorthand="persistent-store"
default-class="org.apache.cocoon.components.store.impl.DefaultPersistentStore"/>
-->
+ <!-- Normally uses the org.apache.excalibur.store.impl.StoreJanitorImpl as
+ the default-class but as that uses its own Thread spawning there is
+ the org.apache.cocoon.components.store.impl.CocoonStoreJanitor class
+ to use a daemon thread from the org.apache.cocoon.components.thread.RunnableManager
+ component
+ NOT: As soon as our patch has been accepted by the Excalibur community and an
+ excalibur-store has been release we can switch back to the original
+ org.apache.excalibur.store.impl.StoreJanitorImpl class
+ -->
<role name="org.apache.excalibur.store.StoreJanitor"
shorthand="store-janitor"
- default-class="org.apache.excalibur.store.impl.StoreJanitorImpl"/>
+ default-class="org.apache.cocoon.components.store.impl.CocoonStoreJanitor"/>
<!--=========================================================================
Sitemap engine
Added: cocoon/branches/BRANCH_2_1_X/src/java/org/apache/cocoon/components/store/impl/CocoonStoreJanitor.java
==============================================================================
--- (empty file)
+++ cocoon/branches/BRANCH_2_1_X/src/java/org/apache/cocoon/components/store/impl/CocoonStoreJanitor.java Tue Nov 9 11:56:23 2004
@@ -0,0 +1,120 @@
+/*
+ * Copyright 1999-2004 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cocoon.components.store.impl;
+
+import org.apache.avalon.framework.CascadingRuntimeException;
+import org.apache.avalon.framework.service.ServiceException;
+import org.apache.avalon.framework.service.ServiceManager;
+import org.apache.avalon.framework.service.Serviceable;
+import org.apache.cocoon.components.thread.RunnableManager;
+
+
+/**
+ * The CocoonStoreJanitor class just subclasses the {@link StoreJanitorImpl} to
+ * overwrite the start method for background thread creation using the Cocoon
+ * {@link RunnableManager}.
+ *
+ * @author <a href="mailto:giacomo.at.apache.org">Giacomo Pati</a>
+ * @version $Id$
+ */
+public class CocoonStoreJanitor
+ extends StoreJanitorImpl
+ implements Serviceable
+{
+ //~ Instance fields --------------------------------------------------------
+
+ /** Our {@link ServiceManager} */
+ private ServiceManager m_serviceManager;
+
+ /** Flags to ignore memory bursts in the startup */
+ private boolean m_firstRun = true;
+
+ /** Flags to ignore memory bursts in the startup */
+ private boolean m_secondRun = false;
+
+ //~ Methods ----------------------------------------------------------------
+
+ /**
+ * The "checker" thread checks if memory is running low in the jvm.
+ */
+ public void run( )
+ {
+ // ignoring memory bursts in the first two invokations
+ if( m_firstRun || m_secondRun )
+ {
+ super.inUse = super.memoryInUse( );
+ m_secondRun = m_firstRun;
+ m_firstRun = false;
+ }
+
+ super.checkMemory( );
+
+ // Relaunch
+ relaunch( super.interval );
+ }
+
+ /**
+ * Get the <code>ServiceManager</code>
+ *
+ * @param serviceManager The <code>ServiceManager</code>
+ *
+ * @throws ServiceException Should not happen
+ */
+ public void service( final ServiceManager serviceManager )
+ throws ServiceException
+ {
+ m_serviceManager = serviceManager;
+ }
+
+ /**
+ * Start this instance using a default thread from the
+ * <code>RunnableManager</code>
+ */
+ public void start( )
+ {
+ relaunch( 0 );
+ }
+
+ /**
+ * Does a delayed (re-)start of this instance using a default thread from
+ * the<code>RunnableManager</code> with a delay
+ *
+ * @param delay the delay to apply before next run
+ *
+ * @throws CascadingRuntimeException in case we cannot get a
+ * <code>RunnableManager</code>
+ */
+ private void relaunch( final long delay )
+ {
+ try
+ {
+ if( getLogger( ).isDebugEnabled( ) )
+ {
+ getLogger( ).debug( "(Re-)Start CocoonStoreJaitor" );
+ }
+
+ final RunnableManager runnableManager =
+ (RunnableManager)m_serviceManager.lookup( RunnableManager.ROLE );
+ runnableManager.execute( this, delay, 0 );
+ m_serviceManager.release( runnableManager );
+ }
+ catch( final ServiceException se )
+ {
+ throw new CascadingRuntimeException( "Cannot lookup RunnableManager",
+ se );
+ }
+ }
+}
Added: cocoon/branches/BRANCH_2_1_X/src/java/org/apache/cocoon/components/store/impl/StoreJanitorImpl.java
==============================================================================
--- (empty file)
+++ cocoon/branches/BRANCH_2_1_X/src/java/org/apache/cocoon/components/store/impl/StoreJanitorImpl.java Tue Nov 9 11:56:23 2004
@@ -0,0 +1,557 @@
+/*
+ * Copyright 2002-2004 The Apache Software Foundation
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied.
+ *
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cocoon.components.store.impl;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.apache.avalon.framework.activity.Startable;
+import org.apache.avalon.framework.logger.AbstractLogEnabled;
+import org.apache.avalon.framework.parameters.ParameterException;
+import org.apache.avalon.framework.parameters.Parameterizable;
+import org.apache.avalon.framework.parameters.Parameters;
+import org.apache.avalon.framework.thread.ThreadSafe;
+import org.apache.excalibur.store.Store;
+import org.apache.excalibur.store.StoreJanitor;
+
+/**
+ * This class is a implentation of a StoreJanitor. Store classes
+ * can register to the StoreJanitor. When memory is too low,
+ * the StoreJanitor frees the registered caches until memory is normal.
+ *
+ * @avalon.component
+ * @avalon.service type=StoreJanitor
+ * @x-avalon.info name=store-janitor
+ * @x-avalon.lifestyle type=singleton
+ *
+ * @author <a href="mailto:dev@avalon.apache.org">Avalon Development Team</a>
+ * @version CVS $Id: StoreJanitorImpl.java,v 1.4 2004/02/28 11:47:31 cziegeler Exp $
+ */
+public class StoreJanitorImpl
+extends AbstractLogEnabled
+implements StoreJanitor,
+ Parameterizable,
+ ThreadSafe,
+ Runnable,
+ Startable
+{
+
+ private boolean doRun = false;
+
+ // Configuration parameters
+ private int minFreeMemory = -1;
+ private int maxHeapSize = -1;
+ private int threadInterval = -1;
+ private int minThreadInterval = 500;
+ private boolean adaptiveThreadInterval = false;
+ private int priority = -1;
+ private double fraction;
+
+ private Runtime jvm;
+ private ArrayList storelist;
+ private int index = -1;
+ /** Should the gc be called on low memory? */
+ protected boolean invokeGC = false;
+ /**
+ * Amount of memory in use before sleep(). Must be initially set a resonable
+ * value; ie. <code>memoryInUse()</code>
+ */
+ protected long inUse;
+ private boolean firstRun = true;
+ /** The calculated delay for the next checker run */
+ protected long interval = Long.MAX_VALUE; // Sleep time in ms
+ private long maxRateOfChange = 1; // Used memory change rate in bytes per second
+
+ /**
+ * Initialize the StoreJanitorImpl.
+ * A few options can be used :
+ * <UL>
+ * <LI><B>freememory</B>: How many bytes shall be always free in the JVM (Default: 1mb)</LI>
+ * <LI><B>heapsize</B>: Maximum possible size of the JVM memory consumption (Default: 64mb)</LI>
+ * <LI><B>cleanupthreadinterval</B>: How often (sec) shall run the cleanup thread (Default: 10s)</LI>
+ * <LI><B>adaptivethreadinterval</B> (experimental): Enable adaptive algorithm to determine thread interval
+ * (Default: false) When true, <code>cleanupthreadinterval</code> defines the maximum cleanup interval.
+ * Cleanup interval then is determined based on the memory fill rate: the faster memory is filled in,
+ * and the less free memory is left, the shorter is the cleanup time.</LI>
+ * <LI><B>threadpriority</B>: priority of the thread (1-10). (Default: 10)</LI>
+ * <LI><B>percent_to_free</B>: What fraction of the store to free when memory is low (1-100). (Default: 10%)</LI>
+ * <LI><B>invokegc</B>: Invoke the gc on low memory first (true|false; default: false)</LI>
+ * </UL>
+ *
+ * @param params the Configuration of the application
+ * @exception ParameterException
+ */
+ public void parameterize(Parameters params) throws ParameterException
+ {
+ if (getLogger().isDebugEnabled())
+ {
+ getLogger().debug("Configure StoreJanitorImpl");
+ }
+ setJVM(Runtime.getRuntime());
+
+ setMinFreeMemory(params.getParameterAsInteger("freememory", 1024 * 1024));
+ setMaxHeapSize(params.getParameterAsInteger("heapsize", 60 * 1024 * 1024));
+ // Parameter value is in seconds, converted to millis
+ setThreadInterval(params.getParameterAsInteger("cleanupthreadinterval", 10) * 1000);
+ setAdaptiveThreadInterval(params.getParameterAsBoolean("adaptivethreadinterval", false));
+ setPriority(params.getParameterAsInteger("threadpriority",
+ Thread.currentThread().getPriority()));
+ int percent = params.getParameterAsInteger("percent_to_free", 10);
+ this.invokeGC = params.getParameterAsBoolean("invokegc", this.invokeGC);
+
+ if (getMinFreeMemory() < 1)
+ {
+ throw new ParameterException("StoreJanitorImpl freememory parameter has to be greater then 1");
+ }
+ if (getMaxHeapSize() < 1)
+ {
+ throw new ParameterException("StoreJanitorImpl heapsize parameter has to be greater then 1");
+ }
+ if (getThreadInterval() < 1)
+ {
+ throw new ParameterException("StoreJanitorImpl cleanupthreadinterval parameter has to be greater then 1");
+ }
+ if (getPriority() < 1 || getPriority() > 10)
+ {
+ throw new ParameterException("StoreJanitorImpl threadpriority has to be between 1 and 10");
+ }
+ if (percent > 100 && percent < 1)
+ {
+ throw new ParameterException("StoreJanitorImpl percent_to_free, has to be between 1 and 100");
+ }
+
+ this.fraction = percent / 100.0D;
+ setStoreList(new ArrayList());
+
+ if ( getLogger().isDebugEnabled() )
+ {
+ getLogger().debug("minimum free memory=" + this.getMinFreeMemory());
+ getLogger().debug("heapsize=" + this.getMaxHeapSize());
+ getLogger().debug("thread interval=" + this.getThreadInterval());
+ getLogger().debug("adaptivethreadinterval=" + this.getAdaptiveThreadInterval());
+ getLogger().debug("priority=" + this.getPriority());
+ getLogger().debug("percent=" + percent);
+ getLogger().debug("invoke gc=" + this.invokeGC);
+ }
+ }
+
+ public void start()
+ {
+ doRun = true;
+ Thread checker = new Thread(this);
+ if (getLogger().isDebugEnabled())
+ {
+ getLogger().debug("Intializing checker thread");
+ }
+ checker.setPriority(getPriority());
+ checker.setDaemon(true);
+ checker.setName("checker");
+ checker.start();
+ }
+
+ public void stop()
+ {
+ doRun = false;
+ }
+
+ /**
+ * The "checker" thread loop.
+ */
+ public void run()
+ {
+ inUse = memoryInUse();
+ while (doRun) {
+ checkMemory();
+
+ // Sleep
+ if (getLogger().isDebugEnabled())
+ {
+ getLogger().debug("Sleeping for " + interval + "ms");
+ }
+ try
+ {
+ Thread.sleep(interval);
+ }
+ catch (InterruptedException ignore) {}
+
+ // Ignore change in memory during the first run (startup)
+ if (firstRun)
+ {
+ firstRun = false;
+ inUse = memoryInUse();
+ }
+ }
+ }
+
+ /**
+ * The "checker" thread checks if memory is running low in the jvm.
+ */
+ protected void checkMemory()
+ {
+ if (getAdaptiveThreadInterval())
+ {
+ // Monitor the rate of change of heap in use.
+ long change = memoryInUse() - inUse;
+ long rateOfChange = longDiv(change * 1000, interval); // bps.
+ if (maxRateOfChange < rateOfChange)
+ {
+ maxRateOfChange = (maxRateOfChange + rateOfChange) / 2;
+ }
+ if (getLogger().isDebugEnabled()) {
+ getLogger().debug("Waking after " + interval + "ms, in use change "
+ + change + "b to " + memoryInUse() + "b, rate "
+ + rateOfChange + "b/sec, max rate " + maxRateOfChange + "b/sec");
+ }
+ }
+
+ // Amount of memory used is greater than heapsize
+ if (memoryLow())
+ {
+ if ( this.invokeGC )
+ {
+ this.freePhysicalMemory();
+ }
+
+ synchronized (this)
+ {
+ if (!this.invokeGC
+ || (memoryLow() && getStoreList().size() > 0))
+ {
+
+ freeMemory();
+ setIndex(getIndex() + 1);
+ }
+ }
+ }
+
+ if (getAdaptiveThreadInterval())
+ {
+ // Calculate sleep interval based on the change rate and free memory left
+ interval = minTimeToFill(maxRateOfChange) * 1000 / 2;
+ if (interval > this.threadInterval)
+ {
+ interval = this.threadInterval;
+ }
+ else if (interval < this.minThreadInterval)
+ {
+ interval = this.minThreadInterval;
+ }
+ inUse = memoryInUse();
+ }
+ else
+ {
+ interval = this.threadInterval;
+ }
+ }
+ /**
+ * Method to check if memory is running low in the JVM.
+ *
+ * @return true if memory is low
+ */
+ private boolean memoryLow()
+ {
+ if (getLogger().isDebugEnabled())
+ {
+ getLogger().debug("JVM Memory total: " + getJVM().totalMemory()
+ + ", free: " + getJVM().freeMemory());
+ }
+
+ if ((getJVM().totalMemory() >= getMaxHeapSize())
+ && (getJVM().freeMemory() < getMinFreeMemory()))
+ {
+ if (getLogger().isDebugEnabled())
+ {
+ getLogger().debug("Memory is low!");
+ }
+ return true;
+ }
+ else
+ {
+ return false;
+ }
+ }
+
+ /**
+ * Calculate the JVM memory in use now.
+ *
+ * @return memory in use.
+ */
+ protected long memoryInUse()
+ {
+ return jvm.totalMemory() - jvm.freeMemory();
+ }
+
+ /**
+ * Calculate amount of time needed to fill all free memory with given
+ * fill rate.
+ *
+ * @param rate memory fill rate in time per bytes
+ * @return amount of time to fill all the memory with given fill rate
+ */
+ private long minTimeToFill(long rate)
+ {
+ return longDiv(jvm.freeMemory(), rate);
+ }
+
+ private long longDiv(long top, long bottom)
+ {
+ try
+ {
+ return top / bottom;
+ }
+ catch (Exception e)
+ {
+ return top > 0 ? Long.MAX_VALUE : Long.MIN_VALUE;
+ }
+ }
+
+ /**
+ * This method register the stores
+ *
+ * @param store the store to be registered
+ */
+ public synchronized void register(Store store)
+ {
+ getStoreList().add(store);
+ if (getLogger().isDebugEnabled())
+ {
+ getLogger().debug("Registered store instance " + store + ". Stores now: "
+ + getStoreList().size());
+ }
+ }
+
+ /**
+ * This method unregister the stores
+ *
+ * @param store the store to be unregistered
+ */
+ public synchronized void unregister(Store store)
+ {
+ getStoreList().remove(store);
+ if (getLogger().isDebugEnabled())
+ {
+ getLogger().debug("Unregistered store instance " + store + ". Stores now: "
+ + getStoreList().size());
+ }
+ }
+
+ /**
+ * This method return a java.util.Iterator of every registered stores
+ *
+ * <i>The iterators returned is fail-fast: if list is structurally
+ * modified at any time after the iterator is created, in any way, the
+ * iterator will throw a ConcurrentModificationException. Thus, in the
+ * face of concurrent modification, the iterator fails quickly and
+ * cleanly, rather than risking arbitrary, non-deterministic behavior at
+ * an undetermined time in the future.</i>
+ *
+ * @return a java.util.Iterator
+ */
+ public Iterator iterator()
+ {
+ return getStoreList().iterator();
+ }
+
+ /**
+ * Round Robin alghorithm for freeing the registered caches.
+ */
+ private void freeMemory()
+ {
+ // TODO: Alternative to RR might be to free same fraction from every storage.
+ try
+ {
+ // Determine the store.
+ if (getIndex() < getStoreList().size())
+ {
+ if (getIndex() == -1)
+ {
+ setIndex(0);
+ }
+ }
+ else
+ {
+ // Store list changed (one or more store has been removed).
+ if (getLogger().isDebugEnabled())
+ {
+ getLogger().debug("Restarting from the beginning");
+ }
+ setIndex(0);
+ }
+
+ // Delete proportionate elements out of the store as configured.
+ Store store = (Store)getStoreList().get(getIndex());
+ int limit = calcToFree(store);
+ if (getLogger().isDebugEnabled())
+ {
+ getLogger().debug("Freeing " + limit + " items from store N " + getIndex());
+ }
+ for (int i=0; i < limit; i++)
+ {
+ try
+ {
+ store.free();
+ }
+ catch (OutOfMemoryError e)
+ {
+ getLogger().error("OutOfMemoryError in freeMemory()");
+ }
+ }
+ }
+ catch (Exception e)
+ {
+ getLogger().error("Error in freeMemory()", e);
+ }
+ catch (OutOfMemoryError e)
+ {
+ getLogger().error("OutOfMemoryError in freeMemory()");
+ }
+ }
+
+ /**
+ * This method claculates the number of Elements to be freememory
+ * out of the Cache.
+ *
+ * @param store the Store which was selected as victim
+ * @return number of elements to be removed!
+ */
+ private int calcToFree(Store store)
+ {
+ int cnt = store.size();
+ if (cnt < 0)
+ {
+ if ( getLogger().isDebugEnabled() )
+ {
+ getLogger().debug("Unknown size of the store: " + store);
+ }
+ return 0;
+ }
+ final int res = (int)(cnt * fraction);
+ if ( getLogger().isDebugEnabled() )
+ {
+ getLogger().debug("Calculating size for store " + store + " with size " + cnt + " : " + res);
+ }
+ return res;
+ }
+
+ /**
+ * This method forces the garbage collector
+ */
+ private void freePhysicalMemory()
+ {
+ if (getLogger().isDebugEnabled())
+ {
+ getLogger().debug("Invoking garbage collection. Memory total: "
+ + getJVM().totalMemory() + ", free: "
+ + getJVM().freeMemory());
+ }
+
+ getJVM().runFinalization();
+ getJVM().gc();
+
+ if (getLogger().isDebugEnabled())
+ {
+ getLogger().debug("Garbage collection complete. Memory total: "
+ + getJVM().totalMemory() + ", free: "
+ + getJVM().freeMemory());
+ }
+ }
+
+
+ private int getMinFreeMemory()
+ {
+ return this.minFreeMemory;
+ }
+
+ private void setMinFreeMemory(int _freememory)
+ {
+ this.minFreeMemory = _freememory;
+ }
+
+ private int getMaxHeapSize()
+ {
+ return this.maxHeapSize;
+ }
+
+ private void setMaxHeapSize(int _heapsize)
+ {
+ this.maxHeapSize = _heapsize;
+ }
+
+ private int getPriority()
+ {
+ return this.priority;
+ }
+
+ private void setPriority(int _priority)
+ {
+ this.priority = _priority;
+ }
+
+ private int getThreadInterval()
+ {
+ return this.threadInterval;
+ }
+
+ private void setThreadInterval(int _threadInterval)
+ {
+ this.threadInterval = _threadInterval;
+ }
+
+ private boolean getAdaptiveThreadInterval()
+ {
+ return this.adaptiveThreadInterval;
+ }
+
+ private void setAdaptiveThreadInterval(boolean _adaptiveThreadInterval)
+ {
+ this.adaptiveThreadInterval = _adaptiveThreadInterval;
+ }
+
+ private Runtime getJVM()
+ {
+ return this.jvm;
+ }
+
+ private void setJVM(Runtime _jvm)
+ {
+ this.jvm = _jvm;
+ }
+
+ private ArrayList getStoreList()
+ {
+ return this.storelist;
+ }
+
+ private void setStoreList(ArrayList _storelist)
+ {
+ this.storelist = _storelist;
+ }
+
+ private void setIndex(int _index)
+ {
+ if (getLogger().isDebugEnabled())
+ {
+ getLogger().debug("Setting index=" + _index);
+ }
+ this.index = _index;
+ }
+
+ private int getIndex()
+ {
+ return this.index;
+ }
+}