You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@velocity.apache.org by wg...@apache.org on 2007/10/07 02:31:03 UTC
svn commit: r582568 - in /velocity/texen/trunk: build/build.xml
src/java/org/apache/texen/Generator.java test/manyfiles/
test/manyfiles/Control.vm test/manyfiles/NumFacts.vm
Author: wglass
Date: Sat Oct 6 17:31:01 2007
New Revision: 582568
URL: http://svn.apache.org/viewvc?rev=582568&view=rev
Log:
Use an LRU map to ensure that Texen can generate thousands of files without running out of file handles. Addresses TEXEN-1.
Added:
velocity/texen/trunk/test/manyfiles/
velocity/texen/trunk/test/manyfiles/Control.vm (with props)
velocity/texen/trunk/test/manyfiles/NumFacts.vm (with props)
Modified:
velocity/texen/trunk/build/build.xml
velocity/texen/trunk/src/java/org/apache/texen/Generator.java
Modified: velocity/texen/trunk/build/build.xml
URL: http://svn.apache.org/viewvc/velocity/texen/trunk/build/build.xml?rev=582568&r1=582567&r2=582568&view=diff
==============================================================================
--- velocity/texen/trunk/build/build.xml (original)
+++ velocity/texen/trunk/build/build.xml Sat Oct 6 17:31:01 2007
@@ -510,7 +510,7 @@
<target name="test"
- depends="build-prepare,compile-test,test-texen,test-texen-classpath"/>
+ depends="build-prepare,compile-test,test-texen,test-texen-classpath,test-manyfiles"/>
<!-- ================================================================ -->
<!-- T E X E N T E S T -->
@@ -538,6 +538,29 @@
</target>
+ <!-- ================================================================ -->
+ <!-- T E S T M A N Y F I L E S -->
+ <!-- ================================================================ -->
+ <!-- Test generation of large numbers of files. -->
+ <!-- ================================================================ -->
+ <target name="test-manyfiles">
+
+ <taskdef name="texen" classname="org.apache.texen.ant.TexenTask"
+ classpathref="test.classpath"/>
+
+ <echo message="Checking to see that Texen can generate many files without running out of filehandles."/>
+
+ <texen
+ controlTemplate="Control.vm"
+ outputDirectory="${build.test}/texen-manyfiles"
+ templatePath="${test.dir}/manyfiles"
+ outputFile="report"
+ />
+
+
+ </target>
+
+
<!-- ================================================================ -->
<!-- T E X E N C L A S S P A T H -->
<!-- ================================================================ -->
Modified: velocity/texen/trunk/src/java/org/apache/texen/Generator.java
URL: http://svn.apache.org/viewvc/velocity/texen/trunk/src/java/org/apache/texen/Generator.java?rev=582568&r1=582567&r2=582568&view=diff
==============================================================================
--- velocity/texen/trunk/src/java/org/apache/texen/Generator.java (original)
+++ velocity/texen/trunk/src/java/org/apache/texen/Generator.java Sat Oct 6 17:31:01 2007
@@ -19,27 +19,30 @@
* under the License.
*/
+import java.io.BufferedInputStream;
+import java.io.BufferedWriter;
import java.io.File;
-import java.io.InputStream;
import java.io.FileInputStream;
-import java.io.BufferedInputStream;
-import java.io.Writer;
+import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
-import java.io.StringWriter;
+import java.io.InputStream;
import java.io.OutputStreamWriter;
-import java.io.BufferedWriter;
-import java.io.FileOutputStream;
-
+import java.io.StringWriter;
+import java.io.Writer;
import java.util.Enumeration;
+import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.Properties;
+import java.util.Set;
+import org.apache.commons.collections.map.AbstractLinkedMap;
+import org.apache.commons.collections.map.LRUMap;
import org.apache.velocity.Template;
-import org.apache.velocity.context.Context;
import org.apache.velocity.VelocityContext;
import org.apache.velocity.app.VelocityEngine;
+import org.apache.velocity.context.Context;
import org.apache.velocity.util.ClassUtils;
/**
@@ -84,8 +87,18 @@
* then once then the additional output will be
* appended to the file instead of overwritting
* the contents.
- */
- private Hashtable writers = new Hashtable();
+ *
+ * However, we don't want to keep too many files open,
+ * so use this to expire them.
+ */
+ private LRUMap writers = new FileClosingLRUMap(100);
+
+ /**
+ * Keep track of every file we ever write to. If we
+ * need to reopen a file we already wrote to, reopen
+ * the file in append mode.
+ */
+ private Set writtenFilenames = new HashSet();
/**
* The generator tools used for creating additional
@@ -284,15 +297,22 @@
* @return A Writer for this generator.
* @throws Exception
*/
- public Writer getWriter(String path, String encoding) throws Exception {
+ public Writer getWriter(String path, String encoding) throws Exception
+ {
Writer writer;
- if (encoding == null || encoding.length() == 0 || encoding.equals("8859-1") || encoding.equals("8859_1")) {
- writer = new FileWriter(path);
+ boolean seenBefore = writtenFilenames.contains(path);
+ if ( encoding == null ||
+ encoding.length() == 0 ||
+ encoding.equals("8859-1") ||
+ encoding.equals("8859_1"))
+ {
+ writer = new FileWriter(path, seenBefore);
}
else
{
- writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(path), encoding));
+ writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(path, seenBefore), encoding));
}
+ writtenFilenames.add(path);
return writer;
}
@@ -395,7 +415,7 @@
if (writers.get(outputFile) == null)
{
/*
- * We have never seen this file before so create
+ * We don't have a writer now so create
* a new file writer for it.
*/
writer = getWriter(
@@ -558,4 +578,41 @@
// clear the file writers cache
writers.clear();
}
+
+
+ private static class FileClosingLRUMap extends LRUMap
+ {
+ private static final long serialVersionUID = -8944324477458767333L;
+
+ public FileClosingLRUMap(int i)
+ {
+ super(i);
+ }
+
+ protected boolean removeLRU(AbstractLinkedMap.LinkEntry entry)
+ {
+ Writer writer = (Writer) entry.getValue();
+ try
+ {
+ writer.flush();
+ }
+ catch (IOException e)
+ {
+ /* do nothing */
+ }
+
+ try
+ {
+ writer.close();
+ }
+ catch (IOException e)
+ {
+ /* do nothing */
+ }
+ return true;
+ }
+ }
+
}
+
+
Added: velocity/texen/trunk/test/manyfiles/Control.vm
URL: http://svn.apache.org/viewvc/velocity/texen/trunk/test/manyfiles/Control.vm?rev=582568&view=auto
==============================================================================
--- velocity/texen/trunk/test/manyfiles/Control.vm (added)
+++ velocity/texen/trunk/test/manyfiles/Control.vm Sat Oct 6 17:31:01 2007
@@ -0,0 +1,4 @@
+#foreach ($num in [1..2000])
+ #set ( $fname = $strings.concat(["Funfacts_", $num, ".txt"]) )
+ $generator.parse("NumFacts.vm", $fname, "num", $num)
+#end
\ No newline at end of file
Propchange: velocity/texen/trunk/test/manyfiles/Control.vm
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: velocity/texen/trunk/test/manyfiles/Control.vm
------------------------------------------------------------------------------
svn:keywords = Id Author Date Revision
Added: velocity/texen/trunk/test/manyfiles/NumFacts.vm
URL: http://svn.apache.org/viewvc/velocity/texen/trunk/test/manyfiles/NumFacts.vm?rev=582568&view=auto
==============================================================================
--- velocity/texen/trunk/test/manyfiles/NumFacts.vm (added)
+++ velocity/texen/trunk/test/manyfiles/NumFacts.vm Sat Oct 6 17:31:01 2007
@@ -0,0 +1 @@
+A good template would be full of facts about the number $num
Propchange: velocity/texen/trunk/test/manyfiles/NumFacts.vm
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: velocity/texen/trunk/test/manyfiles/NumFacts.vm
------------------------------------------------------------------------------
svn:keywords = Id Author Date Revision