You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2011/01/26 10:10:07 UTC
svn commit: r1063647 - in /lucene/dev/trunk/modules/benchmark: ./
CHANGES.txt build.xml
src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java
Author: shaie
Date: Wed Jan 26 09:10:06 2011
New Revision: 1063647
URL: http://svn.apache.org/viewvc?rev=1063647&view=rev
Log:
LUCENE-929: contrib/benchmark build doesn't handle checking if content is properly extracted (trunk)
Modified:
lucene/dev/trunk/modules/benchmark/ (props changed)
lucene/dev/trunk/modules/benchmark/CHANGES.txt
lucene/dev/trunk/modules/benchmark/build.xml
lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java
Modified: lucene/dev/trunk/modules/benchmark/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/benchmark/CHANGES.txt?rev=1063647&r1=1063646&r2=1063647&view=diff
==============================================================================
--- lucene/dev/trunk/modules/benchmark/CHANGES.txt (original)
+++ lucene/dev/trunk/modules/benchmark/CHANGES.txt Wed Jan 26 09:10:06 2011
@@ -2,6 +2,11 @@ Lucene Benchmark Contrib Change Log
The Benchmark contrib package contains code for benchmarking Lucene in a variety of ways.
+01/26/2011
+ LUCENE-929: ExtractReuters first extracts to a tmp dir and then renames. That
+ way, if a previous extract attempt failed, "ant extract-reuters" will still
+ extract the files. (Shai Erera, Doron Cohen, Grant Ingersoll)
+
01/24/2011
LUCENE-2885: Add WaitForMerges task (calls IndexWriter.waitForMerges()).
(Mike McCandless)
Modified: lucene/dev/trunk/modules/benchmark/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/benchmark/build.xml?rev=1063647&r1=1063646&r2=1063647&view=diff
==============================================================================
--- lucene/dev/trunk/modules/benchmark/build.xml (original)
+++ lucene/dev/trunk/modules/benchmark/build.xml Wed Jan 26 09:10:06 2011
@@ -87,7 +87,6 @@
</target>
<target name="extract-reuters" depends="check-files" unless="reuters.extracted">
- <mkdir dir="${working.dir}/reuters-out"/>
<java classname="org.apache.lucene.benchmark.utils.ExtractReuters" maxmemory="1024M" fork="true">
<classpath refid="run.classpath"/>
<arg file="${working.dir}/reuters"/>
Modified: lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java?rev=1063647&r1=1063646&r2=1063647&view=diff
==============================================================================
--- lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java (original)
+++ lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java Wed Jan 26 09:10:06 2011
@@ -29,146 +29,119 @@ import java.util.regex.Pattern;
/**
* Split the Reuters SGML documents into Simple Text files containing: Title, Date, Dateline, Body
*/
-public class ExtractReuters
-{
- private File reutersDir;
- private File outputDir;
- private static final String LINE_SEPARATOR = System.getProperty("line.separator");
-
- public ExtractReuters(File reutersDir, File outputDir)
- {
- this.reutersDir = reutersDir;
- this.outputDir = outputDir;
- System.out.println("Deleting all files in " + outputDir);
- File [] files = outputDir.listFiles();
- for (int i = 0; i < files.length; i++)
- {
- files[i].delete();
- }
-
+public class ExtractReuters {
+ private File reutersDir;
+ private File outputDir;
+ private static final String LINE_SEPARATOR = System.getProperty("line.separator");
+
+ public ExtractReuters(File reutersDir, File outputDir) {
+ this.reutersDir = reutersDir;
+ this.outputDir = outputDir;
+ System.out.println("Deleting all files in " + outputDir);
+ for (File f : outputDir.listFiles()) {
+ f.delete();
}
+ }
- public void extract()
- {
- File [] sgmFiles = reutersDir.listFiles(new FileFilter()
- {
- public boolean accept(File file)
- {
- return file.getName().endsWith(".sgm");
- }
- });
- if (sgmFiles != null && sgmFiles.length > 0)
- {
- for (int i = 0; i < sgmFiles.length; i++)
- {
- File sgmFile = sgmFiles[i];
- extractFile(sgmFile);
- }
- }
- else
- {
- System.err.println("No .sgm files in " + reutersDir);
- }
+ public void extract() {
+ File[] sgmFiles = reutersDir.listFiles(new FileFilter() {
+ public boolean accept(File file) {
+ return file.getName().endsWith(".sgm");
+ }
+ });
+ if (sgmFiles != null && sgmFiles.length > 0) {
+ for (File sgmFile : sgmFiles) {
+ extractFile(sgmFile);
+ }
+ } else {
+ System.err.println("No .sgm files in " + reutersDir);
}
+ }
- Pattern EXTRACTION_PATTERN = Pattern.compile("<TITLE>(.*?)</TITLE>|<DATE>(.*?)</DATE>|<BODY>(.*?)</BODY>");
-
- private static String[] META_CHARS
- = {"&", "<", ">", "\"", "'"};
+ Pattern EXTRACTION_PATTERN = Pattern
+ .compile("<TITLE>(.*?)</TITLE>|<DATE>(.*?)</DATE>|<BODY>(.*?)</BODY>");
- private static String[] META_CHARS_SERIALIZATIONS
- = {"&", "<", ">", """, "'"};
+ private static String[] META_CHARS = { "&", "<", ">", "\"", "'" };
- /**
- * Override if you wish to change what is extracted
- *
- * @param sgmFile
- */
- protected void extractFile(File sgmFile)
- {
- try
- {
- BufferedReader reader = new BufferedReader(new FileReader(sgmFile));
-
- StringBuilder buffer = new StringBuilder(1024);
- StringBuilder outBuffer = new StringBuilder(1024);
-
- String line = null;
- int docNumber = 0;
- while ((line = reader.readLine()) != null)
- {
- //when we see a closing reuters tag, flush the file
-
- if (line.indexOf("</REUTERS") == -1) {
- //Replace the SGM escape sequences
-
- buffer.append(line).append(' ');//accumulate the strings for now, then apply regular expression to get the pieces,
- }
- else
- {
- //Extract the relevant pieces and write to a file in the output dir
- Matcher matcher = EXTRACTION_PATTERN.matcher(buffer);
- while (matcher.find())
- {
- for (int i = 1; i <= matcher.groupCount(); i++)
- {
- if (matcher.group(i) != null)
- {
- outBuffer.append(matcher.group(i));
- }
- }
- outBuffer.append(LINE_SEPARATOR).append(LINE_SEPARATOR);
- }
- String out = outBuffer.toString();
- for (int i = 0; i < META_CHARS_SERIALIZATIONS.length; i++)
- {
- out = out.replaceAll(META_CHARS_SERIALIZATIONS[i], META_CHARS[i]);
- }
- File outFile = new File(outputDir, sgmFile.getName() + "-" + (docNumber++) + ".txt");
- //System.out.println("Writing " + outFile);
- FileWriter writer = new FileWriter(outFile);
- writer.write(out);
- writer.close();
- outBuffer.setLength(0);
- buffer.setLength(0);
- }
+ private static String[] META_CHARS_SERIALIZATIONS = { "&", "<",
+ ">", """, "'" };
+
+ /**
+ * Override if you wish to change what is extracted
+ *
+ * @param sgmFile
+ */
+ protected void extractFile(File sgmFile) {
+ try {
+ BufferedReader reader = new BufferedReader(new FileReader(sgmFile));
+
+ StringBuilder buffer = new StringBuilder(1024);
+ StringBuilder outBuffer = new StringBuilder(1024);
+
+ String line = null;
+ int docNumber = 0;
+ while ((line = reader.readLine()) != null) {
+ // when we see a closing reuters tag, flush the file
+
+ if (line.indexOf("</REUTERS") == -1) {
+ // Replace the SGM escape sequences
+
+ buffer.append(line).append(' ');// accumulate the strings for now,
+ // then apply regular expression to
+ // get the pieces,
+ } else {
+ // Extract the relevant pieces and write to a file in the output dir
+ Matcher matcher = EXTRACTION_PATTERN.matcher(buffer);
+ while (matcher.find()) {
+ for (int i = 1; i <= matcher.groupCount(); i++) {
+ if (matcher.group(i) != null) {
+ outBuffer.append(matcher.group(i));
+ }
}
- reader.close();
- }
-
- catch (
- IOException e
- )
-
- {
- throw new RuntimeException(e);
- }
+ outBuffer.append(LINE_SEPARATOR).append(LINE_SEPARATOR);
+ }
+ String out = outBuffer.toString();
+ for (int i = 0; i < META_CHARS_SERIALIZATIONS.length; i++) {
+ out = out.replaceAll(META_CHARS_SERIALIZATIONS[i], META_CHARS[i]);
+ }
+ File outFile = new File(outputDir, sgmFile.getName() + "-"
+ + (docNumber++) + ".txt");
+ // System.out.println("Writing " + outFile);
+ FileWriter writer = new FileWriter(outFile);
+ writer.write(out);
+ writer.close();
+ outBuffer.setLength(0);
+ buffer.setLength(0);
+ }
+ }
+ reader.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
+ }
-
- public static void main(String[] args)
- {
- if (args.length != 2)
- {
- printUsage();
- }
- File reutersDir = new File(args[0]);
-
- if (reutersDir.exists())
- {
- File outputDir = new File(args[1]);
- outputDir.mkdirs();
- ExtractReuters extractor = new ExtractReuters(reutersDir, outputDir);
- extractor.extract();
- }
- else
- {
- printUsage();
- }
+ public static void main(String[] args) {
+ if (args.length != 2) {
+ printUsage();
}
-
- private static void printUsage()
- {
- System.err.println("Usage: java -cp <...> org.apache.lucene.benchmark.utils.ExtractReuters <Path to Reuters SGM files> <Output Path>");
+ File reutersDir = new File(args[0]);
+ if (!reutersDir.exists()) {
+ printUsage();
+ return;
}
+
+ // First, extract to a tmp directory and only if everything succeeds, rename
+ // to output directory.
+ File outputDir = new File(args[1] + "-tmp");
+ outputDir.mkdirs();
+ ExtractReuters extractor = new ExtractReuters(reutersDir, outputDir);
+ extractor.extract();
+ // Now rename to requested output dir
+ outputDir.renameTo(new File(args[1]));
+ }
+
+ private static void printUsage() {
+ System.err.println("Usage: java -cp <...> org.apache.lucene.benchmark.utils.ExtractReuters <Path to Reuters SGM files> <Output Path>");
+ }
+
}