You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by do...@apache.org on 2011/02/03 10:50:53 UTC
svn commit: r1066771 - in /lucene/dev/branches/branch_3x/lucene:
contrib/benchmark/
contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/
contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/
contrib/benchmark/src/test/org...
Author: doronc
Date: Thu Feb 3 09:50:53 2011
New Revision: 1066771
URL: http://svn.apache.org/viewvc?rev=1066771&view=rev
Log:
LUCENE-1540: Improvements to contrib.benchmark for TREC collections
Added:
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocParser.java
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFBISParser.java
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFR94Parser.java
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFTParser.java
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecGov2Parser.java (with props)
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecLATimesParser.java
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecParserByPath.java
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/trecdocs.zip (with props)
Modified:
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/HTMLParser.java
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StringBuilderReader.java
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/_TestUtil.java
Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/CHANGES.txt?rev=1066771&r1=1066770&r2=1066771&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/CHANGES.txt Thu Feb 3 09:50:53 2011
@@ -2,6 +2,12 @@ Lucene Benchmark Contrib Change Log
The Benchmark contrib package contains code for benchmarking Lucene in a variety of ways.
+02/03/2011
+ LUCENE-1540: Improvements to contrib.benchmark for TREC collections.
+ ContentSource can now process plain text files, gzip files, and bzip2 files.
+ TREC doc parsing now handles the TREC gov2 collection and TREC disks 4&5-CR
+ collection (both used by many TREC tasks). (Shai Erera, Doron Cohen)
+
01/26/2011
LUCENE-929: ExtractReuters first extracts to a tmp dir and then renames. That
way, if a previous extract attempt failed, "ant extract-reuters" will still
Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java?rev=1066771&r1=1066770&r2=1066771&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java Thu Feb 3 09:50:53 2011
@@ -56,11 +56,14 @@ import org.apache.lucene.benchmark.byTas
public abstract class ContentSource {
private static final int BZIP = 0;
- private static final int OTHER = 1;
+ private static final int GZIP = 1;
+ private static final int OTHER = 2;
private static final Map<String,Integer> extensionToType = new HashMap<String,Integer>();
static {
extensionToType.put(".bz2", Integer.valueOf(BZIP));
extensionToType.put(".bzip", Integer.valueOf(BZIP));
+ extensionToType.put(".gz", Integer.valueOf(GZIP));
+ extensionToType.put(".gzip", Integer.valueOf(GZIP));
}
protected static final int BUFFER_SIZE = 1 << 16; // 64K
@@ -78,11 +81,13 @@ public abstract class ContentSource {
private CompressorStreamFactory csFactory = new CompressorStreamFactory();
+ /** update count of bytes generated by this source */
protected final synchronized void addBytes(long numBytes) {
bytesCount += numBytes;
totalBytesCount += numBytes;
}
+ /** update count of documents generated by this source */
protected final synchronized void addDoc() {
++docsCount;
++totalDocsCount;
@@ -130,21 +135,25 @@ public abstract class ContentSource {
type = typeInt.intValue();
}
}
- switch (type) {
- case BZIP:
- try {
+
+ try {
+ switch (type) {
+ case BZIP:
// According to BZip2CompressorInputStream's code, it reads the first
// two file header chars ('B' and 'Z'). It is important to wrap the
// underlying input stream with a buffered one since
// Bzip2CompressorInputStream uses the read() method exclusively.
is = csFactory.createCompressorInputStream("bzip2", is);
- } catch (CompressorException e) {
- IOException ioe = new IOException(e.getMessage());
- ioe.initCause(e);
- throw ioe;
- }
- break;
- default: // Do nothing, stay with FileInputStream
+ break;
+ case GZIP:
+ is = csFactory.createCompressorInputStream("gz", is);
+ break;
+ default: // Do nothing, stay with FileInputStream
+ }
+ } catch (CompressorException e) {
+ IOException ioe = new IOException(e.getMessage());
+ ioe.initCause(e);
+ throw ioe;
}
return is;
Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java?rev=1066771&r1=1066770&r2=1066771&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java Thu Feb 3 09:50:53 2011
@@ -29,11 +29,14 @@ import java.util.Properties;
*/
public class DemoHTMLParser implements org.apache.lucene.benchmark.byTask.feeds.HTMLParser {
- public DocData parse(DocData docData, String name, Date date, Reader reader, DateFormat dateFormat) throws IOException, InterruptedException {
+ public DocData parse(DocData docData, String name, Date date, String title, Reader reader, DateFormat dateFormat) throws IOException, InterruptedException {
org.apache.lucene.demo.html.HTMLParser p = new org.apache.lucene.demo.html.HTMLParser(reader);
// title
- String title = p.getTitle();
+ if (title==null) {
+ title = p.getTitle();
+ }
+
// properties
Properties props = p.getMetaTags();
// body
Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/HTMLParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/HTMLParser.java?rev=1066771&r1=1066770&r2=1066771&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/HTMLParser.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/HTMLParser.java Thu Feb 3 09:50:53 2011
@@ -29,16 +29,18 @@ public interface HTMLParser {
/**
* Parse the input Reader and return DocData.
- * A provided name or date is used for the result, otherwise an attempt is
- * made to set them from the parsed data.
- * @param dateFormat date formatter to use for extracting the date.
- * @param name name of the result doc data. If null, attempt to set by parsed data.
+ * The provided name,title,date are used for the result, unless when they're null,
+ * in which case an attempt is made to set them from the parsed data.
+ * @param docData result reused
+ * @param name name of the result doc data.
* @param date date of the result doc data. If null, attempt to set by parsed data.
- * @param reader of html text to parse.
+ * @param title title of the result doc data. If null, attempt to set by parsed data.
+ * @param reader reader of html text to parse.
+ * @param dateFormat date formatter to use for extracting the date.
* @return Parsed doc data.
* @throws IOException
* @throws InterruptedException
*/
- public DocData parse(DocData docData, String name, Date date, Reader reader, DateFormat dateFormat) throws IOException, InterruptedException;
+ public DocData parse(DocData docData, String name, Date date, String title, Reader reader, DateFormat dateFormat) throws IOException, InterruptedException;
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java?rev=1066771&r1=1066770&r2=1066771&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java Thu Feb 3 09:50:53 2011
@@ -19,8 +19,8 @@ package org.apache.lucene.benchmark.byTa
import java.io.BufferedReader;
import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.text.DateFormat;
@@ -29,8 +29,8 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Locale;
-import java.util.zip.GZIPInputStream;
+import org.apache.lucene.benchmark.byTask.feeds.TrecDocParser.ParsePathType;
import org.apache.lucene.benchmark.byTask.utils.Config;
import org.apache.lucene.benchmark.byTask.utils.StringBuilderReader;
import org.apache.lucene.util.ThreadInterruptedException;
@@ -46,8 +46,10 @@ import org.apache.lucene.util.ThreadInte
* <li><b>docs.dir</b> - specifies the directory where the TREC files reside.
* Can be set to a relative path if "work.dir" is also specified
* (<b>default=trec</b>).
+ * <li><b>trec.doc.parser</b> - specifies the {@link TrecDocParser} class to use for
+ * parsing the TREC documents content (<b>default=TrecGov2Parser</b>).
* <li><b>html.parser</b> - specifies the {@link HTMLParser} class to use for
- * parsing the TREC documents content (<b>default=DemoHTMLParser</b>).
+ * parsing the HTML parts of the TREC documents content (<b>default=DemoHTMLParser</b>).
* <li><b>content.source.encoding</b> - if not specified, ISO-8859-1 is used.
* <li><b>content.source.excludeIteration</b> - if true, do not append iteration number to docname
* </ul>
@@ -59,22 +61,24 @@ public class TrecContentSource extends C
ParsePosition pos;
}
- private static final String DATE = "Date: ";
- private static final String DOCHDR = "<DOCHDR>";
- private static final String TERMINATING_DOCHDR = "</DOCHDR>";
- private static final String DOCNO = "<DOCNO>";
- private static final String TERMINATING_DOCNO = "</DOCNO>";
- private static final String DOC = "<DOC>";
- private static final String TERMINATING_DOC = "</DOC>";
+ public static final String DOCNO = "<DOCNO>";
+ public static final String TERMINATING_DOCNO = "</DOCNO>";
+ public static final String DOC = "<DOC>";
+ public static final String TERMINATING_DOC = "</DOC>";
- private static final String NEW_LINE = System.getProperty("line.separator");
+ /** separator between lines in the byffer */
+ public static final String NEW_LINE = System.getProperty("line.separator");
private static final String DATE_FORMATS [] = {
- "EEE, dd MMM yyyy kk:mm:ss z", // Tue, 09 Dec 2003 22:39:08 GMT
- "EEE MMM dd kk:mm:ss yyyy z", // Tue Dec 09 16:45:08 2003 EST
- "EEE, dd-MMM-':'y kk:mm:ss z", // Tue, 09 Dec 2003 22:39:08 GMT
- "EEE, dd-MMM-yyy kk:mm:ss z", // Tue, 09 Dec 2003 22:39:08 GMT
- "EEE MMM dd kk:mm:ss yyyy", // Tue Dec 09 16:45:08 2003
+ "EEE, dd MMM yyyy kk:mm:ss z", // Tue, 09 Dec 2003 22:39:08 GMT
+ "EEE MMM dd kk:mm:ss yyyy z", // Tue Dec 09 16:45:08 2003 EST
+ "EEE, dd-MMM-':'y kk:mm:ss z", // Tue, 09 Dec 2003 22:39:08 GMT
+ "EEE, dd-MMM-yyy kk:mm:ss z", // Tue, 09 Dec 2003 22:39:08 GMT
+ "EEE MMM dd kk:mm:ss yyyy", // Tue Dec 09 16:45:08 2003
+ "dd MMM yyyy", // 1 March 1994
+ "MMM dd, yyyy", // February 3, 1994
+ "yyMMdd", // 910513
+ "hhmm z.z.z. MMM dd, yyyy", // 0901 u.t.c. April 28, 1994
};
private ThreadLocal<DateFormatInfo> dateFormats = new ThreadLocal<DateFormatInfo>();
@@ -83,7 +87,7 @@ public class TrecContentSource extends C
private File dataDir = null;
private ArrayList<File> inputFiles = new ArrayList<File>();
private int nextFile = 0;
- private int rawDocSize;
+ private int rawDocSize = 0;
// Use to synchronize threads on reading from the TREC documents.
private Object lock = new Object();
@@ -92,7 +96,10 @@ public class TrecContentSource extends C
BufferedReader reader;
int iteration = 0;
HTMLParser htmlParser;
+
private boolean excludeDocnameIteration;
+ private TrecDocParser trecDocParser = new TrecGov2Parser(); // default
+ ParsePathType currPathType; // not private for tests
private DateFormatInfo getDateFormatInfo() {
DateFormatInfo dfi = dateFormats.get();
@@ -118,7 +125,7 @@ public class TrecContentSource extends C
return sb;
}
- private Reader getTrecDocReader(StringBuilder docBuffer) {
+ Reader getTrecDocReader(StringBuilder docBuffer) {
StringBuilderReader r = trecDocReader.get();
if (r == null) {
r = new StringBuilderReader(docBuffer);
@@ -129,10 +136,21 @@ public class TrecContentSource extends C
return r;
}
- // read until finding a line that starts with the specified prefix, or a terminating tag has been found.
- private void read(StringBuilder buf, String prefix, boolean collectMatchLine,
- boolean collectAll, String terminatingTag)
- throws IOException, NoMoreDataException {
+ HTMLParser getHtmlParser() {
+ return htmlParser;
+ }
+
+ /**
+ * Read until a line starting with the specified <code>lineStart</code>.
+ * @param buf buffer for collecting the data if so specified/
+ * @param lineStart line start to look for, must not be null.
+ * @param collectMatchLine whether to collect the matching line into <code>buffer</code>.
+ * @param collectAll whether to collect all lines into <code>buffer</code>.
+ * @throws IOException
+ * @throws NoMoreDataException
+ */
+ private void read(StringBuilder buf, String lineStart,
+ boolean collectMatchLine, boolean collectAll) throws IOException, NoMoreDataException {
String sep = "";
while (true) {
String line = reader.readLine();
@@ -144,20 +162,12 @@ public class TrecContentSource extends C
rawDocSize += line.length();
- if (line.startsWith(prefix)) {
+ if (lineStart!=null && line.startsWith(lineStart)) {
if (collectMatchLine) {
buf.append(sep).append(line);
sep = NEW_LINE;
}
- break;
- }
-
- if (terminatingTag != null && line.startsWith(terminatingTag)) {
- // didn't find the prefix that was asked, but the terminating
- // tag was found. set the length to 0 to signal no match was
- // found.
- buf.setLength(0);
- break;
+ return;
}
if (collectAll) {
@@ -169,7 +179,7 @@ public class TrecContentSource extends C
void openNextFile() throws NoMoreDataException, IOException {
close();
- int retries = 0;
+ currPathType = null;
while (true) {
if (nextFile >= inputFiles.size()) {
// exhausted files, start a new round, unless forever set to false.
@@ -184,13 +194,13 @@ public class TrecContentSource extends C
System.out.println("opening: " + f + " length: " + f.length());
}
try {
- GZIPInputStream zis = new GZIPInputStream(new FileInputStream(f), BUFFER_SIZE);
- reader = new BufferedReader(new InputStreamReader(zis, encoding), BUFFER_SIZE);
+ InputStream inputStream = getInputStream(f); // support either gzip, bzip2, or regular text file, by extension
+ reader = new BufferedReader(new InputStreamReader(inputStream, encoding), BUFFER_SIZE);
+ currPathType = TrecDocParser.pathType(f);
return;
} catch (Exception e) {
- retries++;
- if (retries < 20 && verbose) {
- System.out.println("Skipping 'bad' file " + f.getAbsolutePath() + " #retries=" + retries);
+ if (verbose) {
+ System.out.println("Skipping 'bad' file " + f.getAbsolutePath()+" due to "+e.getMessage());
continue;
}
throw new NoMoreDataException();
@@ -198,7 +208,7 @@ public class TrecContentSource extends C
}
}
- Date parseDate(String dateStr) {
+ public Date parseDate(String dateStr) {
dateStr = dateStr.trim();
DateFormatInfo dfi = getDateFormatInfo();
for (int i = 0; i < dfi.dfs.length; i++) {
@@ -237,70 +247,47 @@ public class TrecContentSource extends C
@Override
public DocData getNextDocData(DocData docData) throws NoMoreDataException, IOException {
- String dateStr = null, name = null;
- Reader r = null;
+ String name = null;
+ StringBuilder docBuf = getDocBuffer();
+ ParsePathType parsedPathType;
+
// protect reading from the TREC files by multiple threads. The rest of the
- // method, i.e., parsing the content and returning the DocData can run
- // unprotected.
+ // method, i.e., parsing the content and returning the DocData can run unprotected.
synchronized (lock) {
if (reader == null) {
openNextFile();
}
-
- StringBuilder docBuf = getDocBuffer();
- // 1. skip until doc start
+ // 1. skip until doc start - required for all TREC formats
docBuf.setLength(0);
- read(docBuf, DOC, false, false, null);
-
- // 2. name
+ read(docBuf, DOC, false, false);
+
+ // save parsedFile for passing trecDataParser after the sync block, in
+ // case another thread will open another file in between.
+ parsedPathType = currPathType;
+
+ // 2. name - required for all TREC formats
docBuf.setLength(0);
- read(docBuf, DOCNO, true, false, null);
+ read(docBuf, DOCNO, true, false);
name = docBuf.substring(DOCNO.length(), docBuf.indexOf(TERMINATING_DOCNO,
- DOCNO.length()));
- if (!excludeDocnameIteration)
+ DOCNO.length())).trim();
+
+ if (!excludeDocnameIteration) {
name = name + "_" + iteration;
-
- // 3. skip until doc header
- docBuf.setLength(0);
- read(docBuf, DOCHDR, false, false, null);
-
- boolean findTerminatingDocHdr = false;
-
- // 4. date - look for the date only until /DOCHDR
- docBuf.setLength(0);
- read(docBuf, DATE, true, false, TERMINATING_DOCHDR);
- if (docBuf.length() != 0) {
- // Date found.
- dateStr = docBuf.substring(DATE.length());
- findTerminatingDocHdr = true;
}
- // 5. skip until end of doc header
- if (findTerminatingDocHdr) {
- docBuf.setLength(0);
- read(docBuf, TERMINATING_DOCHDR, false, false, null);
- }
-
- // 6. collect until end of doc
+ // 3. read all until end of doc
docBuf.setLength(0);
- read(docBuf, TERMINATING_DOC, false, true, null);
-
- // 7. Set up a Reader over the read content
- r = getTrecDocReader(docBuf);
- // Resetting the thread's reader means it will reuse the instance
- // allocated as well as re-read from docBuf.
- r.reset();
-
- // count char length of parsed html text (larger than the plain doc body text).
- addBytes(docBuf.length());
+ read(docBuf, TERMINATING_DOC, false, true);
}
+
+ // count char length of text to be parsed (may be larger than the resulted plain doc body text).
+ addBytes(docBuf.length());
// This code segment relies on HtmlParser being thread safe. When we get
// here, everything else is already private to that thread, so we're safe.
- Date date = dateStr != null ? parseDate(dateStr) : null;
try {
- docData = htmlParser.parse(docData, name, date, r, null);
+ docData = trecDocParser.parse(docData, name, this, docBuf, parsedPathType);
addDoc();
} catch (InterruptedException ie) {
throw new ThreadInterruptedException(ie);
@@ -322,27 +309,40 @@ public class TrecContentSource extends C
@Override
public void setConfig(Config config) {
super.setConfig(config);
+ // dirs
File workDir = new File(config.get("work.dir", "work"));
String d = config.get("docs.dir", "trec");
dataDir = new File(d);
if (!dataDir.isAbsolute()) {
dataDir = new File(workDir, d);
}
+ // files
collectFiles(dataDir, inputFiles);
if (inputFiles.size() == 0) {
throw new IllegalArgumentException("No files in dataDir: " + dataDir);
}
+ // trec doc parser
+ try {
+ String trecDocParserClassName = config.get("trec.doc.parser", "org.apache.lucene.benchmark.byTask.feeds.TrecGov2Parser");
+ trecDocParser = Class.forName(trecDocParserClassName).asSubclass(TrecDocParser.class).newInstance();
+ } catch (Exception e) {
+ // Should not get here. Throw runtime exception.
+ throw new RuntimeException(e);
+ }
+ // html parser
try {
- String parserClassName = config.get("html.parser",
+ String htmlParserClassName = config.get("html.parser",
"org.apache.lucene.benchmark.byTask.feeds.DemoHTMLParser");
- htmlParser = Class.forName(parserClassName).asSubclass(HTMLParser.class).newInstance();
+ htmlParser = Class.forName(htmlParserClassName).asSubclass(HTMLParser.class).newInstance();
} catch (Exception e) {
// Should not get here. Throw runtime exception.
throw new RuntimeException(e);
}
+ // encoding
if (encoding == null) {
encoding = "ISO-8859-1";
}
+ // iteration exclusion in doc name
excludeDocnameIteration = config.get("content.source.excludeIteration", false);
}
Added: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocParser.java?rev=1066771&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocParser.java (added)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocParser.java Thu Feb 3 09:50:53 2011
@@ -0,0 +1,135 @@
+package org.apache.lucene.benchmark.byTask.feeds;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Parser for trec doc content, invoked on doc text excluding <DOC> and <DOCNO>
+ * which are handled in TrecContentSource. Required to be stateless and hence thread safe.
+ */
+public abstract class TrecDocParser {
+
+ /** Types of trec parse paths, */
+ public enum ParsePathType { GOV2, FBIS, FT, FR94, LATIMES }
+
+ /** trec parser type used for unknown extensions */
+ public static final ParsePathType DEFAULT_PATH_TYPE = ParsePathType.GOV2;
+
+ static final Map<ParsePathType,TrecDocParser> pathType2parser = new HashMap<ParsePathType,TrecDocParser>();
+ static {
+ pathType2parser.put(ParsePathType.GOV2, new TrecGov2Parser());
+ pathType2parser.put(ParsePathType.FBIS, new TrecFBISParser());
+ pathType2parser.put(ParsePathType.FR94, new TrecFR94Parser());
+ pathType2parser.put(ParsePathType.FT, new TrecFTParser());
+ pathType2parser.put(ParsePathType.LATIMES, new TrecLATimesParser());
+ }
+
+ static final Map<String,ParsePathType> pathName2Type = new HashMap<String,ParsePathType>();
+ static {
+ for (ParsePathType ppt : ParsePathType.values()) {
+ pathName2Type.put(ppt.name(),ppt);
+ }
+ }
+
+ /** max length of walk up from file to its ancestors when looking for a known path type */
+ private static final int MAX_PATH_LENGTH = 10;
+
+ /**
+ * Compute the path type of a file by inspecting name of file and its parents
+ */
+ public static ParsePathType pathType(File f) {
+ int pathLength = 0;
+ while (f != null && ++pathLength < MAX_PATH_LENGTH) {
+ ParsePathType ppt = pathName2Type.get(f.getName().toUpperCase());
+ if (ppt!=null) {
+ return ppt;
+ }
+ f = f.getParentFile();
+ }
+ return DEFAULT_PATH_TYPE;
+ }
+
+ /**
+ * parse the text prepared in docBuf into a result DocData,
+ * no synchronization is required.
+ * @param docData reusable result
+ * @param name name that should be set to the result
+ * @param trecSrc calling trec content source
+ * @param docBuf text to parse
+ * @param pathType type of parsed file, or null if unknown - may be used by
+ * parsers to alter their behavior according to the file path type.
+ */
+ public abstract DocData parse(DocData docData, String name, TrecContentSource trecSrc,
+ StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException;
+
+ /**
+ * strip tags from <code>buf</code>: each tag is replaced by a single blank.
+ * @return text obtained when stripping all tags from <code>buf</code> (Input StringBuilder is unmodified).
+ */
+ public static String stripTags(StringBuilder buf, int start) {
+ return stripTags(buf.substring(start),0);
+ }
+
+ /**
+ * strip tags from input.
+ * @see #stripTags(StringBuilder, int)
+ */
+ public static String stripTags(String buf, int start) {
+ if (start>0) {
+ buf = buf.substring(0);
+ }
+ return buf.replaceAll("<[^>]*>", " ");
+ }
+
+ /**
+ * Extract from <code>buf</code> the text of interest within specified tags
+ * @param buf entire input text
+ * @param startTag tag marking start of text of interest
+ * @param endTag tag marking end of text of interest
+ * @param maxPos if ≥ 0 sets a limit on start of text of interest
+ * @return text of interest or null if not found
+ */
+ public static String extract(StringBuilder buf, String startTag, String endTag, int maxPos, String noisePrefixes[]) {
+ int k1 = buf.indexOf(startTag);
+ if (k1>=0 && (maxPos<0 || k1<maxPos)) {
+ k1 += startTag.length();
+ int k2 = buf.indexOf(endTag,k1);
+ if (k2>=0 && (maxPos<0 || k2<maxPos)) { // found end tag with allowed range
+ if (noisePrefixes != null) {
+ for (String noise : noisePrefixes) {
+ int k1a = buf.indexOf(noise,k1);
+ if (k1a>=0 && k1a<k2) {
+ k1 = k1a + noise.length();
+ }
+ }
+ }
+ return buf.substring(k1,k2).trim();
+ }
+ }
+ return null;
+ }
+
+ //public static void main(String[] args) {
+ // System.out.println(stripTags("is it true that<space>2<<second space>><almost last space>1<one more space>?",0));
+ //}
+
+}
Added: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFBISParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFBISParser.java?rev=1066771&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFBISParser.java (added)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFBISParser.java Thu Feb 3 09:50:53 2011
@@ -0,0 +1,65 @@
+package org.apache.lucene.benchmark.byTask.feeds;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Date;
+
+/**
+ * Parser for the FBIS docs in trec disks 4+5 collection format
+ */
+public class TrecFBISParser extends TrecDocParser {
+
+ private static final String HEADER = "<HEADER>";
+ private static final String HEADER_END = "</HEADER>";
+ private static final int HEADER_END_LENGTH = HEADER_END.length();
+
+ private static final String DATE1 = "<DATE1>";
+ private static final String DATE1_END = "</DATE1>";
+
+ private static final String TI = "<TI>";
+ private static final String TI_END = "</TI>";
+
+ @Override
+ public DocData parse(DocData docData, String name, TrecContentSource trecSrc,
+ StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException {
+ int mark = 0; // that much is skipped
+ // optionally skip some of the text, set date, title
+ Date date = null;
+ String title = null;
+ int h1 = docBuf.indexOf(HEADER);
+ if (h1>=0) {
+ int h2 = docBuf.indexOf(HEADER_END,h1);
+ mark = h2+HEADER_END_LENGTH;
+ // date...
+ String dateStr = extract(docBuf, DATE1, DATE1_END, h2, null);
+ if (dateStr != null) {
+ date = trecSrc.parseDate(dateStr);
+ }
+ // title...
+ title = extract(docBuf, TI, TI_END, h2, null);
+ }
+ docData.clear();
+ docData.setName(name);
+ docData.setDate(date);
+ docData.setTitle(title);
+ docData.setBody(stripTags(docBuf, mark).toString());
+ return docData;
+ }
+
+}
Added: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFR94Parser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFR94Parser.java?rev=1066771&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFR94Parser.java (added)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFR94Parser.java Thu Feb 3 09:50:53 2011
@@ -0,0 +1,66 @@
+package org.apache.lucene.benchmark.byTask.feeds;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Date;
+
+/**
+ * Parser for the FR94 docs in trec disks 4+5 collection format
+ */
+public class TrecFR94Parser extends TrecDocParser {
+
+ private static final String TEXT = "<TEXT>";
+ private static final int TEXT_LENGTH = TEXT.length();
+ private static final String TEXT_END = "</TEXT>";
+
+ private static final String DATE = "<DATE>";
+ private static final String[] DATE_NOISE_PREFIXES = {
+ "DATE:",
+ "date:", //TODO improve date extraction for this format
+ "t.c.",
+ };
+ private static final String DATE_END = "</DATE>";
+
+ //TODO can we also extract title for this format?
+
+ @Override
+ public DocData parse(DocData docData, String name, TrecContentSource trecSrc,
+ StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException {
+ int mark = 0; // that much is skipped
+ // optionally skip some of the text, set date (no title?)
+ Date date = null;
+ int h1 = docBuf.indexOf(TEXT);
+ if (h1>=0) {
+ int h2 = docBuf.indexOf(TEXT_END,h1);
+ mark = h1+TEXT_LENGTH;
+ // date...
+ String dateStr = extract(docBuf, DATE, DATE_END, h2, DATE_NOISE_PREFIXES);
+ if (dateStr != null) {
+ dateStr = stripTags(dateStr,0).toString();
+ date = trecSrc.parseDate(dateStr.trim());
+ }
+ }
+ docData.clear();
+ docData.setName(name);
+ docData.setDate(date);
+ docData.setBody(stripTags(docBuf, mark).toString());
+ return docData;
+ }
+
+}
Added: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFTParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFTParser.java?rev=1066771&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFTParser.java (added)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFTParser.java Thu Feb 3 09:50:53 2011
@@ -0,0 +1,57 @@
+package org.apache.lucene.benchmark.byTask.feeds;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Date;
+
+/**
+ * Parser for the FT docs in trec disks 4+5 collection format
+ */
+public class TrecFTParser extends TrecDocParser {
+
+ private static final String DATE = "<DATE>";
+ private static final String DATE_END = "</DATE>";
+
+ private static final String HEADLINE = "<HEADLINE>";
+ private static final String HEADLINE_END = "</HEADLINE>";
+
+ @Override
+ public DocData parse(DocData docData, String name, TrecContentSource trecSrc,
+ StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException {
+ int mark = 0; // that much is skipped
+
+ // date...
+ Date date = null;
+ String dateStr = extract(docBuf, DATE, DATE_END, -1, null);
+ if (dateStr != null) {
+ date = trecSrc.parseDate(dateStr);
+ }
+
+ // title...
+ String title = extract(docBuf, HEADLINE, HEADLINE_END, -1, null);
+
+ docData.clear();
+ docData.setName(name);
+ docData.setDate(date);
+ docData.setTitle(title);
+ docData.setBody(stripTags(docBuf, mark).toString());
+ return docData;
+ }
+
+}
Added: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecGov2Parser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecGov2Parser.java?rev=1066771&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecGov2Parser.java (added)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecGov2Parser.java Thu Feb 3 09:50:53 2011
@@ -0,0 +1,59 @@
+package org.apache.lucene.benchmark.byTask.feeds;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Date;
+
+/**
+ * Parser for the GOV2 collection format
+ */
+public class TrecGov2Parser extends TrecDocParser {
+
+ private static final String DATE = "Date: ";
+ private static final String DATE_END = TrecContentSource.NEW_LINE;
+
+ private static final String DOCHDR = "<DOCHDR>";
+ private static final String TERMINATING_DOCHDR = "</DOCHDR>";
+ private static final int TERMINATING_DOCHDR_LENGTH = TERMINATING_DOCHDR.length();
+
+ @Override
+ public DocData parse(DocData docData, String name, TrecContentSource trecSrc,
+ StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException {
+ // Set up a (per-thread) reused Reader over the read content, reset it to re-read from docBuf
+ Reader r = trecSrc.getTrecDocReader(docBuf);
+
+ // skip some of the text, optionally set date
+ Date date = null;
+ int h1 = docBuf.indexOf(DOCHDR);
+ if (h1>=0) {
+ int h2 = docBuf.indexOf(TERMINATING_DOCHDR,h1);
+ String dateStr = extract(docBuf, DATE, DATE_END, h2, null);
+ if (dateStr != null) {
+ date = trecSrc.parseDate(dateStr);
+ }
+ r.mark(h2+TERMINATING_DOCHDR_LENGTH);
+ }
+
+ r.reset();
+ HTMLParser htmlParser = trecSrc.getHtmlParser();
+ return htmlParser.parse(docData, name, date, null, r, null);
+ }
+
+}
Added: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecLATimesParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecLATimesParser.java?rev=1066771&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecLATimesParser.java (added)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecLATimesParser.java Thu Feb 3 09:50:53 2011
@@ -0,0 +1,71 @@
+package org.apache.lucene.benchmark.byTask.feeds;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Date;
+
+/**
+ * Parser for the FT docs in trec disks 4+5 collection format
+ */
+public class TrecLATimesParser extends TrecDocParser {
+
+ private static final String DATE = "<DATE>";
+ private static final String DATE_END = "</DATE>";
+ private static final String DATE_NOISE = "day,"; // anything aftre the ','
+
+ private static final String SUBJECT = "<SUBJECT>";
+ private static final String SUBJECT_END = "</SUBJECT>";
+ private static final String HEADLINE = "<HEADLINE>";
+ private static final String HEADLINE_END = "</HEADLINE>";
+
+ @Override
+ public DocData parse(DocData docData, String name, TrecContentSource trecSrc,
+ StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException {
+ int mark = 0; // that much is skipped
+
+ // date...
+ Date date = null;
+ String dateStr = extract(docBuf, DATE, DATE_END, -1, null);
+ if (dateStr != null) {
+ int d2a = dateStr.indexOf(DATE_NOISE);
+ if (d2a > 0) {
+ dateStr = dateStr.substring(0,d2a+3); // we need the "day" part
+ }
+ dateStr = stripTags(dateStr,0).toString();
+ date = trecSrc.parseDate(dateStr.trim());
+ }
+
+ // title... first try with SUBJECT, them with HEADLINE
+ String title = extract(docBuf, SUBJECT, SUBJECT_END, -1, null);
+ if (title==null) {
+ title = extract(docBuf, HEADLINE, HEADLINE_END, -1, null);
+ }
+ if (title!=null) {
+ title = stripTags(title,0).toString().trim();
+ }
+
+ docData.clear();
+ docData.setName(name);
+ docData.setDate(date);
+ docData.setTitle(title);
+ docData.setBody(stripTags(docBuf, mark).toString());
+ return docData;
+ }
+
+}
Added: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecParserByPath.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecParserByPath.java?rev=1066771&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecParserByPath.java (added)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecParserByPath.java Thu Feb 3 09:50:53 2011
@@ -0,0 +1,33 @@
+package org.apache.lucene.benchmark.byTask.feeds;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+/**
+ * Parser for trec docs which selects the parser to apply according
+ * to the source files path, defaulting to {@link TrecGov2Parser}.
+ */
+public class TrecParserByPath extends TrecDocParser {
+
+ @Override
+ public DocData parse(DocData docData, String name, TrecContentSource trecSrc,
+ StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException {
+ return pathType2parser.get(pathType).parse(docData, name, trecSrc, docBuf, pathType);
+ }
+
+}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StringBuilderReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StringBuilderReader.java?rev=1066771&r1=1066770&r2=1066771&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StringBuilderReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StringBuilderReader.java Thu Feb 3 09:50:53 2011
@@ -158,8 +158,10 @@ public class StringBuilderReader extends
synchronized (lock) {
this.sb = sb;
length = sb.length();
+ next = mark = 0;
}
}
+
@Override
public long skip(long ns) throws IOException {
synchronized (lock) {
Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java?rev=1066771&r1=1066770&r2=1066771&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java Thu Feb 3 09:50:53 2011
@@ -18,14 +18,20 @@ package org.apache.lucene.benchmark.byTa
*/
import java.io.BufferedReader;
+import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.text.ParseException;
+import java.util.Arrays;
import java.util.Date;
+import java.util.HashSet;
+import java.util.Properties;
+import org.apache.lucene.benchmark.byTask.feeds.TrecDocParser.ParsePathType;
import org.apache.lucene.benchmark.byTask.utils.Config;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
public class TrecContentSourceTest extends LuceneTestCase {
@@ -329,5 +335,62 @@ public class TrecContentSourceTest exten
// Don't test that NoMoreDataException is thrown, since the forever flag is
// turned on.
}
+
+ /**
+ * Open a trec content source over a directory with files of all trec path types and all
+ * supported formats - bzip, gzip, txt.
+ */
+ public void testTrecFeedDirAllTypes() throws Exception {
+ File dataDir = _TestUtil.getTempDir("trecFeedAllTypes");
+ _TestUtil.unzip(getDataFile("trecdocs.zip"), dataDir);
+ TrecContentSource tcs = new TrecContentSource();
+ Properties props = new Properties();
+ props.setProperty("print.props", "false");
+ props.setProperty("content.source.verbose", "false");
+ props.setProperty("content.source.excludeIteration", "true");
+ props.setProperty("doc.maker.forever", "false");
+ props.setProperty("docs.dir", dataDir.getCanonicalPath().replace('\\','/'));
+ props.setProperty("trec.doc.parser", TrecParserByPath.class.getName());
+ props.setProperty("content.source.forever", "false");
+ tcs.setConfig(new Config(props));
+ tcs.resetInputs();
+ DocData dd = new DocData();
+ int n = 0;
+ boolean gotExpectedException = false;
+ HashSet<ParsePathType> unseenTypes = new HashSet<ParsePathType>(Arrays.asList(ParsePathType.values()));
+ try {
+ while (n<100) { // arbiterary limit to prevent looping forever in case of test failure
+ dd = tcs.getNextDocData(dd);
+ ++n;
+ assertNotNull("doc data "+n+" should not be null!", dd);
+ unseenTypes.remove(tcs.currPathType);
+ switch(tcs.currPathType) {
+ case GOV2:
+ assertDocData(dd, "TEST-000", "TEST-000 title", "TEST-000 text", tcs.parseDate("Sun, 11 Jan 2009 08:00:00 GMT"));
+ break;
+ case FBIS:
+ assertDocData(dd, "TEST-001", "TEST-001 Title", "TEST-001 text", tcs.parseDate("1 January 1991"));
+ break;
+ case FR94:
+ // no title extraction in this source for now
+ assertDocData(dd, "TEST-002", null, "DEPARTMENT OF SOMETHING", tcs.parseDate("February 3, 1994"));
+ break;
+ case FT:
+ assertDocData(dd, "TEST-003", "Test-003 title", "Some pub text", tcs.parseDate("980424"));
+ break;
+ case LATIMES:
+ assertDocData(dd, "TEST-004", "Test-004 Title", "Some paragraph", tcs.parseDate("January 17, 1997, Sunday"));
+ break;
+ default:
+ assertTrue("Should never get here!", false);
+ }
+ }
+ } catch (NoMoreDataException e) {
+ gotExpectedException = true;
+ }
+ assertTrue("Should have gotten NoMoreDataException!", gotExpectedException);
+ assertEquals("Wrong numbre of documents created by osurce!",5,n);
+ assertTrue("Did not see all types!",unseenTypes.isEmpty());
+ }
}
Added: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/trecdocs.zip
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/trecdocs.zip?rev=1066771&view=auto
==============================================================================
Binary file - no diff available.
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1066771&r1=1066770&r2=1066771&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Thu Feb 3 09:50:53 2011
@@ -17,21 +17,14 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import java.io.BufferedOutputStream;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStream;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
-import java.io.OutputStream;
import java.util.Arrays;
-import java.util.Enumeration;
import java.util.List;
import java.util.ArrayList;
import java.util.Random;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipFile;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
@@ -80,51 +73,17 @@ public class TestBackwardsCompatibility
/*
public void testCreateCFS() throws IOException {
String dirName = "testindex.cfs";
- createIndex(random, dirName, true);
- //rmDir(dirName);
+ File indexDir = createIndex(random, dirName, true);
+ //_TestUtil.rmDir(indexDir);
}
public void testCreateNoCFS() throws IOException {
String dirName = "testindex.nocfs";
- createIndex(random, dirName, false);
- //rmDir(dirName);
+ File indexDir = createIndex(random, dirName, false);
+ //_TestUtil.rmDir(indexDir);
}
*/
- /* Unzips zipName --> dirName, removing dirName
- first */
- public void unzip(File zipName, String destDirName) throws IOException {
-
- ZipFile zipFile = new ZipFile(zipName);
-
- Enumeration<? extends ZipEntry> entries = zipFile.entries();
-
- String dirName = fullDir(destDirName);
-
- File fileDir = new File(dirName);
- rmDir(destDirName);
-
- fileDir.mkdir();
-
- while (entries.hasMoreElements()) {
- ZipEntry entry = entries.nextElement();
-
- InputStream in = zipFile.getInputStream(entry);
- OutputStream out = new BufferedOutputStream(new FileOutputStream(new File(fileDir, entry.getName())));
-
- byte[] buffer = new byte[8192];
- int len;
- while((len = in.read(buffer)) >= 0) {
- out.write(buffer, 0, len);
- }
-
- in.close();
- out.close();
- }
-
- zipFile.close();
- }
-
final String[] oldNames = {"19.cfs",
"19.nocfs",
"20.cfs",
@@ -212,9 +171,9 @@ public class TestBackwardsCompatibility
int hasTested29 = 0;
for(int i=0;i<oldNames.length;i++) {
- unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
- String fullPath = fullDir(oldNames[i]);
- Directory dir = newFSDirectory(new File(fullPath));
+ File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
+ _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
+ Directory dir = newFSDirectory(oldIndxeDir);
if (oldNames[i].startsWith("29.")) {
assertCompressedFields29(dir, true);
@@ -234,7 +193,7 @@ public class TestBackwardsCompatibility
}
dir.close();
- rmDir(oldNames[i]);
+ _TestUtil.rmDir(oldIndxeDir);
}
assertEquals("test for compressed field should have run 4 times", 4, hasTested29);
@@ -242,9 +201,9 @@ public class TestBackwardsCompatibility
public void testAddOldIndexes() throws IOException {
for (String name : oldNames) {
- unzip(getDataFile("index." + name + ".zip"), name);
- String fullPath = fullDir(name);
- Directory dir = newFSDirectory(new File(fullPath));
+ File oldIndxeDir = _TestUtil.getTempDir(name);
+ _TestUtil.unzip(getDataFile("index." + name + ".zip"), oldIndxeDir);
+ Directory dir = newFSDirectory(oldIndxeDir);
Directory targetDir = newDirectory();
IndexWriter w = new IndexWriter(targetDir, newIndexWriterConfig(
@@ -256,15 +215,15 @@ public class TestBackwardsCompatibility
dir.close();
targetDir.close();
- rmDir(name);
+ _TestUtil.rmDir(oldIndxeDir);
}
}
public void testAddOldIndexesReader() throws IOException {
for (String name : oldNames) {
- unzip(getDataFile("index." + name + ".zip"), name);
- String fullPath = fullDir(name);
- Directory dir = newFSDirectory(new File(fullPath));
+ File oldIndxeDir = _TestUtil.getTempDir(name);
+ _TestUtil.unzip(getDataFile("index." + name + ".zip"), oldIndxeDir);
+ Directory dir = newFSDirectory(oldIndxeDir);
IndexReader reader = IndexReader.open(dir);
Directory targetDir = newDirectory();
@@ -278,23 +237,25 @@ public class TestBackwardsCompatibility
dir.close();
targetDir.close();
- rmDir(name);
+ _TestUtil.rmDir(oldIndxeDir);
}
}
public void testSearchOldIndex() throws IOException {
for(int i=0;i<oldNames.length;i++) {
- unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
- searchIndex(oldNames[i], oldNames[i]);
- rmDir(oldNames[i]);
+ File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
+ _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
+ searchIndex(oldIndxeDir, oldNames[i]);
+ _TestUtil.rmDir(oldIndxeDir);
}
}
public void testIndexOldIndexNoAdds() throws IOException {
for(int i=0;i<oldNames.length;i++) {
- unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
- changeIndexNoAdds(random, oldNames[i]);
- rmDir(oldNames[i]);
+ File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
+ _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
+ changeIndexNoAdds(random, oldIndxeDir);
+ _TestUtil.rmDir(oldIndxeDir);
}
}
@@ -303,9 +264,10 @@ public class TestBackwardsCompatibility
if (VERBOSE) {
System.out.println("TEST: oldName=" + oldNames[i]);
}
- unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
- changeIndexWithAdds(random, oldNames[i]);
- rmDir(oldNames[i]);
+ File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
+ _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
+ changeIndexWithAdds(random, oldIndxeDir, oldNames[i]);
+ _TestUtil.rmDir(oldIndxeDir);
}
}
@@ -318,13 +280,11 @@ public class TestBackwardsCompatibility
}
}
- public void searchIndex(String dirName, String oldName) throws IOException {
+ public void searchIndex(File indexDir, String oldName) throws IOException {
//QueryParser parser = new QueryParser("contents", new WhitespaceAnalyzer(TEST_VERSION_CURRENT));
//Query query = parser.parse("handle:1");
- dirName = fullDir(dirName);
-
- Directory dir = newFSDirectory(new File(dirName));
+ Directory dir = newFSDirectory(indexDir);
IndexSearcher searcher = new IndexSearcher(dir, true);
IndexReader reader = searcher.getIndexReader();
@@ -359,7 +319,7 @@ public class TestBackwardsCompatibility
}
TermFreqVector tfv = reader.getTermFreqVector(i, "utf8");
- assertNotNull("docID=" + i + " index=" + dirName, tfv);
+ assertNotNull("docID=" + i + " index=" + indexDir.getName(), tfv);
assertTrue(tfv instanceof TermPositionVector);
}
} else
@@ -401,11 +361,8 @@ public class TestBackwardsCompatibility
/* Open pre-lockless index, add docs, do a delete &
* setNorm, and search */
- public void changeIndexWithAdds(Random random, String dirName) throws IOException {
- String origDirName = dirName;
- dirName = fullDir(dirName);
-
- Directory dir = newFSDirectory(new File(dirName));
+ public void changeIndexWithAdds(Random random, File oldIndexDir, String origOldName) throws IOException {
+ Directory dir = newFSDirectory(oldIndexDir);
// open writer
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND));
writer.setInfoStream(VERBOSE ? System.out : null);
@@ -416,7 +373,7 @@ public class TestBackwardsCompatibility
// make sure writer sees right total -- writer seems not to know about deletes in .del?
final int expected;
- if (compare(origDirName, "24") < 0) {
+ if (compare(origOldName, "24") < 0) {
expected = 44;
} else {
expected = 45;
@@ -470,11 +427,9 @@ public class TestBackwardsCompatibility
/* Open pre-lockless index, add docs, do a delete &
* setNorm, and search */
- public void changeIndexNoAdds(Random random, String dirName) throws IOException {
+ public void changeIndexNoAdds(Random random, File oldIndexDir) throws IOException {
- dirName = fullDir(dirName);
-
- Directory dir = newFSDirectory(new File(dirName));
+ Directory dir = newFSDirectory(oldIndexDir);
// make sure searching sees right # hits
IndexSearcher searcher = new IndexSearcher(dir, true);
@@ -518,13 +473,11 @@ public class TestBackwardsCompatibility
dir.close();
}
- public void createIndex(Random random, String dirName, boolean doCFS) throws IOException {
-
- rmDir(dirName);
+ public File createIndex(Random random, String dirName, boolean doCFS) throws IOException {
- dirName = fullDir(dirName);
-
- Directory dir = newFSDirectory(new File(dirName));
+ File indexDir = _TestUtil.getTempDir(dirName);
+ _TestUtil.rmDir(indexDir);
+ Directory dir = newFSDirectory(indexDir);
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setMaxBufferedDocs(10);
((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(doCFS);
if (doCFS) {
@@ -555,17 +508,20 @@ public class TestBackwardsCompatibility
reader.setNorm(21, "content", (float) 1.5);
reader.close();
dir.close();
+
+ return indexDir;
}
/* Verifies that the expected file names were produced */
public void testExactFileNames() throws IOException {
- String outputDir = "lucene.backwardscompat0.index";
- rmDir(outputDir);
+ String outputDirName = "lucene.backwardscompat0.index";
+ File outputDir = _TestUtil.getTempDir(outputDirName);
+ _TestUtil.rmDir(outputDir);
try {
- Directory dir = newFSDirectory(new File(fullDir(outputDir)));
+ Directory dir = newFSDirectory(outputDir);
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setMaxBufferedDocs(-1).setRAMBufferSizeMB(16.0);
((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(true);
@@ -622,7 +578,7 @@ public class TestBackwardsCompatibility
}
dir.close();
} finally {
- rmDir(outputDir);
+ _TestUtil.rmDir(outputDir);
}
}
@@ -672,23 +628,6 @@ public class TestBackwardsCompatibility
writer.addDocument(doc);
}
- private void rmDir(String dir) throws IOException {
- File fileDir = new File(fullDir(dir));
- if (fileDir.exists()) {
- File[] files = fileDir.listFiles();
- if (files != null) {
- for (int i = 0; i < files.length; i++) {
- files[i].delete();
- }
- }
- fileDir.delete();
- }
- }
-
- public static String fullDir(String dirName) throws IOException {
- return new File(TEMP_DIR, dirName).getCanonicalPath();
- }
-
static final String TEXT_TO_COMPRESS = "this is a compressed field and should appear in 3.0 as an uncompressed field after merge";
// FieldSelectorResult.SIZE returns compressed size for compressed fields, which are internally handled as binary;
// do it in the same way like FieldsWriter, do not use CompressionTools.compressString() for compressed fields:
@@ -712,9 +651,9 @@ public class TestBackwardsCompatibility
// only test indexes >= 3.0
if (oldNames[i].compareTo("30.") < 0) continue;
- unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
- String fullPath = fullDir(oldNames[i]);
- Directory dir = newFSDirectory(new File(fullPath));
+ File oldIndexDir = _TestUtil.getTempDir(oldNames[i]);
+ _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndexDir);
+ Directory dir = newFSDirectory(oldIndexDir);
IndexSearcher searcher = new IndexSearcher(dir, true);
for (int id=10; id<15; id++) {
@@ -749,7 +688,7 @@ public class TestBackwardsCompatibility
searcher.close();
dir.close();
- rmDir(oldNames[i]);
+ _TestUtil.rmDir(oldIndexDir);
}
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/_TestUtil.java?rev=1066771&r1=1066770&r2=1066771&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/_TestUtil.java Thu Feb 3 09:50:53 2011
@@ -17,13 +17,20 @@ package org.apache.lucene.util;
* limitations under the License.
*/
+import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.io.PrintStream;
+import java.util.Enumeration;
import java.util.Random;
import java.util.Map;
import java.util.HashMap;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
import org.junit.Assert;
@@ -59,6 +66,49 @@ public class _TestUtil {
}
}
+ /**
+ * Convenience method: Unzip zipName + ".zip" under destDir, removing destDir first
+ */
+ public static void unzip(File zipName, File destDir) throws IOException {
+
+ ZipFile zipFile = new ZipFile(zipName);
+
+ Enumeration<? extends ZipEntry> entries = zipFile.entries();
+
+ rmDir(destDir);
+
+ destDir.mkdir();
+
+ while (entries.hasMoreElements()) {
+ ZipEntry entry = entries.nextElement();
+
+ InputStream in = zipFile.getInputStream(entry);
+ File targetFile = new File(destDir, entry.getName());
+ if (entry.isDirectory()) {
+ // allow unzipping with directory structure
+ targetFile.mkdirs();
+ } else {
+ if (targetFile.getParentFile()!=null) {
+ // be on the safe side: do not rely on that directories are always extracted
+ // before their children (although this makes sense, but is it guaranteed?)
+ targetFile.getParentFile().mkdirs();
+ }
+ OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile));
+
+ byte[] buffer = new byte[8192];
+ int len;
+ while((len = in.read(buffer)) >= 0) {
+ out.write(buffer, 0, len);
+ }
+
+ in.close();
+ out.close();
+ }
+ }
+
+ zipFile.close();
+ }
+
public static void syncConcurrentMerges(IndexWriter writer) {
syncConcurrentMerges(writer.getConfig().getMergeScheduler());
}