You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by do...@apache.org on 2011/02/05 01:35:10 UTC
svn commit: r1067359 - in /lucene/dev/trunk: ./ lucene/
lucene/src/test/org/apache/lucene/index/
lucene/src/test/org/apache/lucene/util/ modules/benchmark/
modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/
modules/benchmark/src/java/...
Author: doronc
Date: Sat Feb 5 00:35:09 2011
New Revision: 1067359
URL: http://svn.apache.org/viewvc?rev=1067359&view=rev
Log:
LUCENE-1540: Improvements to contrib.benchmark for TREC collections - port/merge from 3x.
Added:
lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocParser.java
- copied unchanged from r1066771, lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocParser.java
lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFBISParser.java
- copied unchanged from r1066771, lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFBISParser.java
lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFR94Parser.java
- copied unchanged from r1066771, lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFR94Parser.java
lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFTParser.java
- copied unchanged from r1066771, lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFTParser.java
lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecGov2Parser.java
- copied unchanged from r1066771, lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecGov2Parser.java
lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecLATimesParser.java
- copied unchanged from r1066771, lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecLATimesParser.java
lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecParserByPath.java
- copied unchanged from r1066771, lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecParserByPath.java
lucene/dev/trunk/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/trecdocs.zip
- copied unchanged from r1066771, lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/trecdocs.zip
Modified:
lucene/dev/trunk/ (props changed)
lucene/dev/trunk/lucene/ (props changed)
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java
lucene/dev/trunk/modules/benchmark/CHANGES.txt
lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java
lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java
lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/HTMLParser.java
lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java
lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StringBuilderReader.java
lucene/dev/trunk/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1067359&r1=1067358&r2=1067359&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Sat Feb 5 00:35:09 2011
@@ -17,20 +17,13 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.io.PrintStream;
import java.util.Arrays;
-import java.util.Enumeration;
import java.util.List;
import java.util.Random;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipFile;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -78,39 +71,6 @@ public class TestBackwardsCompatibility
}
*/
- /* Unzips zipName --> dirName, removing dirName
- first */
- public void unzip(File zipName, String destDirName) throws IOException {
-
- ZipFile zipFile = new ZipFile(zipName);
-
- Enumeration<? extends ZipEntry> entries = zipFile.entries();
-
- String dirName = fullDir(destDirName);
-
- File fileDir = new File(dirName);
- rmDir(destDirName);
-
- fileDir.mkdir();
-
- while (entries.hasMoreElements()) {
- ZipEntry entry = entries.nextElement();
-
- InputStream in = zipFile.getInputStream(entry);
- OutputStream out = new BufferedOutputStream(new FileOutputStream(new File(fileDir, entry.getName())));
-
- byte[] buffer = new byte[8192];
- int len;
- while((len = in.read(buffer)) >= 0) {
- out.write(buffer, 0, len);
- }
-
- in.close();
- out.close();
- }
-
- zipFile.close();
- }
/*
public void testCreateCFS() throws IOException {
String dirName = "testindex.cfs";
@@ -153,10 +113,9 @@ public class TestBackwardsCompatibility
if (VERBOSE) {
System.out.println("TEST: index " + unsupportedNames[i]);
}
- unzip(getDataFile("unsupported." + unsupportedNames[i] + ".zip"), unsupportedNames[i]);
-
- String fullPath = fullDir(unsupportedNames[i]);
- Directory dir = newFSDirectory(new File(fullPath));
+ File oldIndxeDir = _TestUtil.getTempDir(unsupportedNames[i]);
+ _TestUtil.unzip(getDataFile("unsupported." + unsupportedNames[i] + ".zip"), oldIndxeDir);
+ Directory dir = newFSDirectory(oldIndxeDir);
IndexReader reader = null;
IndexWriter writer = null;
@@ -200,7 +159,7 @@ public class TestBackwardsCompatibility
assertTrue(bos.toString().contains(IndexFormatTooOldException.class.getName()));
dir.close();
- rmDir(unsupportedNames[i]);
+ _TestUtil.rmDir(oldIndxeDir);
}
}
@@ -209,10 +168,9 @@ public class TestBackwardsCompatibility
if (VERBOSE) {
System.out.println("\nTEST: index=" + oldNames[i]);
}
- unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
-
- String fullPath = fullDir(oldNames[i]);
- Directory dir = newFSDirectory(new File(fullPath));
+ File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
+ _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
+ Directory dir = newFSDirectory(oldIndxeDir);
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer()));
@@ -223,15 +181,15 @@ public class TestBackwardsCompatibility
_TestUtil.checkIndex(dir);
dir.close();
- rmDir(oldNames[i]);
+ _TestUtil.rmDir(oldIndxeDir);
}
}
public void testAddOldIndexes() throws IOException {
for (String name : oldNames) {
- unzip(getDataFile("index." + name + ".zip"), name);
- String fullPath = fullDir(name);
- Directory dir = newFSDirectory(new File(fullPath));
+ File oldIndxeDir = _TestUtil.getTempDir(name);
+ _TestUtil.unzip(getDataFile("index." + name + ".zip"), oldIndxeDir);
+ Directory dir = newFSDirectory(oldIndxeDir);
Directory targetDir = newDirectory();
IndexWriter w = new IndexWriter(targetDir, newIndexWriterConfig(
@@ -243,15 +201,15 @@ public class TestBackwardsCompatibility
dir.close();
targetDir.close();
- rmDir(name);
+ _TestUtil.rmDir(oldIndxeDir);
}
}
public void testAddOldIndexesReader() throws IOException {
for (String name : oldNames) {
- unzip(getDataFile("index." + name + ".zip"), name);
- String fullPath = fullDir(name);
- Directory dir = newFSDirectory(new File(fullPath));
+ File oldIndxeDir = _TestUtil.getTempDir(name);
+ _TestUtil.unzip(getDataFile("index." + name + ".zip"), oldIndxeDir);
+ Directory dir = newFSDirectory(oldIndxeDir);
IndexReader reader = IndexReader.open(dir);
Directory targetDir = newDirectory();
@@ -265,23 +223,25 @@ public class TestBackwardsCompatibility
dir.close();
targetDir.close();
- rmDir(name);
+ _TestUtil.rmDir(oldIndxeDir);
}
}
public void testSearchOldIndex() throws IOException {
for(int i=0;i<oldNames.length;i++) {
- unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
- searchIndex(oldNames[i], oldNames[i]);
- rmDir(oldNames[i]);
+ File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
+ _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
+ searchIndex(oldIndxeDir, oldNames[i]);
+ _TestUtil.rmDir(oldIndxeDir);
}
}
public void testIndexOldIndexNoAdds() throws IOException {
for(int i=0;i<oldNames.length;i++) {
- unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
- changeIndexNoAdds(random, oldNames[i]);
- rmDir(oldNames[i]);
+ File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
+ _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
+ changeIndexNoAdds(random, oldIndxeDir);
+ _TestUtil.rmDir(oldIndxeDir);
}
}
@@ -290,9 +250,10 @@ public class TestBackwardsCompatibility
if (VERBOSE) {
System.out.println("TEST: oldName=" + oldNames[i]);
}
- unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
- changeIndexWithAdds(random, oldNames[i]);
- rmDir(oldNames[i]);
+ File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
+ _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
+ changeIndexWithAdds(random, oldIndxeDir, oldNames[i]);
+ _TestUtil.rmDir(oldIndxeDir);
}
}
@@ -305,13 +266,11 @@ public class TestBackwardsCompatibility
}
}
- public void searchIndex(String dirName, String oldName) throws IOException {
+ public void searchIndex(File indexDir, String oldName) throws IOException {
//QueryParser parser = new QueryParser("contents", new MockAnalyzer());
//Query query = parser.parse("handle:1");
- dirName = fullDir(dirName);
-
- Directory dir = newFSDirectory(new File(dirName));
+ Directory dir = newFSDirectory(indexDir);
IndexSearcher searcher = new IndexSearcher(dir, true);
IndexReader reader = searcher.getIndexReader();
@@ -343,7 +302,7 @@ public class TestBackwardsCompatibility
}
TermFreqVector tfv = reader.getTermFreqVector(i, "utf8");
- assertNotNull("docID=" + i + " index=" + dirName, tfv);
+ assertNotNull("docID=" + i + " index=" + indexDir.getName(), tfv);
assertTrue(tfv instanceof TermPositionVector);
} else
// Only ID 7 is deleted
@@ -376,11 +335,9 @@ public class TestBackwardsCompatibility
return v0 - v1;
}
- public void changeIndexWithAdds(Random random, String dirName) throws IOException {
- String origDirName = dirName;
- dirName = fullDir(dirName);
+ public void changeIndexWithAdds(Random random, File oldIndexDir, String origOldName) throws IOException {
- Directory dir = newFSDirectory(new File(dirName));
+ Directory dir = newFSDirectory(oldIndexDir);
// open writer
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND));
writer.setInfoStream(VERBOSE ? System.out : null);
@@ -391,7 +348,7 @@ public class TestBackwardsCompatibility
// make sure writer sees right total -- writer seems not to know about deletes in .del?
final int expected;
- if (compare(origDirName, "24") < 0) {
+ if (compare(origOldName, "24") < 0) {
expected = 44;
} else {
expected = 45;
@@ -442,11 +399,9 @@ public class TestBackwardsCompatibility
dir.close();
}
- public void changeIndexNoAdds(Random random, String dirName) throws IOException {
-
- dirName = fullDir(dirName);
+ public void changeIndexNoAdds(Random random, File oldIndexDir) throws IOException {
- Directory dir = newFSDirectory(new File(dirName));
+ Directory dir = newFSDirectory(oldIndexDir);
// make sure searching sees right # hits
IndexSearcher searcher = new IndexSearcher(dir, true);
@@ -489,13 +444,12 @@ public class TestBackwardsCompatibility
dir.close();
}
- public void createIndex(Random random, String dirName, boolean doCFS) throws IOException {
-
- rmDir(dirName);
-
- dirName = fullDir(dirName);
+ public File createIndex(Random random, String dirName, boolean doCFS) throws IOException {
- Directory dir = newFSDirectory(new File(dirName));
+ File indexDir = _TestUtil.getTempDir(dirName);
+ _TestUtil.rmDir(indexDir);
+ Directory dir = newFSDirectory(indexDir);
+
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(10);
((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(doCFS);
IndexWriter writer = new IndexWriter(dir, conf);
@@ -522,17 +476,21 @@ public class TestBackwardsCompatibility
// Set one norm so we get a .s0 file:
reader.setNorm(21, "content", conf.getSimilarityProvider().get("content").encodeNormValue(1.5f));
reader.close();
+ dir.close();
+
+ return indexDir;
}
/* Verifies that the expected file names were produced */
public void testExactFileNames() throws IOException {
- String outputDir = "lucene.backwardscompat0.index";
- rmDir(outputDir);
+ String outputDirName = "lucene.backwardscompat0.index";
+ File outputDir = _TestUtil.getTempDir(outputDirName);
+ _TestUtil.rmDir(outputDir);
try {
- Directory dir = newFSDirectory(new File(fullDir(outputDir)));
+ Directory dir = newFSDirectory(outputDir);
LogMergePolicy mergePolicy = newLogMergePolicy(true, 10);
mergePolicy.setNoCFSRatio(1); // This test expects all of its segments to be in CFS
@@ -595,7 +553,7 @@ public class TestBackwardsCompatibility
}
dir.close();
} finally {
- rmDir(outputDir);
+ _TestUtil.rmDir(outputDir);
}
}
@@ -636,23 +594,6 @@ public class TestBackwardsCompatibility
writer.addDocument(doc);
}
- private void rmDir(String dir) throws IOException {
- File fileDir = new File(fullDir(dir));
- if (fileDir.exists()) {
- File[] files = fileDir.listFiles();
- if (files != null) {
- for (int i = 0; i < files.length; i++) {
- files[i].delete();
- }
- }
- fileDir.delete();
- }
- }
-
- public static String fullDir(String dirName) throws IOException {
- return new File(TEMP_DIR, dirName).getCanonicalPath();
- }
-
private int countDocs(DocsEnum docs) throws IOException {
int count = 0;
while((docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
@@ -664,9 +605,9 @@ public class TestBackwardsCompatibility
// flex: test basics of TermsEnum api on non-flex index
public void testNextIntoWrongField() throws Exception {
for(int i=0;i<oldNames.length;i++) {
- unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
- String fullPath = fullDir(oldNames[i]);
- Directory dir = newFSDirectory(new File(fullPath));
+ File oldIndexDir = _TestUtil.getTempDir(oldNames[i]);
+ _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndexDir);
+ Directory dir = newFSDirectory(oldIndexDir);
IndexReader r = IndexReader.open(dir);
TermsEnum terms = MultiFields.getFields(r).terms("content").iterator();
BytesRef t = terms.next();
@@ -703,16 +644,16 @@ public class TestBackwardsCompatibility
r.close();
dir.close();
- rmDir(oldNames[i]);
+ _TestUtil.rmDir(oldIndexDir);
}
}
public void testNumericFields() throws Exception {
for(int i=0;i<oldNames.length;i++) {
- unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
- String fullPath = fullDir(oldNames[i]);
- Directory dir = newFSDirectory(new File(fullPath));
+ File oldIndexDir = _TestUtil.getTempDir(oldNames[i]);
+ _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndexDir);
+ Directory dir = newFSDirectory(oldIndexDir);
IndexSearcher searcher = new IndexSearcher(dir, true);
for (int id=10; id<15; id++) {
@@ -747,7 +688,7 @@ public class TestBackwardsCompatibility
searcher.close();
dir.close();
- rmDir(oldNames[i]);
+ _TestUtil.rmDir(oldIndexDir);
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java?rev=1067359&r1=1067358&r2=1067359&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java Sat Feb 5 00:35:09 2011
@@ -17,13 +17,20 @@ package org.apache.lucene.util;
* limitations under the License.
*/
+import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.io.PrintStream;
+import java.util.Enumeration;
import java.util.Random;
import java.util.Map;
import java.util.HashMap;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
import org.junit.Assert;
@@ -61,6 +68,49 @@ public class _TestUtil {
}
}
+ /**
+ * Convenience method: Unzip zipName + ".zip" under destDir, removing destDir first
+ */
+ public static void unzip(File zipName, File destDir) throws IOException {
+
+ ZipFile zipFile = new ZipFile(zipName);
+
+ Enumeration<? extends ZipEntry> entries = zipFile.entries();
+
+ rmDir(destDir);
+
+ destDir.mkdir();
+
+ while (entries.hasMoreElements()) {
+ ZipEntry entry = entries.nextElement();
+
+ InputStream in = zipFile.getInputStream(entry);
+ File targetFile = new File(destDir, entry.getName());
+ if (entry.isDirectory()) {
+ // allow unzipping with directory structure
+ targetFile.mkdirs();
+ } else {
+ if (targetFile.getParentFile()!=null) {
+ // be on the safe side: do not rely on that directories are always extracted
+ // before their children (although this makes sense, but is it guaranteed?)
+ targetFile.getParentFile().mkdirs();
+ }
+ OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile));
+
+ byte[] buffer = new byte[8192];
+ int len;
+ while((len = in.read(buffer)) >= 0) {
+ out.write(buffer, 0, len);
+ }
+
+ in.close();
+ out.close();
+ }
+ }
+
+ zipFile.close();
+ }
+
public static void syncConcurrentMerges(IndexWriter writer) {
syncConcurrentMerges(writer.getConfig().getMergeScheduler());
}
Modified: lucene/dev/trunk/modules/benchmark/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/benchmark/CHANGES.txt?rev=1067359&r1=1067358&r2=1067359&view=diff
==============================================================================
--- lucene/dev/trunk/modules/benchmark/CHANGES.txt (original)
+++ lucene/dev/trunk/modules/benchmark/CHANGES.txt Sat Feb 5 00:35:09 2011
@@ -2,6 +2,12 @@ Lucene Benchmark Contrib Change Log
The Benchmark contrib package contains code for benchmarking Lucene in a variety of ways.
+02/05/2011
+ LUCENE-1540: Improvements to contrib.benchmark for TREC collections.
+ ContentSource can now process plain text files, gzip files, and bzip2 files.
+ TREC doc parsing now handles the TREC gov2 collection and TREC disks 4&5-CR
+ collection (both used by many TREC tasks). (Shai Erera, Doron Cohen)
+
01/26/2011
LUCENE-929: ExtractReuters first extracts to a tmp dir and then renames. That
way, if a previous extract attempt failed, "ant extract-reuters" will still
Modified: lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java?rev=1067359&r1=1067358&r2=1067359&view=diff
==============================================================================
--- lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java (original)
+++ lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java Sat Feb 5 00:35:09 2011
@@ -56,11 +56,14 @@ import org.apache.lucene.benchmark.byTas
public abstract class ContentSource {
private static final int BZIP = 0;
- private static final int OTHER = 1;
+ private static final int GZIP = 1;
+ private static final int OTHER = 2;
private static final Map<String,Integer> extensionToType = new HashMap<String,Integer>();
static {
extensionToType.put(".bz2", Integer.valueOf(BZIP));
extensionToType.put(".bzip", Integer.valueOf(BZIP));
+ extensionToType.put(".gz", Integer.valueOf(GZIP));
+ extensionToType.put(".gzip", Integer.valueOf(GZIP));
}
protected static final int BUFFER_SIZE = 1 << 16; // 64K
@@ -78,11 +81,13 @@ public abstract class ContentSource {
private CompressorStreamFactory csFactory = new CompressorStreamFactory();
+ /** update count of bytes generated by this source */
protected final synchronized void addBytes(long numBytes) {
bytesCount += numBytes;
totalBytesCount += numBytes;
}
+ /** update count of documents generated by this source */
protected final synchronized void addDoc() {
++docsCount;
++totalDocsCount;
@@ -130,21 +135,25 @@ public abstract class ContentSource {
type = typeInt.intValue();
}
}
- switch (type) {
- case BZIP:
- try {
+
+ try {
+ switch (type) {
+ case BZIP:
// According to BZip2CompressorInputStream's code, it reads the first
// two file header chars ('B' and 'Z'). It is important to wrap the
// underlying input stream with a buffered one since
// Bzip2CompressorInputStream uses the read() method exclusively.
is = csFactory.createCompressorInputStream("bzip2", is);
- } catch (CompressorException e) {
- IOException ioe = new IOException(e.getMessage());
- ioe.initCause(e);
- throw ioe;
- }
- break;
- default: // Do nothing, stay with FileInputStream
+ break;
+ case GZIP:
+ is = csFactory.createCompressorInputStream("gz", is);
+ break;
+ default: // Do nothing, stay with FileInputStream
+ }
+ } catch (CompressorException e) {
+ IOException ioe = new IOException(e.getMessage());
+ ioe.initCause(e);
+ throw ioe;
}
return is;
Modified: lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java?rev=1067359&r1=1067358&r2=1067359&view=diff
==============================================================================
--- lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java (original)
+++ lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java Sat Feb 5 00:35:09 2011
@@ -29,11 +29,14 @@ import java.util.Properties;
*/
public class DemoHTMLParser implements org.apache.lucene.benchmark.byTask.feeds.HTMLParser {
- public DocData parse(DocData docData, String name, Date date, Reader reader, DateFormat dateFormat) throws IOException, InterruptedException {
+ public DocData parse(DocData docData, String name, Date date, String title, Reader reader, DateFormat dateFormat) throws IOException, InterruptedException {
org.apache.lucene.demo.html.HTMLParser p = new org.apache.lucene.demo.html.HTMLParser(reader);
// title
- String title = p.getTitle();
+ if (title==null) {
+ title = p.getTitle();
+ }
+
// properties
Properties props = p.getMetaTags();
// body
Modified: lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/HTMLParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/HTMLParser.java?rev=1067359&r1=1067358&r2=1067359&view=diff
==============================================================================
--- lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/HTMLParser.java (original)
+++ lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/HTMLParser.java Sat Feb 5 00:35:09 2011
@@ -29,16 +29,18 @@ public interface HTMLParser {
/**
* Parse the input Reader and return DocData.
- * A provided name or date is used for the result, otherwise an attempt is
- * made to set them from the parsed data.
- * @param dateFormat date formatter to use for extracting the date.
- * @param name name of the result doc data. If null, attempt to set by parsed data.
+ * The provided name,title,date are used for the result, unless when they're null,
+ * in which case an attempt is made to set them from the parsed data.
+ * @param docData result reused
+ * @param name name of the result doc data.
* @param date date of the result doc data. If null, attempt to set by parsed data.
- * @param reader of html text to parse.
+ * @param title title of the result doc data. If null, attempt to set by parsed data.
+ * @param reader reader of html text to parse.
+ * @param dateFormat date formatter to use for extracting the date.
* @return Parsed doc data.
* @throws IOException
* @throws InterruptedException
*/
- public DocData parse(DocData docData, String name, Date date, Reader reader, DateFormat dateFormat) throws IOException, InterruptedException;
+ public DocData parse(DocData docData, String name, Date date, String title, Reader reader, DateFormat dateFormat) throws IOException, InterruptedException;
}
Modified: lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java?rev=1067359&r1=1067358&r2=1067359&view=diff
==============================================================================
--- lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java (original)
+++ lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java Sat Feb 5 00:35:09 2011
@@ -19,8 +19,8 @@ package org.apache.lucene.benchmark.byTa
import java.io.BufferedReader;
import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.text.DateFormat;
@@ -29,8 +29,8 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Locale;
-import java.util.zip.GZIPInputStream;
+import org.apache.lucene.benchmark.byTask.feeds.TrecDocParser.ParsePathType;
import org.apache.lucene.benchmark.byTask.utils.Config;
import org.apache.lucene.benchmark.byTask.utils.StringBuilderReader;
import org.apache.lucene.util.ThreadInterruptedException;
@@ -46,8 +46,10 @@ import org.apache.lucene.util.ThreadInte
* <li><b>docs.dir</b> - specifies the directory where the TREC files reside.
* Can be set to a relative path if "work.dir" is also specified
* (<b>default=trec</b>).
+ * <li><b>trec.doc.parser</b> - specifies the {@link TrecDocParser} class to use for
+ * parsing the TREC documents content (<b>default=TrecGov2Parser</b>).
* <li><b>html.parser</b> - specifies the {@link HTMLParser} class to use for
- * parsing the TREC documents content (<b>default=DemoHTMLParser</b>).
+ * parsing the HTML parts of the TREC documents content (<b>default=DemoHTMLParser</b>).
* <li><b>content.source.encoding</b> - if not specified, ISO-8859-1 is used.
* <li><b>content.source.excludeIteration</b> - if true, do not append iteration number to docname
* </ul>
@@ -59,22 +61,24 @@ public class TrecContentSource extends C
ParsePosition pos;
}
- private static final String DATE = "Date: ";
- private static final String DOCHDR = "<DOCHDR>";
- private static final String TERMINATING_DOCHDR = "</DOCHDR>";
- private static final String DOCNO = "<DOCNO>";
- private static final String TERMINATING_DOCNO = "</DOCNO>";
- private static final String DOC = "<DOC>";
- private static final String TERMINATING_DOC = "</DOC>";
+ public static final String DOCNO = "<DOCNO>";
+ public static final String TERMINATING_DOCNO = "</DOCNO>";
+ public static final String DOC = "<DOC>";
+ public static final String TERMINATING_DOC = "</DOC>";
- private static final String NEW_LINE = System.getProperty("line.separator");
+ /** separator between lines in the byffer */
+ public static final String NEW_LINE = System.getProperty("line.separator");
private static final String DATE_FORMATS [] = {
- "EEE, dd MMM yyyy kk:mm:ss z", // Tue, 09 Dec 2003 22:39:08 GMT
- "EEE MMM dd kk:mm:ss yyyy z", // Tue Dec 09 16:45:08 2003 EST
- "EEE, dd-MMM-':'y kk:mm:ss z", // Tue, 09 Dec 2003 22:39:08 GMT
- "EEE, dd-MMM-yyy kk:mm:ss z", // Tue, 09 Dec 2003 22:39:08 GMT
- "EEE MMM dd kk:mm:ss yyyy", // Tue Dec 09 16:45:08 2003
+ "EEE, dd MMM yyyy kk:mm:ss z", // Tue, 09 Dec 2003 22:39:08 GMT
+ "EEE MMM dd kk:mm:ss yyyy z", // Tue Dec 09 16:45:08 2003 EST
+ "EEE, dd-MMM-':'y kk:mm:ss z", // Tue, 09 Dec 2003 22:39:08 GMT
+ "EEE, dd-MMM-yyy kk:mm:ss z", // Tue, 09 Dec 2003 22:39:08 GMT
+ "EEE MMM dd kk:mm:ss yyyy", // Tue Dec 09 16:45:08 2003
+ "dd MMM yyyy", // 1 March 1994
+ "MMM dd, yyyy", // February 3, 1994
+ "yyMMdd", // 910513
+ "hhmm z.z.z. MMM dd, yyyy", // 0901 u.t.c. April 28, 1994
};
private ThreadLocal<DateFormatInfo> dateFormats = new ThreadLocal<DateFormatInfo>();
@@ -83,7 +87,7 @@ public class TrecContentSource extends C
private File dataDir = null;
private ArrayList<File> inputFiles = new ArrayList<File>();
private int nextFile = 0;
- private int rawDocSize;
+ private int rawDocSize = 0;
// Use to synchronize threads on reading from the TREC documents.
private Object lock = new Object();
@@ -92,7 +96,10 @@ public class TrecContentSource extends C
BufferedReader reader;
int iteration = 0;
HTMLParser htmlParser;
+
private boolean excludeDocnameIteration;
+ private TrecDocParser trecDocParser = new TrecGov2Parser(); // default
+ ParsePathType currPathType; // not private for tests
private DateFormatInfo getDateFormatInfo() {
DateFormatInfo dfi = dateFormats.get();
@@ -118,7 +125,7 @@ public class TrecContentSource extends C
return sb;
}
- private Reader getTrecDocReader(StringBuilder docBuffer) {
+ Reader getTrecDocReader(StringBuilder docBuffer) {
StringBuilderReader r = trecDocReader.get();
if (r == null) {
r = new StringBuilderReader(docBuffer);
@@ -129,10 +136,21 @@ public class TrecContentSource extends C
return r;
}
- // read until finding a line that starts with the specified prefix, or a terminating tag has been found.
- private void read(StringBuilder buf, String prefix, boolean collectMatchLine,
- boolean collectAll, String terminatingTag)
- throws IOException, NoMoreDataException {
+ HTMLParser getHtmlParser() {
+ return htmlParser;
+ }
+
+ /**
+ * Read until a line starting with the specified <code>lineStart</code>.
+ * @param buf buffer for collecting the data if so specified/
+ * @param lineStart line start to look for, must not be null.
+ * @param collectMatchLine whether to collect the matching line into <code>buffer</code>.
+ * @param collectAll whether to collect all lines into <code>buffer</code>.
+ * @throws IOException
+ * @throws NoMoreDataException
+ */
+ private void read(StringBuilder buf, String lineStart,
+ boolean collectMatchLine, boolean collectAll) throws IOException, NoMoreDataException {
String sep = "";
while (true) {
String line = reader.readLine();
@@ -144,20 +162,12 @@ public class TrecContentSource extends C
rawDocSize += line.length();
- if (line.startsWith(prefix)) {
+ if (lineStart!=null && line.startsWith(lineStart)) {
if (collectMatchLine) {
buf.append(sep).append(line);
sep = NEW_LINE;
}
- break;
- }
-
- if (terminatingTag != null && line.startsWith(terminatingTag)) {
- // didn't find the prefix that was asked, but the terminating
- // tag was found. set the length to 0 to signal no match was
- // found.
- buf.setLength(0);
- break;
+ return;
}
if (collectAll) {
@@ -169,7 +179,7 @@ public class TrecContentSource extends C
void openNextFile() throws NoMoreDataException, IOException {
close();
- int retries = 0;
+ currPathType = null;
while (true) {
if (nextFile >= inputFiles.size()) {
// exhausted files, start a new round, unless forever set to false.
@@ -184,13 +194,13 @@ public class TrecContentSource extends C
System.out.println("opening: " + f + " length: " + f.length());
}
try {
- GZIPInputStream zis = new GZIPInputStream(new FileInputStream(f), BUFFER_SIZE);
- reader = new BufferedReader(new InputStreamReader(zis, encoding), BUFFER_SIZE);
+ InputStream inputStream = getInputStream(f); // support either gzip, bzip2, or regular text file, by extension
+ reader = new BufferedReader(new InputStreamReader(inputStream, encoding), BUFFER_SIZE);
+ currPathType = TrecDocParser.pathType(f);
return;
} catch (Exception e) {
- retries++;
- if (retries < 20 && verbose) {
- System.out.println("Skipping 'bad' file " + f.getAbsolutePath() + " #retries=" + retries);
+ if (verbose) {
+ System.out.println("Skipping 'bad' file " + f.getAbsolutePath()+" due to "+e.getMessage());
continue;
}
throw new NoMoreDataException();
@@ -198,7 +208,7 @@ public class TrecContentSource extends C
}
}
- Date parseDate(String dateStr) {
+ public Date parseDate(String dateStr) {
dateStr = dateStr.trim();
DateFormatInfo dfi = getDateFormatInfo();
for (int i = 0; i < dfi.dfs.length; i++) {
@@ -237,70 +247,47 @@ public class TrecContentSource extends C
@Override
public DocData getNextDocData(DocData docData) throws NoMoreDataException, IOException {
- String dateStr = null, name = null;
- Reader r = null;
+ String name = null;
+ StringBuilder docBuf = getDocBuffer();
+ ParsePathType parsedPathType;
+
// protect reading from the TREC files by multiple threads. The rest of the
- // method, i.e., parsing the content and returning the DocData can run
- // unprotected.
+ // method, i.e., parsing the content and returning the DocData can run unprotected.
synchronized (lock) {
if (reader == null) {
openNextFile();
}
-
- StringBuilder docBuf = getDocBuffer();
- // 1. skip until doc start
+ // 1. skip until doc start - required for all TREC formats
docBuf.setLength(0);
- read(docBuf, DOC, false, false, null);
-
- // 2. name
+ read(docBuf, DOC, false, false);
+
+ // save parsedFile for passing trecDataParser after the sync block, in
+ // case another thread will open another file in between.
+ parsedPathType = currPathType;
+
+ // 2. name - required for all TREC formats
docBuf.setLength(0);
- read(docBuf, DOCNO, true, false, null);
+ read(docBuf, DOCNO, true, false);
name = docBuf.substring(DOCNO.length(), docBuf.indexOf(TERMINATING_DOCNO,
- DOCNO.length()));
- if (!excludeDocnameIteration)
+ DOCNO.length())).trim();
+
+ if (!excludeDocnameIteration) {
name = name + "_" + iteration;
-
- // 3. skip until doc header
- docBuf.setLength(0);
- read(docBuf, DOCHDR, false, false, null);
-
- boolean findTerminatingDocHdr = false;
-
- // 4. date - look for the date only until /DOCHDR
- docBuf.setLength(0);
- read(docBuf, DATE, true, false, TERMINATING_DOCHDR);
- if (docBuf.length() != 0) {
- // Date found.
- dateStr = docBuf.substring(DATE.length());
- findTerminatingDocHdr = true;
}
- // 5. skip until end of doc header
- if (findTerminatingDocHdr) {
- docBuf.setLength(0);
- read(docBuf, TERMINATING_DOCHDR, false, false, null);
- }
-
- // 6. collect until end of doc
+ // 3. read all until end of doc
docBuf.setLength(0);
- read(docBuf, TERMINATING_DOC, false, true, null);
-
- // 7. Set up a Reader over the read content
- r = getTrecDocReader(docBuf);
- // Resetting the thread's reader means it will reuse the instance
- // allocated as well as re-read from docBuf.
- r.reset();
-
- // count char length of parsed html text (larger than the plain doc body text).
- addBytes(docBuf.length());
+ read(docBuf, TERMINATING_DOC, false, true);
}
+
+ // count char length of text to be parsed (may be larger than the resulted plain doc body text).
+ addBytes(docBuf.length());
// This code segment relies on HtmlParser being thread safe. When we get
// here, everything else is already private to that thread, so we're safe.
- Date date = dateStr != null ? parseDate(dateStr) : null;
try {
- docData = htmlParser.parse(docData, name, date, r, null);
+ docData = trecDocParser.parse(docData, name, this, docBuf, parsedPathType);
addDoc();
} catch (InterruptedException ie) {
throw new ThreadInterruptedException(ie);
@@ -322,27 +309,40 @@ public class TrecContentSource extends C
@Override
public void setConfig(Config config) {
super.setConfig(config);
+ // dirs
File workDir = new File(config.get("work.dir", "work"));
String d = config.get("docs.dir", "trec");
dataDir = new File(d);
if (!dataDir.isAbsolute()) {
dataDir = new File(workDir, d);
}
+ // files
collectFiles(dataDir, inputFiles);
if (inputFiles.size() == 0) {
throw new IllegalArgumentException("No files in dataDir: " + dataDir);
}
+ // trec doc parser
+ try {
+ String trecDocParserClassName = config.get("trec.doc.parser", "org.apache.lucene.benchmark.byTask.feeds.TrecGov2Parser");
+ trecDocParser = Class.forName(trecDocParserClassName).asSubclass(TrecDocParser.class).newInstance();
+ } catch (Exception e) {
+ // Should not get here. Throw runtime exception.
+ throw new RuntimeException(e);
+ }
+ // html parser
try {
- String parserClassName = config.get("html.parser",
+ String htmlParserClassName = config.get("html.parser",
"org.apache.lucene.benchmark.byTask.feeds.DemoHTMLParser");
- htmlParser = Class.forName(parserClassName).asSubclass(HTMLParser.class).newInstance();
+ htmlParser = Class.forName(htmlParserClassName).asSubclass(HTMLParser.class).newInstance();
} catch (Exception e) {
// Should not get here. Throw runtime exception.
throw new RuntimeException(e);
}
+ // encoding
if (encoding == null) {
encoding = "ISO-8859-1";
}
+ // iteration exclusion in doc name
excludeDocnameIteration = config.get("content.source.excludeIteration", false);
}
Modified: lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StringBuilderReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StringBuilderReader.java?rev=1067359&r1=1067358&r2=1067359&view=diff
==============================================================================
--- lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StringBuilderReader.java (original)
+++ lucene/dev/trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StringBuilderReader.java Sat Feb 5 00:35:09 2011
@@ -158,8 +158,10 @@ public class StringBuilderReader extends
synchronized (lock) {
this.sb = sb;
length = sb.length();
+ next = mark = 0;
}
}
+
@Override
public long skip(long ns) throws IOException {
synchronized (lock) {
Modified: lucene/dev/trunk/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java?rev=1067359&r1=1067358&r2=1067359&view=diff
==============================================================================
--- lucene/dev/trunk/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java (original)
+++ lucene/dev/trunk/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java Sat Feb 5 00:35:09 2011
@@ -18,14 +18,20 @@ package org.apache.lucene.benchmark.byTa
*/
import java.io.BufferedReader;
+import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.text.ParseException;
+import java.util.Arrays;
import java.util.Date;
+import java.util.HashSet;
+import java.util.Properties;
+import org.apache.lucene.benchmark.byTask.feeds.TrecDocParser.ParsePathType;
import org.apache.lucene.benchmark.byTask.utils.Config;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
public class TrecContentSourceTest extends LuceneTestCase {
@@ -329,5 +335,62 @@ public class TrecContentSourceTest exten
// Don't test that NoMoreDataException is thrown, since the forever flag is
// turned on.
}
+
+ /**
+ * Open a trec content source over a directory with files of all trec path types and all
+ * supported formats - bzip, gzip, txt.
+ */
+ public void testTrecFeedDirAllTypes() throws Exception {
+ File dataDir = _TestUtil.getTempDir("trecFeedAllTypes");
+ _TestUtil.unzip(getDataFile("trecdocs.zip"), dataDir);
+ TrecContentSource tcs = new TrecContentSource();
+ Properties props = new Properties();
+ props.setProperty("print.props", "false");
+ props.setProperty("content.source.verbose", "false");
+ props.setProperty("content.source.excludeIteration", "true");
+ props.setProperty("doc.maker.forever", "false");
+ props.setProperty("docs.dir", dataDir.getCanonicalPath().replace('\\','/'));
+ props.setProperty("trec.doc.parser", TrecParserByPath.class.getName());
+ props.setProperty("content.source.forever", "false");
+ tcs.setConfig(new Config(props));
+ tcs.resetInputs();
+ DocData dd = new DocData();
+ int n = 0;
+ boolean gotExpectedException = false;
+ HashSet<ParsePathType> unseenTypes = new HashSet<ParsePathType>(Arrays.asList(ParsePathType.values()));
+ try {
+ while (n<100) { // arbiterary limit to prevent looping forever in case of test failure
+ dd = tcs.getNextDocData(dd);
+ ++n;
+ assertNotNull("doc data "+n+" should not be null!", dd);
+ unseenTypes.remove(tcs.currPathType);
+ switch(tcs.currPathType) {
+ case GOV2:
+ assertDocData(dd, "TEST-000", "TEST-000 title", "TEST-000 text", tcs.parseDate("Sun, 11 Jan 2009 08:00:00 GMT"));
+ break;
+ case FBIS:
+ assertDocData(dd, "TEST-001", "TEST-001 Title", "TEST-001 text", tcs.parseDate("1 January 1991"));
+ break;
+ case FR94:
+ // no title extraction in this source for now
+ assertDocData(dd, "TEST-002", null, "DEPARTMENT OF SOMETHING", tcs.parseDate("February 3, 1994"));
+ break;
+ case FT:
+ assertDocData(dd, "TEST-003", "Test-003 title", "Some pub text", tcs.parseDate("980424"));
+ break;
+ case LATIMES:
+ assertDocData(dd, "TEST-004", "Test-004 Title", "Some paragraph", tcs.parseDate("January 17, 1997, Sunday"));
+ break;
+ default:
+ assertTrue("Should never get here!", false);
+ }
+ }
+ } catch (NoMoreDataException e) {
+ gotExpectedException = true;
+ }
+ assertTrue("Should have gotten NoMoreDataException!", gotExpectedException);
+ assertEquals("Wrong numbre of documents created by osurce!",5,n);
+ assertTrue("Did not see all types!",unseenTypes.isEmpty());
+ }
}