You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by do...@apache.org on 2012/12/09 14:47:18 UTC
svn commit: r1418955 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/benchmark/ lucene/benchmark/conf/
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/
lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/
Author: doronc
Date: Sun Dec 9 13:47:15 2012
New Revision: 1418955
URL: http://svn.apache.org/viewvc?rev=1418955&view=rev
Log:
LUCENE-4590: Merge from trunk: Add WriteEnwikiLineDoc which writes Wikipedia category pages to a separate file.
Added:
lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTask.java
- copied, changed from r1418852, lucene/dev/trunk/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTask.java
lucene/dev/branches/branch_4x/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTaskTest.java
- copied unchanged from r1418852, lucene/dev/trunk/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTaskTest.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/benchmark/ (props changed)
lucene/dev/branches/branch_4x/lucene/benchmark/conf/extractWikipedia.alg
lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java
lucene/dev/branches/branch_4x/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1418955&r1=1418954&r2=1418955&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Sun Dec 9 13:47:15 2012
@@ -113,6 +113,11 @@ New Features
rule files in the ICU RuleBasedBreakIterator format.
(Shawn Heisey, Robert Muir, Steve Rowe)
+* LUCENE-4590: Added WriteEnwikiLineDocTask - a benchmark task for writing
+ Wikipedia category pages and non-category pages into separate line files.
+ extractWikipedia.alg was changed to use this task, so now it creates two
+ files. (Doron Cohen)
+
API Changes
* LUCENE-4399: Deprecated AppendingCodec. Lucene's term dictionaries
Modified: lucene/dev/branches/branch_4x/lucene/benchmark/conf/extractWikipedia.alg
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/benchmark/conf/extractWikipedia.alg?rev=1418955&r1=1418954&r2=1418955&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/benchmark/conf/extractWikipedia.alg (original)
+++ lucene/dev/branches/branch_4x/lucene/benchmark/conf/extractWikipedia.alg Sun Dec 9 13:47:15 2012
@@ -41,4 +41,4 @@ content.source.forever=false
# -------------------------------------------------------------------------------------
# Process all documents, appending each one to the line file:
-{WriteLineDoc() > : *
+{WriteEnwikiLineDoc() > : *
Copied: lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTask.java (from r1418852, lucene/dev/trunk/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTask.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTask.java?p2=lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTask.java&p1=lucene/dev/trunk/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTask.java&r1=1418852&r2=1418955&rev=1418955&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTask.java (original)
+++ lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTask.java Sun Dec 9 13:47:15 2012
@@ -10,7 +10,7 @@ import org.apache.lucene.benchmark.byTas
import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
import org.apache.lucene.benchmark.byTask.utils.StreamUtils;
import org.apache.lucene.document.Document;
-import org.apache.lucene.index.StorableField;
+import org.apache.lucene.index.IndexableField;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -60,7 +60,7 @@ public class WriteEnwikiLineDocTask exte
@Override
protected PrintWriter lineFileOut(Document doc) {
- StorableField titleField = doc.getField(DocMaker.TITLE_FIELD);
+ IndexableField titleField = doc.getField(DocMaker.TITLE_FIELD);
if (titleField!=null && titleField.stringValue().startsWith("Category:")) {
return categoryLineFileOut;
}
Modified: lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java?rev=1418955&r1=1418954&r2=1418955&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java (original)
+++ lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java Sun Dec 9 13:47:15 2012
@@ -83,18 +83,20 @@ public class WriteLineDocTask extends Pe
public static final String DEFAULT_SUFFICIENT_FIELDS = DocMaker.TITLE_FIELD +',' + DocMaker.BODY_FIELD;
private int docSize = 0;
- private PrintWriter lineFileOut = null;
- private DocMaker docMaker;
- private ThreadLocal<StringBuilder> threadBuffer = new ThreadLocal<StringBuilder>();
- private ThreadLocal<Matcher> threadNormalizer = new ThreadLocal<Matcher>();
- private final String[] fieldsToWrite;;
+ protected final String fname;
+ private final PrintWriter lineFileOut;
+ private final DocMaker docMaker;
+ private final ThreadLocal<StringBuilder> threadBuffer = new ThreadLocal<StringBuilder>();
+ private final ThreadLocal<Matcher> threadNormalizer = new ThreadLocal<Matcher>();
+ private final String[] fieldsToWrite;
private final boolean[] sufficientFields;
private final boolean checkSufficientFields;
+
public WriteLineDocTask(PerfRunData runData) throws Exception {
super(runData);
Config config = runData.getConfig();
- String fname = config.get("line.file.out", null);
+ fname = config.get("line.file.out", null);
if (fname == null) {
throw new IllegalArgumentException("line.file.out must be set");
}
@@ -128,13 +130,13 @@ public class WriteLineDocTask extends Pe
}
}
- writeHeader();
+ writeHeader(lineFileOut);
}
/**
- * Write a header to the lines file - indicating how to read the file later
+ * Write header to the lines file - indicating how to read the file later.
*/
- private void writeHeader() {
+ protected void writeHeader(PrintWriter out) {
StringBuilder sb = threadBuffer.get();
if (sb == null) {
sb = new StringBuilder();
@@ -145,7 +147,7 @@ public class WriteLineDocTask extends Pe
for (String f : fieldsToWrite) {
sb.append(SEP).append(f);
}
- lineFileOut.println(sb.toString());
+ out.println(sb.toString());
}
@Override
@@ -180,12 +182,20 @@ public class WriteLineDocTask extends Pe
if (sufficient) {
sb.setLength(sb.length()-1); // remove redundant last separator
// lineFileOut is a PrintWriter, which synchronizes internally in println.
- lineFileOut.println(sb.toString());
+ lineFileOut(doc).println(sb.toString());
}
return 1;
}
+ /**
+ * Selects output line file by written doc.
+ * Default: original output line file.
+ */
+ protected PrintWriter lineFileOut(Document doc) {
+ return lineFileOut;
+ }
+
@Override
public void close() throws Exception {
lineFileOut.close();
Modified: lucene/dev/branches/branch_4x/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java?rev=1418955&r1=1418954&r2=1418955&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java Sun Dec 9 13:47:15 2012
@@ -188,7 +188,7 @@ public class WriteLineDocTaskTest extend
}
}
- private void assertHeaderLine(String line) {
+ static void assertHeaderLine(String line) {
assertTrue("First line should be a header line",line.startsWith(WriteLineDocTask.FIELDS_HEADER_INDICATOR));
}