You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/01/08 14:51:37 UTC
svn commit: r1056705 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/contrib/icu/src/tools/java/org/apache/lucene/analysis/icu/
lucene/src/java/org/apache/lucene/analysis/standard/
lucene/src/test/org/apache/lucene/analysis/ lucene/src/test/org/...
Author: mikemccand
Date: Sat Jan 8 13:51:37 2011
New Revision: 1056705
URL: http://svn.apache.org/viewvc?rev=1056705&view=rev
Log:
fix LineFileDocs to seek to random start on open
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/icu/src/tools/java/org/apache/lucene/analysis/icu/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/standard/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/LineFileDocs.java
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java?rev=1056705&r1=1056704&r2=1056705&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java Sat Jan 8 13:51:37 2011
@@ -53,7 +53,7 @@ public class TestNRTThreads extends Luce
final long t0 = System.currentTimeMillis();
- final LineFileDocs docs = new LineFileDocs(true);
+ final LineFileDocs docs = new LineFileDocs(random);
final File tempDir = _TestUtil.getTempDir("nrtopenfiles");
final MockDirectoryWrapper dir = new MockDirectoryWrapper(random, FSDirectory.open(tempDir));
final IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer());
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/LineFileDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/LineFileDocs.java?rev=1056705&r1=1056704&r2=1056705&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/LineFileDocs.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/LineFileDocs.java Sat Jan 8 13:51:37 2011
@@ -18,6 +18,7 @@ package org.apache.lucene.util;
*/
import java.io.Closeable;
+import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.BufferedReader;
@@ -26,6 +27,7 @@ import java.io.InputStream;
import java.io.BufferedInputStream;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.GZIPInputStream;
+import java.util.Random;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -36,21 +38,19 @@ import org.apache.lucene.document.Field;
public class LineFileDocs implements Closeable {
private BufferedReader reader;
- private final boolean forever;
private final static int BUFFER_SIZE = 1 << 16; // 64K
private final AtomicInteger id = new AtomicInteger();
private final String path;
// If forever is true, we rewind the file at EOF (repeat
// the docs over and over)
- public LineFileDocs(String path, boolean forever) throws IOException {
+ public LineFileDocs(Random random, String path) throws IOException {
this.path = path;
- this.forever = forever;
- open();
+ open(random);
}
- public LineFileDocs(boolean forever) throws IOException {
- this(LuceneTestCase.TEST_LINE_DOCS_FILE, forever);
+ public LineFileDocs(Random random) throws IOException {
+ this(random, LuceneTestCase.TEST_LINE_DOCS_FILE);
}
public synchronized void close() throws IOException {
@@ -60,22 +60,49 @@ public class LineFileDocs implements Clo
}
}
- private synchronized void open() throws IOException {
+ private synchronized void open(Random random) throws IOException {
InputStream is = getClass().getResourceAsStream(path);
if (is == null) {
// if its not in classpath, we load it as absolute filesystem path (e.g. Hudson's home dir)
is = new FileInputStream(path);
}
+ File file = new File(path);
+ long size;
+ if (file.exists()) {
+ size = file.length();
+ } else {
+ size = is.available();
+ }
if (path.endsWith(".gz")) {
is = new GZIPInputStream(is);
+ // guestimate:
+ size *= 2.8;
}
+
final InputStream in = new BufferedInputStream(is, BUFFER_SIZE);
reader = new BufferedReader(new InputStreamReader(in, "UTF-8"), BUFFER_SIZE);
+
+ // Override sizes for currently "known" line files:
+ if (path.equals("europarl.lines.txt.gz")) {
+ size = 15129506L;
+ } else if (path.equals("/home/hudson/lucene-data/enwiki.random.lines.txt.gz")) {
+ size = 3038178822L;
+ }
+
+ // Randomly seek to starting point:
+ if (random != null && size > 3) {
+ final long seekTo = (random.nextLong()&Long.MAX_VALUE) % (size/3);
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("TEST: LineFileDocs: seek to fp=" + seekTo + " on open");
+ }
+ reader.skip(seekTo);
+ reader.readLine();
+ }
}
- public synchronized void reset() throws IOException {
+ public synchronized void reset(Random random) throws IOException {
close();
- open();
+ open(random);
id.set(0);
}
@@ -117,15 +144,13 @@ public class LineFileDocs implements Clo
synchronized(this) {
line = reader.readLine();
if (line == null) {
- if (forever) {
- if (LuceneTestCase.VERBOSE) {
- System.out.println("TEST: LineFileDocs: now rewind file...");
- }
- close();
- open();
- line = reader.readLine();
+ // Always rewind at end:
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("TEST: LineFileDocs: now rewind file...");
}
- return null;
+ close();
+ open(null);
+ line = reader.readLine();
}
}