You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2012/05/11 01:10:19 UTC
svn commit: r1336942 -
/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java
Author: uschindler
Date: Thu May 10 23:10:18 2012
New Revision: 1336942
URL: http://svn.apache.org/viewvc?rev=1336942&view=rev
Log:
Optimize rcmuir's commit to seek using RandomAccessFile and later use input stream (after seeking). Also fail on UTF-8 coding errors
Modified:
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java?rev=1336942&r1=1336941&r2=1336942&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java Thu May 10 23:10:18 2012
@@ -24,6 +24,9 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.io.RandomAccessFile;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
@@ -71,46 +74,66 @@ public class LineFileDocs implements Clo
reader = null;
}
}
+
+ private long randomSeekPos(Random random, long size) {
+ if (random == null || size <= 3L)
+ return 0L;
+ return (random.nextLong()&Long.MAX_VALUE) % (size/3);
+ }
private synchronized void open(Random random) throws IOException {
InputStream is = getClass().getResourceAsStream(path);
+ boolean needSkip = true;
+ long size = 0L;
if (is == null) {
// if its not in classpath, we load it as absolute filesystem path (e.g. Hudson's home dir)
- is = new FileInputStream(path);
- }
- File file = new File(path);
- long size;
- if (file.exists()) {
- size = file.length();
+ File file = new File(path);
+ if (path.endsWith(".gz")) {
+ // if it is a gzip file, we need to use InputStream and slowly skipTo:
+ is = new FileInputStream(file);
+ } else {
+ // optimized seek using RandomAccessFile:
+ size = file.length();
+ final long seekTo = randomSeekPos(random, size);
+ final FileChannel channel = new RandomAccessFile(path, "r").getChannel();
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("TEST: LineFileDocs: file seek to fp=" + seekTo + " on open");
+ }
+ channel.position(seekTo);
+ is = Channels.newInputStream(channel);
+ needSkip = false;
+ }
} else {
+ // if the file comes from Classpath:
size = is.available();
}
+
if (path.endsWith(".gz")) {
is = new GZIPInputStream(is);
// guestimate:
size *= 2.8;
}
-
- // Override sizes for currently "known" line files:
- if (path.equals("europarl.lines.txt.gz")) {
- size = 15129506L;
- } else if (path.equals("/home/hudson/lucene-data/enwiki.random.lines.txt.gz")) {
- size = 3038178822L;
- }
-
- // Randomly seek to starting point:
- if (random != null && size > 3) {
- final long seekTo = (random.nextLong()&Long.MAX_VALUE) % (size/3);
+
+ // If we only have an InputStream, we need to seek now,
+ // but this seek is a scan, so very inefficient!!!
+ if (needSkip) {
+ final long skipTo = randomSeekPos(random, size);
if (LuceneTestCase.VERBOSE) {
- System.out.println("TEST: LineFileDocs: seek to fp=" + seekTo + " on open");
+ System.out.println("TEST: LineFileDocs: stream skip to fp=" + skipTo + " on open");
}
- is.skip(seekTo);
- CharsetDecoder decoder = Charset.forName("UTF-8").newDecoder()
- .onMalformedInput(CodingErrorAction.IGNORE)
- .onUnmappableCharacter(CodingErrorAction.IGNORE);
- reader = new BufferedReader(new InputStreamReader(is, decoder), BUFFER_SIZE);
- reader.readLine();
+ is.skip(skipTo);
}
+
+ int b;
+ do {
+ b = is.read();
+ } while (b >= 0 && b != 13 && b != 10);
+ CharsetDecoder decoder = Charset.forName("UTF-8").newDecoder()
+ .onMalformedInput(CodingErrorAction.REPORT)
+ .onUnmappableCharacter(CodingErrorAction.REPORT);
+ reader = new BufferedReader(new InputStreamReader(is, decoder), BUFFER_SIZE);
+ // read one more line, to make sure we are not inside a Windows linebreak (\r\n):
+ reader.readLine();
}
public synchronized void reset(Random random) throws IOException {