You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2010/11/05 22:51:57 UTC
svn commit: r1031832 -
/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java
Author: mikemccand
Date: Fri Nov 5 21:51:56 2010
New Revision: 1031832
URL: http://svn.apache.org/viewvc?rev=1031832&view=rev
Log:
LUCENE-2743: speed up SimpleText codec
Modified:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java?rev=1031832&r1=1031831&r2=1031832&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java Fri Nov 5 21:51:56 2010
@@ -33,7 +33,11 @@ import org.apache.lucene.util.StringHelp
import java.io.IOException;
import java.util.Comparator;
import java.util.Map;
+import java.util.Set;
import java.util.HashMap;
+import java.util.TreeMap;
+import java.util.SortedMap;
+import java.util.Iterator;
class SimpleTextFieldsReader extends FieldsProducer {
@@ -78,7 +82,7 @@ class SimpleTextFieldsReader extends Fie
private class SimpleTextFieldsEnum extends FieldsEnum {
private final IndexInput in;
private final BytesRef scratch = new BytesRef(10);
- private boolean omitTF;
+ private String current;
public SimpleTextFieldsEnum() {
this.in = (IndexInput) SimpleTextFieldsReader.this.in.clone();
@@ -89,11 +93,12 @@ class SimpleTextFieldsReader extends Fie
while(true) {
readLine(in, scratch);
if (scratch.equals(END)) {
+ current = null;
return null;
}
if (scratch.startsWith(FIELD)) {
String field = StringHelper.intern(new String(scratch.bytes, scratch.offset + FIELD.length, scratch.length - FIELD.length, "UTF-8"));
- omitTF = fieldInfos.fieldInfo(field).omitTermFreqAndPositions;
+ current = field;
return field;
}
}
@@ -101,7 +106,7 @@ class SimpleTextFieldsReader extends Fie
@Override
public TermsEnum terms() throws IOException {
- return new SimpleTextTermsEnum(in.getFilePointer(), omitTF);
+ return SimpleTextFieldsReader.this.terms(current).iterator();
}
}
@@ -109,21 +114,42 @@ class SimpleTextFieldsReader extends Fie
private final IndexInput in;
private final boolean omitTF;
private BytesRef current;
- private final long fieldStart;
- private final BytesRef scratch = new BytesRef(10);
- private final BytesRef scratch2 = new BytesRef(10);
private int docFreq;
private long docsStart;
private boolean ended;
+ private final TreeMap<BytesRef,TermData> allTerms;
+ private Iterator<Map.Entry<BytesRef,TermData>> iter;
- public SimpleTextTermsEnum(long offset, boolean omitTF) throws IOException {
+ public SimpleTextTermsEnum(TreeMap<BytesRef,TermData> allTerms, boolean omitTF) throws IOException {
this.in = (IndexInput) SimpleTextFieldsReader.this.in.clone();
- this.in.seek(offset);
+ this.allTerms = allTerms;
this.omitTF = omitTF;
- fieldStart = offset;
+ iter = allTerms.entrySet().iterator();
}
public SeekStatus seek(BytesRef text, boolean useCache /* ignored */) throws IOException {
+
+ final SortedMap<BytesRef,TermData> tailMap = allTerms.tailMap(text);
+
+ if (tailMap.isEmpty()) {
+ current = null;
+ return SeekStatus.END;
+ } else {
+ current = tailMap.firstKey();
+ final TermData td = tailMap.get(current);
+ docsStart = td.docsStart;
+ docFreq = td.docFreq;
+ iter = tailMap.entrySet().iterator();
+ assert iter.hasNext();
+ iter.next();
+ if (current.equals(text)) {
+ return SeekStatus.FOUND;
+ } else {
+ return SeekStatus.NOT_FOUND;
+ }
+ }
+
+ /*
if (current != null) {
final int cmp = current.compareTo(text);
if (cmp == 0) {
@@ -153,6 +179,7 @@ class SimpleTextFieldsReader extends Fie
current = null;
ended = true;
return SeekStatus.END;
+ */
}
@Override
@@ -162,6 +189,20 @@ class SimpleTextFieldsReader extends Fie
@Override
public BytesRef next() throws IOException {
assert !ended;
+
+ if (iter.hasNext()) {
+ Map.Entry<BytesRef,TermData> ent = iter.next();
+ current = ent.getKey();
+ TermData td = ent.getValue();
+ docFreq = td.docFreq;
+ docsStart = td.docsStart;
+ return current;
+ } else {
+ current = null;
+ return null;
+ }
+
+ /*
readLine(in, scratch);
if (scratch.equals(END) || scratch.startsWith(FIELD)) {
ended = true;
@@ -192,6 +233,7 @@ class SimpleTextFieldsReader extends Fie
in.seek(lineStart);
return current;
}
+ */
}
@Override
@@ -447,20 +489,70 @@ class SimpleTextFieldsReader extends Fie
}
}
+ static class TermData {
+ public long docsStart;
+ public int docFreq;
+
+ public TermData(long docsStart, int docFreq) {
+ this.docsStart = docsStart;
+ this.docFreq = docFreq;
+ }
+ }
+
private class SimpleTextTerms extends Terms {
private final String field;
private final long termsStart;
private final boolean omitTF;
- public SimpleTextTerms(String field, long termsStart) {
+ // NOTE: horribly, horribly RAM consuming, but then
+ // SimpleText should never be used in production
+ private final TreeMap<BytesRef,TermData> allTerms = new TreeMap<BytesRef,TermData>();
+
+ private final BytesRef scratch = new BytesRef(10);
+
+ public SimpleTextTerms(String field, long termsStart) throws IOException {
this.field = StringHelper.intern(field);
this.termsStart = termsStart;
omitTF = fieldInfos.fieldInfo(field).omitTermFreqAndPositions;
+ loadTerms();
+ }
+
+ private void loadTerms() throws IOException {
+ IndexInput in = (IndexInput) SimpleTextFieldsReader.this.in.clone();
+ in.seek(termsStart);
+ final BytesRef lastTerm = new BytesRef(10);
+ long lastDocsStart = -1;
+ int docFreq = 0;
+ while(true) {
+ readLine(in, scratch);
+ if (scratch.equals(END) || scratch.startsWith(FIELD)) {
+ if (lastDocsStart != -1) {
+ allTerms.put(new BytesRef(lastTerm),
+ new TermData(lastDocsStart, docFreq));
+ }
+ break;
+ } else if (scratch.startsWith(DOC)) {
+ docFreq++;
+ } else if (scratch.startsWith(TERM)) {
+ if (lastDocsStart != -1) {
+ allTerms.put(new BytesRef(lastTerm),
+ new TermData(lastDocsStart, docFreq));
+ }
+ lastDocsStart = in.getFilePointer();
+ final int len = scratch.length - TERM.length;
+ if (len > lastTerm.length) {
+ lastTerm.grow(len);
+ }
+ System.arraycopy(scratch.bytes, TERM.length, lastTerm.bytes, 0, len);
+ lastTerm.length = len;
+ docFreq = 0;
+ }
+ }
}
@Override
public TermsEnum iterator() throws IOException {
- return new SimpleTextTermsEnum(termsStart, omitTF);
+ return new SimpleTextTermsEnum(allTerms, omitTF);
}
@Override