You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/01/18 18:57:06 UTC
svn commit: r1232974 - in /lucene/dev/trunk/lucene/src:
java/org/apache/lucene/codecs/ java/org/apache/lucene/codecs/lucene3x/
java/org/apache/lucene/codecs/lucene40/
java/org/apache/lucene/codecs/simpletext/ java/org/apache/lucene/index/
test-framewor...
Author: rmuir
Date: Wed Jan 18 17:57:05 2012
New Revision: 1232974
URL: http://svn.apache.org/viewvc?rev=1232974&view=rev
Log:
LUCENE-3576: add surrogates shuffle for 3.x term vectors, test surrogates in TestBackwardsCompatibility
Added:
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/index.36.surrogates.zip (with props)
Modified:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWTermVectorsFormat.java
lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWTermVectorsWriter.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/TermVectorsWriter.java?rev=1232974&r1=1232973&r2=1232974&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/TermVectorsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/TermVectorsWriter.java Wed Jan 18 17:57:05 2012
@@ -19,6 +19,7 @@ package org.apache.lucene.codecs;
import java.io.Closeable;
import java.io.IOException;
+import java.util.Comparator;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
@@ -273,4 +274,8 @@ public abstract class TermVectorsWriter
assert termCount == numTerms;
}
}
+
+ /** Return the BytesRef Comparator used to sort terms
+ * before feeding to this API. */
+ public abstract Comparator<BytesRef> getComparator() throws IOException;
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java?rev=1232974&r1=1232973&r2=1232974&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java Wed Jan 18 17:57:05 2012
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.lucene3
*/
import java.io.IOException;
+import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
@@ -265,11 +266,13 @@ public class Lucene3xTermVectorsReader e
private class TVTerms extends Terms {
private final int numTerms;
private final long tvfFPStart;
+ private final boolean unicodeSortOrder;
public TVTerms(long tvfFP) throws IOException {
tvf.seek(tvfFP);
numTerms = tvf.readVInt();
tvfFPStart = tvf.getFilePointer();
+ unicodeSortOrder = sortTermsByUnicode();
}
@Override
@@ -283,7 +286,7 @@ public class Lucene3xTermVectorsReader e
} else {
termsEnum = new TVTermsEnum();
}
- termsEnum.reset(numTerms, tvfFPStart);
+ termsEnum.reset(numTerms, tvfFPStart, unicodeSortOrder);
return termsEnum;
}
@@ -310,27 +313,32 @@ public class Lucene3xTermVectorsReader e
@Override
public Comparator<BytesRef> getComparator() {
- // TODO: really indexer hardwires
- // this...? I guess codec could buffer and re-sort...
- return BytesRef.getUTF8SortedAsUnicodeComparator();
+ if (unicodeSortOrder) {
+ return BytesRef.getUTF8SortedAsUnicodeComparator();
+ } else {
+ return BytesRef.getUTF8SortedAsUTF16Comparator();
+ }
}
}
+ static class TermAndPostings {
+ BytesRef term;
+ int freq;
+ int[] positions;
+ int[] startOffsets;
+ int[] endOffsets;
+ }
+
private class TVTermsEnum extends TermsEnum {
+ private boolean unicodeSortOrder;
private final IndexInput origTVF;
private final IndexInput tvf;
private int numTerms;
- private int nextTerm;
- private int freq;
- private BytesRef lastTerm = new BytesRef();
- private BytesRef term = new BytesRef();
+ private int currentTerm;
private boolean storePositions;
private boolean storeOffsets;
- private long tvfFP;
-
- private int[] positions;
- private int[] startOffsets;
- private int[] endOffsets;
+
+ private TermAndPostings[] termAndPostings;
// NOTE: tvf is pre-positioned by caller
public TVTermsEnum() throws IOException {
@@ -342,37 +350,81 @@ public class Lucene3xTermVectorsReader e
return tvf == origTVF;
}
- public void reset(int numTerms, long tvfFPStart) throws IOException {
+ public void reset(int numTerms, long tvfFPStart, boolean unicodeSortOrder) throws IOException {
this.numTerms = numTerms;
- nextTerm = 0;
+ currentTerm = -1;
tvf.seek(tvfFPStart);
final byte bits = tvf.readByte();
storePositions = (bits & STORE_POSITIONS_WITH_TERMVECTOR) != 0;
storeOffsets = (bits & STORE_OFFSET_WITH_TERMVECTOR) != 0;
- tvfFP = 1+tvfFPStart;
- positions = null;
- startOffsets = null;
- endOffsets = null;
+ this.unicodeSortOrder = unicodeSortOrder;
+ readVectors();
+ if (unicodeSortOrder) {
+ Arrays.sort(termAndPostings, new Comparator<TermAndPostings>() {
+ public int compare(TermAndPostings left, TermAndPostings right) {
+ return left.term.compareTo(right.term);
+ }
+ });
+ }
}
+
+ private void readVectors() throws IOException {
+ termAndPostings = new TermAndPostings[numTerms];
+ BytesRef lastTerm = new BytesRef();
+ for (int i = 0; i < numTerms; i++) {
+ TermAndPostings t = new TermAndPostings();
+ BytesRef term = new BytesRef();
+ term.copyBytes(lastTerm);
+ final int start = tvf.readVInt();
+ final int deltaLen = tvf.readVInt();
+ term.length = start + deltaLen;
+ term.grow(term.length);
+ tvf.readBytes(term.bytes, start, deltaLen);
+ t.term = term;
+ int freq = tvf.readVInt();
+ t.freq = freq;
+
+ if (storePositions) {
+ int positions[] = new int[freq];
+ int pos = 0;
+ for(int posUpto=0;posUpto<freq;posUpto++) {
+ pos += tvf.readVInt();
+ positions[posUpto] = pos;
+ }
+ t.positions = positions;
+ }
- // NOTE: slow! (linear scan)
- @Override
- public SeekStatus seekCeil(BytesRef text, boolean useCache)
- throws IOException {
- if (nextTerm != 0 && text.compareTo(term) < 0) {
- nextTerm = 0;
- tvf.seek(tvfFP);
+ if (storeOffsets) {
+ int startOffsets[] = new int[freq];
+ int endOffsets[] = new int[freq];
+ int offset = 0;
+ for(int posUpto=0;posUpto<freq;posUpto++) {
+ startOffsets[posUpto] = offset + tvf.readVInt();
+ offset = endOffsets[posUpto] = startOffsets[posUpto] + tvf.readVInt();
+ }
+ t.startOffsets = startOffsets;
+ t.endOffsets = endOffsets;
+ }
+ lastTerm.copyBytes(term);
+ termAndPostings[i] = t;
}
+ }
- while (next() != null) {
- final int cmp = text.compareTo(term);
+ // NOTE: slow! (linear scan)
+ @Override
+ public SeekStatus seekCeil(BytesRef text, boolean useCache) throws IOException {
+ Comparator<BytesRef> comparator = getComparator();
+ for (int i = 0; i < numTerms; i++) {
+ int cmp = comparator.compare(text, termAndPostings[i].term);
if (cmp < 0) {
+ currentTerm = i;
return SeekStatus.NOT_FOUND;
} else if (cmp == 0) {
+ currentTerm = i;
return SeekStatus.FOUND;
}
}
-
+ currentTerm = termAndPostings.length;
return SeekStatus.END;
}
@@ -383,47 +435,15 @@ public class Lucene3xTermVectorsReader e
@Override
public BytesRef next() throws IOException {
- if (nextTerm >= numTerms) {
+ if (++currentTerm >= numTerms) {
return null;
}
- term.copyBytes(lastTerm);
- final int start = tvf.readVInt();
- final int deltaLen = tvf.readVInt();
- term.length = start + deltaLen;
- term.grow(term.length);
- tvf.readBytes(term.bytes, start, deltaLen);
- freq = tvf.readVInt();
-
- if (storePositions) {
- // TODO: we could maybe reuse last array, if we can
- // somehow be careful about consumer never using two
- // D&PEnums at once...
- positions = new int[freq];
- int pos = 0;
- for(int posUpto=0;posUpto<freq;posUpto++) {
- pos += tvf.readVInt();
- positions[posUpto] = pos;
- }
- }
-
- if (storeOffsets) {
- startOffsets = new int[freq];
- endOffsets = new int[freq];
- int offset = 0;
- for(int posUpto=0;posUpto<freq;posUpto++) {
- startOffsets[posUpto] = offset + tvf.readVInt();
- offset = endOffsets[posUpto] = startOffsets[posUpto] + tvf.readVInt();
- }
- }
-
- lastTerm.copyBytes(term);
- nextTerm++;
- return term;
+ return term();
}
@Override
public BytesRef term() {
- return term;
+ return termAndPostings[currentTerm].term;
}
@Override
@@ -438,7 +458,7 @@ public class Lucene3xTermVectorsReader e
@Override
public long totalTermFreq() {
- return freq;
+ return termAndPostings[currentTerm].freq;
}
@Override
@@ -449,7 +469,7 @@ public class Lucene3xTermVectorsReader e
} else {
docsEnum = new TVDocsEnum();
}
- docsEnum.reset(liveDocs, freq);
+ docsEnum.reset(liveDocs, termAndPostings[currentTerm]);
return docsEnum;
}
@@ -469,15 +489,17 @@ public class Lucene3xTermVectorsReader e
} else {
docsAndPositionsEnum = new TVDocsAndPositionsEnum();
}
- docsAndPositionsEnum.reset(liveDocs, positions, startOffsets, endOffsets);
+ docsAndPositionsEnum.reset(liveDocs, termAndPostings[currentTerm]);
return docsAndPositionsEnum;
}
@Override
public Comparator<BytesRef> getComparator() {
- // TODO: really indexer hardwires
- // this...? I guess codec could buffer and re-sort...
- return BytesRef.getUTF8SortedAsUnicodeComparator();
+ if (unicodeSortOrder) {
+ return BytesRef.getUTF8SortedAsUnicodeComparator();
+ } else {
+ return BytesRef.getUTF8SortedAsUTF16Comparator();
+ }
}
}
@@ -518,9 +540,9 @@ public class Lucene3xTermVectorsReader e
}
}
- public void reset(Bits liveDocs, int freq) {
+ public void reset(Bits liveDocs, TermAndPostings termAndPostings) {
this.liveDocs = liveDocs;
- this.freq = freq;
+ this.freq = termAndPostings.freq;
this.doc = -1;
didNext = false;
}
@@ -569,11 +591,11 @@ public class Lucene3xTermVectorsReader e
}
}
- public void reset(Bits liveDocs, int[] positions, int[] startOffsets, int[] endOffsets) {
+ public void reset(Bits liveDocs, TermAndPostings termAndPostings) {
this.liveDocs = liveDocs;
- this.positions = positions;
- this.startOffsets = startOffsets;
- this.endOffsets = endOffsets;
+ this.positions = termAndPostings.positions;
+ this.startOffsets = termAndPostings.startOffsets;
+ this.endOffsets = termAndPostings.endOffsets;
this.doc = -1;
didNext = false;
nextPos = 0;
@@ -668,5 +690,14 @@ public class Lucene3xTermVectorsReader e
}
}
}
+
+ // If this returns, we do the surrogates shuffle so that the
+ // terms are sorted by unicode sort order. This should be
+ // true when segments are used for "normal" searching;
+ // it's only false during testing, to create a pre-flex
+ // index, using the test-only PreFlexRW.
+ protected boolean sortTermsByUnicode() {
+ return true;
+ }
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java?rev=1232974&r1=1232973&r2=1232974&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java Wed Jan 18 17:57:05 2012
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.lucene4
*/
import java.io.IOException;
+import java.util.Comparator;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.codecs.TermVectorsWriter;
@@ -365,4 +366,9 @@ public final class Lucene40TermVectorsWr
IOUtils.close(tvx, tvd, tvf);
tvx = tvd = tvf = null;
}
+
+ @Override
+ public Comparator<BytesRef> getComparator() throws IOException {
+ return BytesRef.getUTF8SortedAsUnicodeComparator();
+ }
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java?rev=1232974&r1=1232973&r2=1232974&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java Wed Jan 18 17:57:05 2012
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.simplet
*/
import java.io.IOException;
+import java.util.Comparator;
import org.apache.lucene.codecs.TermVectorsWriter;
import org.apache.lucene.index.FieldInfo;
@@ -170,6 +171,11 @@ public class SimpleTextTermVectorsWriter
}
}
+ @Override
+ public Comparator<BytesRef> getComparator() throws IOException {
+ return BytesRef.getUTF8SortedAsUnicodeComparator();
+ }
+
private void write(String s) throws IOException {
SimpleTextUtil.write(out, s, scratch);
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java?rev=1232974&r1=1232973&r2=1232974&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java Wed Jan 18 17:57:05 2012
@@ -118,9 +118,7 @@ final class TermVectorsConsumerPerField
TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
final TermVectorsWriter tv = termsWriter.writer;
- // TODO: we may want to make this sort in same order
- // as Codec's terms dict?
- final int[] termIDs = termsHashPerField.sortPostings(BytesRef.getUTF8SortedAsUnicodeComparator());
+ final int[] termIDs = termsHashPerField.sortPostings(tv.getComparator());
tv.startField(fieldInfo, numPostings, doVectorPositions, doVectorOffsets);
Modified: lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWTermVectorsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWTermVectorsFormat.java?rev=1232974&r1=1232973&r2=1232974&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWTermVectorsFormat.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWTermVectorsFormat.java Wed Jan 18 17:57:05 2012
@@ -19,10 +19,15 @@ package org.apache.lucene.codecs.preflex
import java.io.IOException;
+import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.codecs.TermVectorsWriter;
import org.apache.lucene.codecs.lucene3x.Lucene3xTermVectorsFormat;
+import org.apache.lucene.codecs.lucene3x.Lucene3xTermVectorsReader;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.LuceneTestCase;
public class PreFlexRWTermVectorsFormat extends Lucene3xTermVectorsFormat {
@@ -30,4 +35,30 @@ public class PreFlexRWTermVectorsFormat
public TermVectorsWriter vectorsWriter(Directory directory, String segment, IOContext context) throws IOException {
return new PreFlexRWTermVectorsWriter(directory, segment, context);
}
+
+ @Override
+ public TermVectorsReader vectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) throws IOException {
+ return new Lucene3xTermVectorsReader(directory, segmentInfo, fieldInfos, context) {
+ @Override
+ protected boolean sortTermsByUnicode() {
+ // We carefully peek into stack track above us: if
+ // we are part of a "merge", we must sort by UTF16:
+ boolean unicodeSortOrder = true;
+
+ StackTraceElement[] trace = new Exception().getStackTrace();
+ for (int i = 0; i < trace.length; i++) {
+ //System.out.println(trace[i].getClassName());
+ if ("merge".equals(trace[i].getMethodName())) {
+ unicodeSortOrder = false;
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("NOTE: PreFlexRW codec: forcing legacy UTF16 vector term sort order");
+ }
+ break;
+ }
+ }
+
+ return unicodeSortOrder;
+ }
+ };
+ }
}
Modified: lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWTermVectorsWriter.java?rev=1232974&r1=1232973&r2=1232974&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWTermVectorsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWTermVectorsWriter.java Wed Jan 18 17:57:05 2012
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.preflex
*/
import java.io.IOException;
+import java.util.Comparator;
import org.apache.lucene.codecs.TermVectorsWriter;
import org.apache.lucene.codecs.lucene3x.Lucene3xTermVectorsReader;
@@ -32,7 +33,6 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.StringHelper;
-// TODO: surrogates dance!
public final class PreFlexRWTermVectorsWriter extends TermVectorsWriter {
private final Directory directory;
private final String segment;
@@ -213,4 +213,9 @@ public final class PreFlexRWTermVectorsW
IOUtils.close(tvx, tvd, tvf);
tvx = tvd = tvf = null;
}
+
+ @Override
+ public Comparator<BytesRef> getComparator() throws IOException {
+ return BytesRef.getUTF8SortedAsUTF16Comparator();
+ }
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1232974&r1=1232973&r2=1232974&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Wed Jan 18 17:57:05 2012
@@ -734,5 +734,16 @@ public class TestBackwardsCompatibility
dir.close();
}
}
+
+ public static final String surrogatesIndexName = "index.36.surrogates.zip";
+
+ public void testSurrogates() throws Exception {
+ File oldIndexDir = _TestUtil.getTempDir("surrogates");
+ _TestUtil.unzip(getDataFile(surrogatesIndexName), oldIndexDir);
+ Directory dir = newFSDirectory(oldIndexDir);
+ // TODO: more tests
+ _TestUtil.checkIndex(dir);
+ dir.close();
+ }
}
Added: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/index.36.surrogates.zip
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/index.36.surrogates.zip?rev=1232974&view=auto
==============================================================================
Binary file - no diff available.