You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/09/19 22:57:11 UTC
svn commit: r1524840 [3/4] - in /lucene/dev/trunk/lucene: ./
codecs/src/java/org/apache/lucene/codecs/blockterms/
codecs/src/java/org/apache/lucene/codecs/bloom/
codecs/src/java/org/apache/lucene/codecs/memory/
codecs/src/java/org/apache/lucene/codecs/...
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java Thu Sep 19 20:57:09 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
-import java.util.Comparator;
import java.util.LinkedList;
import org.apache.lucene.analysis.NumericTokenStream; // for javadocs
@@ -392,7 +391,6 @@ public final class NumericRangeQuery<T e
private BytesRef currentLowerBound, currentUpperBound;
private final LinkedList<BytesRef> rangeBounds = new LinkedList<BytesRef>();
- private final Comparator<BytesRef> termComp;
NumericRangeTermsEnum(final TermsEnum tenum) {
super(tenum);
@@ -481,15 +479,13 @@ public final class NumericRangeQuery<T e
// should never happen
throw new IllegalArgumentException("Invalid NumericType");
}
-
- termComp = getComparator();
}
private void nextRange() {
assert rangeBounds.size() % 2 == 0;
currentLowerBound = rangeBounds.removeFirst();
- assert currentUpperBound == null || termComp.compare(currentUpperBound, currentLowerBound) <= 0 :
+ assert currentUpperBound == null || currentUpperBound.compareTo(currentLowerBound) <= 0 :
"The current upper bound must be <= the new lower bound";
currentUpperBound = rangeBounds.removeFirst();
@@ -501,10 +497,10 @@ public final class NumericRangeQuery<T e
nextRange();
// if the new upper bound is before the term parameter, the sub-range is never a hit
- if (term != null && termComp.compare(term, currentUpperBound) > 0)
+ if (term != null && term.compareTo(currentUpperBound) > 0)
continue;
// never seek backwards, so use current term if lower bound is smaller
- return (term != null && termComp.compare(term, currentLowerBound) > 0) ?
+ return (term != null && term.compareTo(currentLowerBound) > 0) ?
term : currentLowerBound;
}
@@ -516,11 +512,11 @@ public final class NumericRangeQuery<T e
@Override
protected final AcceptStatus accept(BytesRef term) {
- while (currentUpperBound == null || termComp.compare(term, currentUpperBound) > 0) {
+ while (currentUpperBound == null || term.compareTo(currentUpperBound) > 0) {
if (rangeBounds.isEmpty())
return AcceptStatus.END;
// peek next sub-range, only seek if the current term is smaller than next lower bound
- if (termComp.compare(term, rangeBounds.getFirst()) < 0)
+ if (term.compareTo(rangeBounds.getFirst()) < 0)
return AcceptStatus.NO_AND_SEEK;
// step forward to next range without seeking, as next lower range bound is less or equal current term
nextRange();
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/PrefixTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/PrefixTermsEnum.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/PrefixTermsEnum.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/PrefixTermsEnum.java Thu Sep 19 20:57:09 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.util.StringHelp
* Subclass of FilteredTermEnum for enumerating all terms that match the
* specified prefix filter term.
* <p>Term enumerations are always ordered by
- * {@link #getComparator}. Each term in the enumeration is
+ * {@link BytesRef#compareTo}. Each term in the enumeration is
* greater than all that precede it.</p>
*/
public class PrefixTermsEnum extends FilteredTermsEnum {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java Thu Sep 19 20:57:09 2013
@@ -109,7 +109,7 @@ public abstract class ScoringRewrite<Q e
final int size = col.terms.size();
if (size > 0) {
- final int sort[] = col.terms.sort(col.termsEnum.getComparator());
+ final int sort[] = col.terms.sort(BytesRef.getUTF8SortedAsUnicodeComparator());
final float[] boost = col.array.boost;
final TermContext[] termStates = col.array.termState;
for (int i = 0; i < size; i++) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java Thu Sep 19 20:57:09 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
-import java.util.Comparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.Fields;
@@ -47,7 +46,6 @@ abstract class TermCollectingRewrite<Q e
final void collectTerms(IndexReader reader, MultiTermQuery query, TermCollector collector) throws IOException {
IndexReaderContext topReaderContext = reader.getContext();
- Comparator<BytesRef> lastTermComp = null;
for (AtomicReaderContext context : topReaderContext.leaves()) {
final Fields fields = context.reader().fields();
if (fields == null) {
@@ -67,11 +65,6 @@ abstract class TermCollectingRewrite<Q e
if (termsEnum == TermsEnum.EMPTY)
continue;
- // Check comparator compatibility:
- final Comparator<BytesRef> newTermComp = termsEnum.getComparator();
- if (lastTermComp != null && newTermComp != null && newTermComp != lastTermComp)
- throw new RuntimeException("term comparator should not change between segments: "+lastTermComp+" != "+newTermComp);
- lastTermComp = newTermComp;
collector.setReaderContext(topReaderContext, context);
collector.setNextEnum(termsEnum);
BytesRef bytes;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TermRangeTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TermRangeTermsEnum.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TermRangeTermsEnum.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TermRangeTermsEnum.java Thu Sep 19 20:57:09 2013
@@ -17,18 +17,13 @@ package org.apache.lucene.search;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.Comparator;
-
import org.apache.lucene.index.FilteredTermsEnum;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
/**
* Subclass of FilteredTermEnum for enumerating all terms that match the
- * specified range parameters.
- * <p>Term enumerations are always ordered by
- * {@link #getComparator}. Each term in the enumeration is
+ * specified range parameters. Each term in the enumeration is
* greater than all that precede it.</p>
*/
public class TermRangeTermsEnum extends FilteredTermsEnum {
@@ -37,7 +32,6 @@ public class TermRangeTermsEnum extends
final private boolean includeUpper;
final private BytesRef lowerBytesRef;
final private BytesRef upperBytesRef;
- private final Comparator<BytesRef> termComp;
/**
* Enumerates all terms greater/equal than <code>lowerTerm</code>
@@ -82,7 +76,6 @@ public class TermRangeTermsEnum extends
}
setInitialSeekTerm(lowerBytesRef);
- termComp = getComparator();
}
@Override
@@ -92,7 +85,7 @@ public class TermRangeTermsEnum extends
// Use this field's default sort ordering
if (upperBytesRef != null) {
- final int cmp = termComp.compare(upperBytesRef, term);
+ final int cmp = upperBytesRef.compareTo(term);
/*
* if beyond the upper term, or is exclusive and this is equal to
* the upper term, break out
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TopTermsRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TopTermsRewrite.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TopTermsRewrite.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TopTermsRewrite.java Thu Sep 19 20:57:09 2013
@@ -70,20 +70,18 @@ public abstract class TopTermsRewrite<Q
private final Map<BytesRef,ScoreTerm> visitedTerms = new HashMap<BytesRef,ScoreTerm>();
private TermsEnum termsEnum;
- private Comparator<BytesRef> termComp;
private BoostAttribute boostAtt;
private ScoreTerm st;
@Override
public void setNextEnum(TermsEnum termsEnum) {
this.termsEnum = termsEnum;
- this.termComp = termsEnum.getComparator();
assert compareToLastTerm(null);
// lazy init the initial ScoreTerm because comparator is not known on ctor:
if (st == null)
- st = new ScoreTerm(this.termComp, new TermContext(topReaderContext));
+ st = new ScoreTerm(new TermContext(topReaderContext));
boostAtt = termsEnum.attributes().addAttribute(BoostAttribute.class);
}
@@ -95,7 +93,7 @@ public abstract class TopTermsRewrite<Q
} else if (t == null) {
lastTerm = null;
} else {
- assert termsEnum.getComparator().compare(lastTerm, t) < 0: "lastTerm=" + lastTerm + " t=" + t;
+ assert lastTerm.compareTo(t) < 0: "lastTerm=" + lastTerm + " t=" + t;
lastTerm.copyBytes(t);
}
return true;
@@ -115,7 +113,7 @@ public abstract class TopTermsRewrite<Q
final ScoreTerm t = stQueue.peek();
if (boost < t.boost)
return true;
- if (boost == t.boost && termComp.compare(bytes, t.bytes) > 0)
+ if (boost == t.boost && bytes.compareTo(t.bytes) > 0)
return true;
}
ScoreTerm t = visitedTerms.get(bytes);
@@ -139,7 +137,7 @@ public abstract class TopTermsRewrite<Q
visitedTerms.remove(st.bytes);
st.termState.clear(); // reset the termstate!
} else {
- st = new ScoreTerm(termComp, new TermContext(topReaderContext));
+ st = new ScoreTerm(new TermContext(topReaderContext));
}
assert stQueue.size() <= maxSize : "the PQ size must be limited to maxSize";
// set maxBoostAtt with values to help FuzzyTermsEnum to optimize
@@ -185,26 +183,22 @@ public abstract class TopTermsRewrite<Q
new Comparator<ScoreTerm>() {
@Override
public int compare(ScoreTerm st1, ScoreTerm st2) {
- assert st1.termComp == st2.termComp :
- "term comparator should not change between segments";
- return st1.termComp.compare(st1.bytes, st2.bytes);
+ return st1.bytes.compareTo(st2.bytes);
}
};
static final class ScoreTerm implements Comparable<ScoreTerm> {
- public final Comparator<BytesRef> termComp;
public final BytesRef bytes = new BytesRef();
public float boost;
public final TermContext termState;
- public ScoreTerm(Comparator<BytesRef> termComp, TermContext termState) {
- this.termComp = termComp;
+ public ScoreTerm(TermContext termState) {
this.termState = termState;
}
@Override
public int compareTo(ScoreTerm other) {
if (this.boost == other.boost)
- return termComp.compare(other.bytes, this.bytes);
+ return other.bytes.compareTo(this.bytes);
else
return Float.compare(this.boost, other.boost);
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/package.html?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/package.html (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/package.html Thu Sep 19 20:57:09 2013
@@ -173,7 +173,7 @@ section for more notes on the process.
{@link org.apache.lucene.index.Term Term}
and an upper
{@link org.apache.lucene.index.Term Term}
- according to {@link org.apache.lucene.index.TermsEnum#getComparator TermsEnum.getComparator()}. It is not intended
+ according to {@link org.apache.lucene.util.BytesRef#compareTo BytesRef.compareTo()}. It is not intended
for numerical ranges; use {@link org.apache.lucene.search.NumericRangeQuery NumericRangeQuery} instead.
For example, one could find all documents
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/BytesRefIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/BytesRefIterator.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/BytesRefIterator.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/BytesRefIterator.java Thu Sep 19 20:57:09 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.util;
*/
import java.io.IOException;
-import java.util.Comparator;
/**
* A simple iterator interface for {@link BytesRef} iteration.
@@ -38,14 +37,6 @@ public interface BytesRefIterator {
*/
public BytesRef next() throws IOException;
- /**
- * Return the {@link BytesRef} Comparator used to sort terms provided by the
- * iterator. This may return null if there are no items or the iterator is not
- * sorted. Callers may invoke this method many times, so it's best to cache a
- * single instance & reuse it.
- */
- public Comparator<BytesRef> getComparator();
-
/** Singleton BytesRefIterator that iterates over 0 BytesRefs. */
public static final BytesRefIterator EMPTY = new BytesRefIterator() {
@@ -53,10 +44,5 @@ public interface BytesRefIterator {
public BytesRef next() {
return null;
}
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return null;
- }
};
}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java Thu Sep 19 20:57:09 2013
@@ -261,7 +261,6 @@ public class TestBlockPostingsFormat3 ex
* checks collection-level statistics on Terms
*/
public void assertTermsStatistics(Terms leftTerms, Terms rightTerms) throws Exception {
- assert leftTerms.getComparator() == rightTerms.getComparator();
if (leftTerms.getDocCount() != -1 && rightTerms.getDocCount() != -1) {
assertEquals(leftTerms.getDocCount(), rightTerms.getDocCount());
}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java Thu Sep 19 20:57:09 2013
@@ -25,11 +25,7 @@ import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
-import org.apache.lucene.codecs.PostingsConsumer;
-import org.apache.lucene.codecs.TermStats;
-import org.apache.lucene.codecs.TermsConsumer;
import org.apache.lucene.codecs.lucene40.Lucene40RWCodec;
import org.apache.lucene.codecs.lucene41.Lucene41RWCodec;
import org.apache.lucene.codecs.lucene42.Lucene42RWCodec;
@@ -48,11 +44,11 @@ import org.apache.lucene.search.PhraseQu
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util._TestUtil;
import org.junit.BeforeClass;
@@ -145,22 +141,6 @@ public class TestCodecs extends LuceneTe
public int compareTo(final FieldData other) {
return fieldInfo.name.compareTo(other.fieldInfo.name);
}
-
- public void write(final FieldsConsumer consumer) throws Throwable {
- Arrays.sort(terms);
- final TermsConsumer termsConsumer = consumer.addField(fieldInfo);
- long sumTotalTermCount = 0;
- long sumDF = 0;
- OpenBitSet visitedDocs = new OpenBitSet();
- for (final TermData term : terms) {
- for (int i = 0; i < term.docs.length; i++) {
- visitedDocs.set(term.docs[i]);
- }
- sumDF += term.docs.length;
- sumTotalTermCount += term.write(termsConsumer);
- }
- termsConsumer.finish(omitTF ? -1 : sumTotalTermCount, sumDF, (int) visitedDocs.cardinality());
- }
}
class PositionData {
@@ -191,30 +171,6 @@ public class TestCodecs extends LuceneTe
public int compareTo(final TermData o) {
return text.compareTo(o.text);
}
-
- public long write(final TermsConsumer termsConsumer) throws Throwable {
- final PostingsConsumer postingsConsumer = termsConsumer.startTerm(text);
- long totTF = 0;
- for(int i=0;i<docs.length;i++) {
- final int termDocFreq;
- if (field.omitTF) {
- termDocFreq = -1;
- } else {
- termDocFreq = positions[i].length;
- }
- postingsConsumer.startDoc(docs[i], termDocFreq);
- if (!field.omitTF) {
- totTF += positions[i].length;
- for(int j=0;j<positions[i].length;j++) {
- final PositionData pos = positions[i][j];
- postingsConsumer.addPosition(pos.pos, pos.payload, -1, -1);
- }
- }
- postingsConsumer.finishDoc();
- }
- termsConsumer.finishTerm(text, new TermStats(docs.length, field.omitTF ? -1 : totTF));
- return totTF;
- }
}
final private static String SEGMENT = "0";
@@ -588,18 +544,16 @@ public class TestCodecs extends LuceneTe
term = field.terms[upto];
if (random().nextInt(3) == 1) {
final DocsEnum docs;
- final DocsEnum docsAndFreqs;
final DocsAndPositionsEnum postings;
if (!field.omitTF) {
postings = termsEnum.docsAndPositions(null, null);
if (postings != null) {
- docs = docsAndFreqs = postings;
+ docs = postings;
} else {
- docs = docsAndFreqs = _TestUtil.docs(random(), termsEnum, null, null, DocsEnum.FLAG_FREQS);
+ docs = _TestUtil.docs(random(), termsEnum, null, null, DocsEnum.FLAG_FREQS);
}
} else {
postings = null;
- docsAndFreqs = null;
docs = _TestUtil.docs(random(), termsEnum, null, null, DocsEnum.FLAG_NONE);
}
assertNotNull(docs);
@@ -657,18 +611,250 @@ public class TestCodecs extends LuceneTe
}
}
+ private static class DataFields extends Fields {
+ private final FieldData[] fields;
+
+ public DataFields(FieldData[] fields) {
+ // already sorted:
+ this.fields = fields;
+ }
+
+ @Override
+ public Iterator<String> iterator() {
+ return new Iterator<String>() {
+ int upto = -1;
+
+ @Override
+ public boolean hasNext() {
+ return upto+1 < fields.length;
+ }
+
+ @Override
+ public String next() {
+ upto++;
+ return fields[upto].fieldInfo.name;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ @Override
+ public Terms terms(String field) {
+ // Slow linear search:
+ for(FieldData fieldData : fields) {
+ if (fieldData.fieldInfo.name.equals(field)) {
+ return new DataTerms(fieldData);
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public int size() {
+ return fields.length;
+ }
+ }
+
+ private static class DataTerms extends Terms {
+ final FieldData fieldData;
+
+ public DataTerms(FieldData fieldData) {
+ this.fieldData = fieldData;
+ }
+
+ @Override
+ public TermsEnum iterator(TermsEnum reuse) {
+ return new DataTermsEnum(fieldData);
+ }
+
+ @Override
+ public long size() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getSumTotalTermFreq() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getSumDocFreq() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getDocCount() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean hasOffsets() {
+ return fieldData.fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+ }
+
+ @Override
+ public boolean hasPositions() {
+ return fieldData.fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ }
+
+ @Override
+ public boolean hasPayloads() {
+ return fieldData.fieldInfo.hasPayloads();
+ }
+ }
+
+ private static class DataTermsEnum extends TermsEnum {
+ final FieldData fieldData;
+ private int upto = -1;
+
+ public DataTermsEnum(FieldData fieldData) {
+ this.fieldData = fieldData;
+ }
+
+ @Override
+ public BytesRef next() {
+ upto++;
+ if (upto == fieldData.terms.length) {
+ return null;
+ }
+
+ return term();
+ }
+
+ @Override
+ public BytesRef term() {
+ return fieldData.terms[upto].text;
+ }
+
+ @Override
+ public SeekStatus seekCeil(BytesRef text) {
+ // Stupid linear impl:
+ for(int i=0;i<fieldData.terms.length;i++) {
+ int cmp = fieldData.terms[i].text.compareTo(text);
+ if (cmp == 0) {
+ upto = i;
+ return SeekStatus.FOUND;
+ } else if (cmp > 0) {
+ upto = i;
+ return SeekStatus.NOT_FOUND;
+ }
+ }
+
+ return SeekStatus.END;
+ }
+
+ @Override
+ public void seekExact(long ord) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long ord() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int docFreq() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long totalTermFreq() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) {
+ assert liveDocs == null;
+ return new DataDocsAndPositionsEnum(fieldData.terms[upto]);
+ }
+
+ @Override
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) {
+ assert liveDocs == null;
+ return new DataDocsAndPositionsEnum(fieldData.terms[upto]);
+ }
+ }
+
+ private static class DataDocsAndPositionsEnum extends DocsAndPositionsEnum {
+ final TermData termData;
+ int docUpto = -1;
+ int posUpto;
+
+ public DataDocsAndPositionsEnum(TermData termData) {
+ this.termData = termData;
+ }
+
+ @Override
+ public long cost() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int nextDoc() {
+ docUpto++;
+ if (docUpto == termData.docs.length) {
+ return NO_MORE_DOCS;
+ }
+ posUpto = -1;
+ return docID();
+ }
+
+ @Override
+ public int docID() {
+ return termData.docs[docUpto];
+ }
+
+ @Override
+ public int advance(int target) {
+ // Slow linear impl:
+ nextDoc();
+ while (docID() < target) {
+ nextDoc();
+ }
+
+ return docID();
+ }
+
+ @Override
+ public int freq() {
+ return termData.positions[docUpto].length;
+ }
+
+ @Override
+ public int nextPosition() {
+ posUpto++;
+ return termData.positions[docUpto][posUpto].pos;
+ }
+
+ @Override
+ public BytesRef getPayload() {
+ return termData.positions[docUpto][posUpto].payload;
+ }
+
+ @Override
+ public int startOffset() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int endOffset() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
private void write(final FieldInfos fieldInfos, final Directory dir, final FieldData[] fields) throws Throwable {
final Codec codec = Codec.getDefault();
final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, false, codec, null, null);
final SegmentWriteState state = new SegmentWriteState(InfoStream.getDefault(), dir, si, fieldInfos, null, newIOContext(random()));
- final FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(state);
Arrays.sort(fields);
- for (final FieldData field : fields) {
- field.write(consumer);
- }
- consumer.close();
+ codec.postingsFormat().fieldsConsumer(state).write(new DataFields(fields));
}
public void testDocsOnlyFreq() throws Exception {
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java Thu Sep 19 20:57:09 2013
@@ -215,6 +215,7 @@ public class TestConcurrentMergeSchedule
IndexWriter writer = new IndexWriter(
directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).
+ // Force excessive merging:
setMaxBufferedDocs(2).
setMergePolicy(newLogMergePolicy(100))
);
@@ -249,7 +250,9 @@ public class TestConcurrentMergeSchedule
directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).
setOpenMode(OpenMode.APPEND).
- setMergePolicy(newLogMergePolicy(100))
+ setMergePolicy(newLogMergePolicy(100)).
+ // Force excessive merging:
+ setMaxBufferedDocs(2)
);
}
writer.close();
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java Thu Sep 19 20:57:09 2013
@@ -1094,7 +1094,6 @@ public void testFilesOpenClose() throws
File tempDir = _TestUtil.getTempDir("testIndexExistsOnNonExistentDirectory");
tempDir.delete();
Directory dir = newFSDirectory(tempDir);
- System.out.println("dir=" + dir);
assertFalse(DirectoryReader.indexExists(dir));
dir.close();
}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Thu Sep 19 20:57:09 2013
@@ -29,6 +29,7 @@ import java.util.Random;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.analysis.*;
+import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -537,21 +538,15 @@ public class TestIndexWriterExceptions e
public void eval(MockDirectoryWrapper dir) throws IOException {
if (doFail) {
StackTraceElement[] trace = new Exception().getStackTrace();
- boolean sawAppend = false;
boolean sawFlush = false;
for (int i = 0; i < trace.length; i++) {
- if (sawAppend && sawFlush) {
- break;
- }
- if (FreqProxTermsWriterPerField.class.getName().equals(trace[i].getClassName()) && "flush".equals(trace[i].getMethodName())) {
- sawAppend = true;
- }
if ("flush".equals(trace[i].getMethodName())) {
sawFlush = true;
+ break;
}
}
- if (sawAppend && sawFlush && count++ >= 30) {
+ if (sawFlush && count++ >= 30) {
doFail = false;
throw new IOException("now failing during flush");
}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java Thu Sep 19 20:57:09 2013
@@ -199,6 +199,9 @@ public class TestLongPostings extends Lu
}
if (random().nextInt(6) == 3) {
+ if (VERBOSE) {
+ System.out.println(" check positions");
+ }
final int freq = postings.freq();
assertTrue(freq >=1 && freq <= 4);
for(int pos=0;pos<freq;pos++) {
Modified: lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Thu Sep 19 20:57:09 2013
@@ -804,11 +804,6 @@ public class MemoryIndex {
}
@Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
- @Override
public long size() {
return info.terms.size();
}
@@ -966,11 +961,6 @@ public class MemoryIndex {
}
@Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
- @Override
public void seekExact(BytesRef term, TermState state) throws IOException {
assert state != null;
this.seekExact(((OrdTermState)state).ord);
Modified: lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeTermsEnum.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeTermsEnum.java (original)
+++ lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeTermsEnum.java Thu Sep 19 20:57:09 2013
@@ -17,7 +17,6 @@ package org.apache.lucene.sandbox.querie
* limitations under the License.
*/
-import java.io.IOException;
import java.text.Collator;
import org.apache.lucene.index.TermsEnum;
@@ -28,7 +27,7 @@ import org.apache.lucene.util.BytesRef;
* Subclass of FilteredTermEnum for enumerating all terms that match the
* specified range parameters.
* <p>Term enumerations are always ordered by
- * {@link #getComparator}. Each term in the enumeration is
+ * {@link BytesRef#compareTo}. Each term in the enumeration is
* greater than all that precede it.</p>
* @deprecated Index collation keys with CollationKeyAnalyzer or ICUCollationKeyAnalyzer instead.
* This class will be removed in Lucene 5.0
Modified: lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java (original)
+++ lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java Thu Sep 19 20:57:09 2013
@@ -38,7 +38,7 @@ import org.apache.lucene.util.UnicodeUti
* fuzzy terms enum method by calling FuzzyTermsEnum's getAutomatonEnum.
* </p>
* <p>Term enumerations are always ordered by
- * {@link #getComparator}. Each term in the enumeration is
+ * {@link BytesRef#compareTo}. Each term in the enumeration is
* greater than all that precede it.</p>
*
* @deprecated Use {@link FuzzyTermsEnum} instead.
Modified: lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeFilter.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeFilter.java (original)
+++ lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeFilter.java Thu Sep 19 20:57:09 2013
@@ -17,7 +17,9 @@ package org.apache.lucene.spatial.prefix
* limitations under the License.
*/
-import com.spatial4j.core.shape.Shape;
+import java.io.IOException;
+import java.util.Iterator;
+
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSet;
@@ -26,9 +28,7 @@ import org.apache.lucene.spatial.prefix.
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.StringHelper;
-
-import java.io.IOException;
-import java.util.Iterator;
+import com.spatial4j.core.shape.Shape;
/**
* Traverses a {@link SpatialPrefixTree} indexed field, using the template &
@@ -176,7 +176,7 @@ public abstract class AbstractVisitingPr
//Seek to curVNode's cell (or skip if termsEnum has moved beyond)
curVNodeTerm.bytes = curVNode.cell.getTokenBytes();
curVNodeTerm.length = curVNodeTerm.bytes.length;
- int compare = termsEnum.getComparator().compare(thisTerm, curVNodeTerm);
+ int compare = thisTerm.compareTo(curVNodeTerm);
if (compare > 0) {
// leap frog (termsEnum is beyond where we would otherwise seek)
assert ! context.reader().terms(fieldName).iterator(null).seekExact(curVNodeTerm) : "should be absent";
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java Thu Sep 19 20:57:09 2013
@@ -18,7 +18,6 @@
package org.apache.lucene.search.spell;
import java.io.IOException;
-import java.util.Comparator;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.TermsEnum;
@@ -99,14 +98,5 @@ public class HighFrequencyDictionary imp
}
return null;
}
-
- @Override
- public Comparator<BytesRef> getComparator() {
- if (termsEnum == null) {
- return null;
- } else {
- return termsEnum.getComparator();
- }
- }
}
}
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java Thu Sep 19 20:57:09 2013
@@ -17,9 +17,11 @@ package org.apache.lucene.search.spell;
* limitations under the License.
*/
-
-import java.util.Comparator;
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
@@ -96,11 +98,5 @@ public class PlainTextDictionary impleme
}
return result;
}
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return null;
- }
}
-
}
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqIterator.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqIterator.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqIterator.java Thu Sep 19 20:57:09 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.search.spell;
*/
import java.io.IOException;
-import java.util.Comparator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
@@ -55,10 +54,5 @@ public interface TermFreqIterator extend
public BytesRef next() throws IOException {
return wrapped.next();
}
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return wrapped.getComparator();
- }
}
}
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java Thu Sep 19 20:57:09 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.search.suggest
*/
import java.io.IOException;
-import java.util.Comparator;
import org.apache.lucene.search.spell.TermFreqIterator;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
@@ -37,11 +36,9 @@ public class BufferingTermFreqIteratorWr
/** buffered weights, parallel with {@link #entries} */
protected long[] freqs = new long[1];
private final BytesRef spare = new BytesRef();
- private final Comparator<BytesRef> comp;
/** Creates a new iterator, buffering entries from the specified iterator */
public BufferingTermFreqIteratorWrapper(TermFreqIterator source) throws IOException {
- this.comp = source.getComparator();
BytesRef spare;
int freqIndex = 0;
while((spare = source.next()) != null) {
@@ -67,11 +64,4 @@ public class BufferingTermFreqIteratorWr
}
return null;
}
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return comp;
- }
-
-
}
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BytesRefArray.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BytesRefArray.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BytesRefArray.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BytesRefArray.java Thu Sep 19 20:57:09 2013
@@ -187,11 +187,6 @@ public final class BytesRefArray {
}
return null;
}
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return comp;
- }
};
}
}
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java Thu Sep 19 20:57:09 2013
@@ -19,7 +19,6 @@ package org.apache.lucene.search.suggest
import java.io.*;
-import java.util.Comparator;
import org.apache.lucene.search.spell.Dictionary;
import org.apache.lucene.search.spell.TermFreqIterator;
@@ -99,11 +98,5 @@ public class FileDictionary implements D
return null;
}
}
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return null;
- }
}
-
}
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java Thu Sep 19 20:57:09 2013
@@ -65,11 +65,6 @@ public class SortedTermFreqIteratorWrapp
}
@Override
- public Comparator<BytesRef> getComparator() {
- return comparator;
- }
-
- @Override
public BytesRef next() throws IOException {
boolean success = false;
if (done) {
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java Thu Sep 19 20:57:09 2013
@@ -66,8 +66,7 @@ public class ExternalRefSorter implement
input = null;
}
- return new ByteSequenceIterator(new Sort.ByteSequencesReader(sorted),
- sort.getComparator());
+ return new ByteSequenceIterator(new Sort.ByteSequencesReader(sorted));
}
private void closeWriter() throws IOException {
@@ -96,12 +95,9 @@ public class ExternalRefSorter implement
class ByteSequenceIterator implements BytesRefIterator {
private final ByteSequencesReader reader;
private BytesRef scratch = new BytesRef();
- private final Comparator<BytesRef> comparator;
- public ByteSequenceIterator(ByteSequencesReader reader,
- Comparator<BytesRef> comparator) {
+ public ByteSequenceIterator(ByteSequencesReader reader) {
this.reader = reader;
- this.comparator = comparator;
}
@Override
@@ -128,11 +124,6 @@ public class ExternalRefSorter implement
}
}
}
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return comparator;
- }
}
@Override
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java Thu Sep 19 20:57:09 2013
@@ -28,7 +28,6 @@ import java.util.List;
import org.apache.lucene.search.spell.TermFreqIterator;
import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.search.suggest.Lookup;
-import org.apache.lucene.search.suggest.UnsortedTermFreqIteratorWrapper;
import org.apache.lucene.search.suggest.jaspell.JaspellTernarySearchTrie.TSTNode;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
@@ -57,11 +56,6 @@ public class JaspellLookup extends Looku
if (tfit instanceof TermFreqPayloadIterator) {
throw new IllegalArgumentException("this suggester doesn't support payloads");
}
- if (tfit.getComparator() != null) {
- // make sure it's unsorted
- // WTF - this could result in yet another sorted iteration....
- tfit = new UnsortedTermFreqIteratorWrapper(tfit);
- }
trie = new JaspellTernarySearchTrie();
trie.setMatchAlmostDiff(editDistance);
BytesRef spare;
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java Thu Sep 19 20:57:09 2013
@@ -56,11 +56,9 @@ public class TSTLookup extends Lookup {
throw new IllegalArgumentException("this suggester doesn't support payloads");
}
root = new TernaryTreeNode();
- // buffer first
- if (tfit.getComparator() != BytesRef.getUTF8SortedAsUTF16Comparator()) {
- // make sure it's sorted and the comparator uses UTF16 sort order
- tfit = new SortedTermFreqIteratorWrapper(tfit, BytesRef.getUTF8SortedAsUTF16Comparator());
- }
+
+ // make sure it's sorted and the comparator uses UTF16 sort order
+ tfit = new SortedTermFreqIteratorWrapper(tfit, BytesRef.getUTF8SortedAsUTF16Comparator());
ArrayList<String> tokens = new ArrayList<String>();
ArrayList<Number> vals = new ArrayList<Number>();
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java Thu Sep 19 20:57:09 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.search.suggest
*/
import java.util.Arrays;
-import java.util.Comparator;
import java.util.Iterator;
import org.apache.lucene.search.spell.TermFreqIterator;
@@ -58,9 +57,4 @@ public final class TermFreqArrayIterator
}
return null;
}
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return null;
- }
}
\ No newline at end of file
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java Thu Sep 19 20:57:09 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.search.suggest
*/
import java.util.Arrays;
-import java.util.Comparator;
import java.util.Iterator;
import org.apache.lucene.search.spell.TermFreqIterator;
@@ -64,9 +63,4 @@ public final class TermFreqPayloadArrayI
public BytesRef payload() {
return current.payload;
}
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return null;
- }
}
\ No newline at end of file
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestHighFrequencyDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestHighFrequencyDictionary.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestHighFrequencyDictionary.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestHighFrequencyDictionary.java Thu Sep 19 20:57:09 2013
@@ -36,7 +36,6 @@ public class TestHighFrequencyDictionary
IndexReader ir = DirectoryReader.open(dir);
Dictionary dictionary = new HighFrequencyDictionary(ir, "bogus", 0.1f);
BytesRefIterator tf = dictionary.getWordsIterator();
- assertNull(tf.getComparator());
assertNull(tf.next());
dir.close();
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java Thu Sep 19 20:57:09 2013
@@ -25,15 +25,20 @@ import org.apache.lucene.codecs.FieldsCo
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsConsumer;
import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.PushFieldsConsumer;
import org.apache.lucene.codecs.TermStats;
import org.apache.lucene.codecs.TermsConsumer;
import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
import org.apache.lucene.index.AssertingAtomicReader;
-import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.Fields;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.OpenBitSet;
@@ -49,7 +54,12 @@ public final class AssertingPostingsForm
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- return new AssertingFieldsConsumer(in.fieldsConsumer(state));
+ FieldsConsumer fieldsConsumer = in.fieldsConsumer(state);
+ if (fieldsConsumer instanceof PushFieldsConsumer) {
+ return new AssertingPushFieldsConsumer(state, (PushFieldsConsumer) fieldsConsumer);
+ } else {
+ return new AssertingFieldsConsumer(state, fieldsConsumer);
+ }
}
@Override
@@ -92,11 +102,12 @@ public final class AssertingPostingsForm
return in.ramBytesUsed();
}
}
-
- static class AssertingFieldsConsumer extends FieldsConsumer {
- private final FieldsConsumer in;
+
+ static class AssertingPushFieldsConsumer extends PushFieldsConsumer {
+ private final PushFieldsConsumer in;
- AssertingFieldsConsumer(FieldsConsumer in) {
+ AssertingPushFieldsConsumer(SegmentWriteState writeState, PushFieldsConsumer in) {
+ super(writeState);
this.in = in;
}
@@ -112,6 +123,113 @@ public final class AssertingPostingsForm
in.close();
}
}
+
+ static class AssertingFieldsConsumer extends FieldsConsumer {
+ private final FieldsConsumer in;
+ private final SegmentWriteState writeState;
+
+ AssertingFieldsConsumer(SegmentWriteState writeState, FieldsConsumer in) {
+ this.writeState = writeState;
+ this.in = in;
+ }
+
+ @Override
+ public void write(Fields fields) throws IOException {
+ in.write(fields);
+
+ // TODO: more asserts? can we somehow run a
+ // "limited" CheckIndex here??? Or ... can we improve
+ // AssertingFieldsProducer and us it also to wrap the
+ // incoming Fields here?
+
+ String lastField = null;
+ TermsEnum termsEnum = null;
+
+ for(String field : fields) {
+
+ FieldInfo fieldInfo = writeState.fieldInfos.fieldInfo(field);
+ assert fieldInfo != null;
+ assert lastField == null || lastField.compareTo(field) < 0;
+ lastField = field;
+
+ Terms terms = fields.terms(field);
+ assert terms != null;
+
+ termsEnum = terms.iterator(termsEnum);
+ BytesRef lastTerm = null;
+ DocsEnum docsEnum = null;
+ DocsAndPositionsEnum posEnum = null;
+
+ boolean hasFreqs = fieldInfo.getIndexOptions().compareTo(FieldInfo.IndexOptions.DOCS_AND_FREQS) >= 0;
+ boolean hasPositions = fieldInfo.getIndexOptions().compareTo(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ boolean hasOffsets = fieldInfo.getIndexOptions().compareTo(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+
+ assert hasPositions == terms.hasPositions();
+ assert hasOffsets == terms.hasOffsets();
+
+ while(true) {
+ BytesRef term = termsEnum.next();
+ if (term == null) {
+ break;
+ }
+ assert lastTerm == null || lastTerm.compareTo(term) < 0;
+ if (lastTerm == null) {
+ lastTerm = BytesRef.deepCopyOf(term);
+ } else {
+ lastTerm.copyBytes(term);
+ }
+
+ if (hasPositions == false) {
+ int flags = 0;
+ if (hasFreqs) {
+ flags = flags | DocsEnum.FLAG_FREQS;
+ }
+ docsEnum = termsEnum.docs(null, docsEnum, flags);
+ } else {
+ int flags = DocsAndPositionsEnum.FLAG_PAYLOADS;
+ if (hasOffsets) {
+ flags = flags | DocsAndPositionsEnum.FLAG_OFFSETS;
+ }
+ posEnum = termsEnum.docsAndPositions(null, posEnum, flags);
+ docsEnum = posEnum;
+ }
+
+ int lastDocID = -1;
+
+ while(true) {
+ int docID = docsEnum.nextDoc();
+ if (docID == DocsEnum.NO_MORE_DOCS) {
+ break;
+ }
+ assert docID > lastDocID;
+ lastDocID = docID;
+ if (hasFreqs) {
+ int freq = docsEnum.freq();
+ assert freq > 0;
+
+ if (hasPositions) {
+ int lastPos = -1;
+ int lastStartOffset = -1;
+ for(int i=0;i<freq;i++) {
+ int pos = posEnum.nextPosition();
+ assert pos > lastPos;
+ lastPos = pos;
+
+ if (hasOffsets) {
+ int startOffset = posEnum.startOffset();
+ int endOffset = posEnum.endOffset();
+ assert endOffset > startOffset;
+ assert startOffset >= lastStartOffset;
+ lastStartOffset = startOffset;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
static enum TermsConsumerState { INITIAL, START, FINISHED };
static class AssertingTermsConsumer extends TermsConsumer {
@@ -123,6 +241,7 @@ public final class AssertingPostingsForm
private long sumTotalTermFreq = 0;
private long sumDocFreq = 0;
private OpenBitSet visitedDocs = new OpenBitSet();
+ private static final Comparator<BytesRef> termComp = BytesRef.getUTF8SortedAsUnicodeComparator();
AssertingTermsConsumer(TermsConsumer in, FieldInfo fieldInfo) {
this.in = in;
@@ -133,7 +252,7 @@ public final class AssertingPostingsForm
public PostingsConsumer startTerm(BytesRef text) throws IOException {
assert state == TermsConsumerState.INITIAL || state == TermsConsumerState.START && lastPostingsConsumer.docFreq == 0;
state = TermsConsumerState.START;
- assert lastTerm == null || in.getComparator().compare(text, lastTerm) > 0;
+ assert lastTerm == null || termComp.compare(text, lastTerm) > 0;
lastTerm = BytesRef.deepCopyOf(text);
return lastPostingsConsumer = new AssertingPostingsConsumer(in.startTerm(text), fieldInfo, visitedDocs);
}
@@ -171,11 +290,6 @@ public final class AssertingPostingsForm
}
in.finish(sumTotalTermFreq, sumDocFreq, docCount);
}
-
- @Override
- public Comparator<BytesRef> getComparator() throws IOException {
- return in.getComparator();
- }
}
static enum PostingsConsumerState { INITIAL, START };
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java Thu Sep 19 20:57:09 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.codecs.asserti
*/
import java.io.IOException;
-import java.util.Comparator;
import org.apache.lucene.codecs.TermVectorsFormat;
import org.apache.lucene.codecs.TermVectorsReader;
@@ -181,11 +180,6 @@ public class AssertingTermVectorsFormat
}
@Override
- public Comparator<BytesRef> getComparator() throws IOException {
- return in.getComparator();
- }
-
- @Override
public void close() throws IOException {
in.close();
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java Thu Sep 19 20:57:09 2013
@@ -20,7 +20,6 @@ package org.apache.lucene.codecs.ramonly
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -34,6 +33,7 @@ import org.apache.lucene.codecs.FieldsCo
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsConsumer;
import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.PushFieldsConsumer;
import org.apache.lucene.codecs.TermStats;
import org.apache.lucene.codecs.TermsConsumer;
import org.apache.lucene.index.DocsAndPositionsEnum;
@@ -60,41 +60,6 @@ import org.apache.lucene.util.RamUsageEs
public final class RAMOnlyPostingsFormat extends PostingsFormat {
- // For fun, test that we can override how terms are
- // sorted, and basic things still work -- this comparator
- // sorts in reversed unicode code point order:
- private static final Comparator<BytesRef> reverseUnicodeComparator = new Comparator<BytesRef>() {
- @Override
- public int compare(BytesRef t1, BytesRef t2) {
- byte[] b1 = t1.bytes;
- byte[] b2 = t2.bytes;
- int b1Stop;
- int b1Upto = t1.offset;
- int b2Upto = t2.offset;
- if (t1.length < t2.length) {
- b1Stop = t1.offset + t1.length;
- } else {
- b1Stop = t1.offset + t2.length;
- }
- while(b1Upto < b1Stop) {
- final int bb1 = b1[b1Upto++] & 0xff;
- final int bb2 = b2[b2Upto++] & 0xff;
- if (bb1 != bb2) {
- //System.out.println("cmp 1=" + t1 + " 2=" + t2 + " return " + (bb2-bb1));
- return bb2 - bb1;
- }
- }
-
- // One is prefix of another, or they are equal
- return t2.length-t1.length;
- }
-
- @Override
- public boolean equals(Object other) {
- return this == other;
- }
- };
-
public RAMOnlyPostingsFormat() {
super("RAMOnly");
}
@@ -180,11 +145,6 @@ public final class RAMOnlyPostingsFormat
}
@Override
- public Comparator<BytesRef> getComparator() {
- return reverseUnicodeComparator;
- }
-
- @Override
public boolean hasOffsets() {
return info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
}
@@ -243,12 +203,13 @@ public final class RAMOnlyPostingsFormat
}
// Classes for writing to the postings state
- private static class RAMFieldsConsumer extends FieldsConsumer {
+ private static class RAMFieldsConsumer extends PushFieldsConsumer {
private final RAMPostings postings;
private final RAMTermsConsumer termsConsumer = new RAMTermsConsumer();
- public RAMFieldsConsumer(RAMPostings postings) {
+ public RAMFieldsConsumer(SegmentWriteState writeState, RAMPostings postings) {
+ super(writeState);
this.postings = postings;
}
@@ -286,12 +247,6 @@ public final class RAMOnlyPostingsFormat
return postingsWriter;
}
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
@Override
public void finishTerm(BytesRef text, TermStats stats) {
assert stats.docFreq > 0;
@@ -355,11 +310,6 @@ public final class RAMOnlyPostingsFormat
}
@Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
- @Override
public BytesRef next() {
if (it == null) {
if (current == null) {
@@ -586,7 +536,7 @@ public final class RAMOnlyPostingsFormat
}
final RAMPostings postings = new RAMPostings();
- final RAMFieldsConsumer consumer = new RAMFieldsConsumer(postings);
+ final RAMFieldsConsumer consumer = new RAMFieldsConsumer(writeState, postings);
synchronized(state) {
state.put(id, postings);