You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/06/16 22:34:04 UTC
svn commit: r1136656 [2/4] - in /lucene/dev/branches/flexscoring: ./
dev-tools/idea/.idea/ dev-tools/idea/lucene/ dev-tools/idea/lucene/contrib/
dev-tools/maven/ lucene/ lucene/backwards/ lucene/contrib/
lucene/contrib/misc/src/java/org/apache/lucene/i...
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java Thu Jun 16 20:33:59 2011
@@ -26,6 +26,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.PagedBytes;
// Simplest storage: stores fixed length byte[] per
@@ -45,11 +46,10 @@ class FixedStraightBytesImpl {
private int lastDocID = -1;
private byte[] oneRecord;
- protected Writer(Directory dir, String id) throws IOException {
+ public Writer(Directory dir, String id) throws IOException {
super(dir, id, CODEC_NAME, VERSION_CURRENT, false, null, null);
}
- // TODO - impl bulk copy here!
@Override
public void add(int docID, BytesRef bytes) throws IOException {
@@ -66,13 +66,6 @@ class FixedStraightBytesImpl {
datOut.writeBytes(bytes.bytes, bytes.offset, bytes.length);
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.lucene.index.values.Writer#merge(org.apache.lucene.index.values
- * .Writer.MergeState)
- */
@Override
protected void merge(MergeState state) throws IOException {
if (state.bits == null && state.reader instanceof Reader) {
@@ -96,8 +89,9 @@ class FixedStraightBytesImpl {
}
lastDocID += maxDocs - 1;
- } else
+ } else {
super.merge(state);
+ }
}
// Fills up to but not including this docID
@@ -126,7 +120,7 @@ class FixedStraightBytesImpl {
return oneRecord == null ? 0 : oneRecord.length;
}
}
-
+
public static class Reader extends BytesReaderBase {
private final int size;
private final int maxDoc;
@@ -139,19 +133,67 @@ class FixedStraightBytesImpl {
@Override
public Source load() throws IOException {
- return new Source(cloneData(), size, maxDoc);
+ return size == 1 ? new SingleByteSource(cloneData(), maxDoc) :
+ new StraightBytesSource(cloneData(), size, maxDoc);
}
@Override
public void close() throws IOException {
datIn.close();
}
+
+ // specialized version for single bytes
+ private static class SingleByteSource extends Source {
+ private final int maxDoc;
+ private final byte[] data;
- private static class Source extends BytesBaseSource {
+ public SingleByteSource(IndexInput datIn, int maxDoc) throws IOException {
+ this.maxDoc = maxDoc;
+ try {
+ data = new byte[maxDoc];
+ datIn.readBytes(data, 0, data.length, false);
+ } finally {
+ IOUtils.closeSafely(false, datIn);
+ }
+
+ }
+
+ @Override
+ public BytesRef getBytes(int docID, BytesRef bytesRef) {
+ bytesRef.length = 1;
+ bytesRef.bytes = data;
+ bytesRef.offset = docID;
+ return bytesRef;
+ }
+
+ @Override
+ public ValueType type() {
+ return ValueType.BYTES_FIXED_STRAIGHT;
+ }
+
+ @Override
+ public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
+ return new SourceEnum(attrSource, type(), this, maxDoc) {
+ @Override
+ public int advance(int target) throws IOException {
+ if (target >= numDocs) {
+ return pos = NO_MORE_DOCS;
+ }
+ bytesRef.length = 1;
+ bytesRef.bytes = data;
+ bytesRef.offset = target;
+ return pos = target;
+ }
+ };
+ }
+
+ }
+
+ private static class StraightBytesSource extends BytesBaseSource {
private final int size;
private final int maxDoc;
- public Source(IndexInput datIn, int size, int maxDoc)
+ public StraightBytesSource(IndexInput datIn, int size, int maxDoc)
throws IOException {
super(datIn, null, new PagedBytes(PAGED_BYTES_BITS), size * maxDoc);
this.size = size;
@@ -162,10 +204,10 @@ class FixedStraightBytesImpl {
public BytesRef getBytes(int docID, BytesRef bytesRef) {
return data.fillSlice(bytesRef, docID * size, size);
}
-
+
@Override
public int getValueCount() {
- throw new UnsupportedOperationException();
+ return maxDoc;
}
@Override
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldComparator.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldComparator.java Thu Jun 16 20:33:59 2011
@@ -96,7 +96,7 @@ import org.apache.lucene.util.packed.Pac
*
* @lucene.experimental
*/
-public abstract class FieldComparator {
+public abstract class FieldComparator<T> {
/**
* Compare hit at slot1 with hit at slot2.
@@ -176,13 +176,21 @@ public abstract class FieldComparator {
* Return the actual value in the slot.
*
* @param slot the value
- * @return value in this slot upgraded to Comparable
+ * @return value in this slot
*/
- public abstract Comparable<?> value(int slot);
+ public abstract T value(int slot);
-
+ /** Returns -1 if first is less than second. Default
+ * impl to assume the type implements Comparable and
+ * invoke .compareTo; be sure to override this method if
+ * your FieldComparator's type isn't a Comparable or
+ * if your values may sometimes be null */
+ @SuppressWarnings("unchecked")
+ public int compareValues(T first, T second) {
+ return ((Comparable<T>) first).compareTo(second);
+ }
- public static abstract class NumericComparator<T extends CachedArray> extends FieldComparator {
+ public static abstract class NumericComparator<T extends CachedArray, U extends Number> extends FieldComparator<U> {
protected final CachedArrayCreator<T> creator;
protected T cached;
protected final boolean checkMissing;
@@ -203,7 +211,7 @@ public abstract class FieldComparator {
/** Parses field's values as byte (using {@link
* FieldCache#getBytes} and sorts by ascending value */
- public static final class ByteComparator extends NumericComparator<ByteValues> {
+ public static final class ByteComparator extends NumericComparator<ByteValues,Byte> {
private byte[] docValues;
private final byte[] values;
private final byte missingValue;
@@ -252,7 +260,7 @@ public abstract class FieldComparator {
}
@Override
- public Comparable<?> value(int slot) {
+ public Byte value(int slot) {
return Byte.valueOf(values[slot]);
}
}
@@ -260,13 +268,12 @@ public abstract class FieldComparator {
/** Parses field's values as double (using {@link
* FieldCache#getDoubles} and sorts by ascending value */
- public static final class DoubleComparator extends NumericComparator<DoubleValues> {
+ public static final class DoubleComparator extends NumericComparator<DoubleValues,Double> {
private double[] docValues;
private final double[] values;
private final double missingValue;
private double bottom;
-
DoubleComparator(int numHits, DoubleValuesCreator creator, Double missingValue ) {
super( creator, missingValue != null );
values = new double[numHits];
@@ -324,13 +331,13 @@ public abstract class FieldComparator {
}
@Override
- public Comparable<?> value(int slot) {
+ public Double value(int slot) {
return Double.valueOf(values[slot]);
}
}
/** Uses float index values to sort by ascending value */
- public static final class FloatDocValuesComparator extends FieldComparator {
+ public static final class FloatDocValuesComparator extends FieldComparator<Double> {
private final double[] values;
private Source currentReaderValues;
private final String field;
@@ -386,14 +393,14 @@ public abstract class FieldComparator {
}
@Override
- public Comparable<Double> value(int slot) {
+ public Double value(int slot) {
return Double.valueOf(values[slot]);
}
}
/** Parses field's values as float (using {@link
* FieldCache#getFloats} and sorts by ascending value */
- public static final class FloatComparator extends NumericComparator<FloatValues> {
+ public static final class FloatComparator extends NumericComparator<FloatValues,Float> {
private float[] docValues;
private final float[] values;
private final float missingValue;
@@ -460,14 +467,14 @@ public abstract class FieldComparator {
}
@Override
- public Comparable<?> value(int slot) {
+ public Float value(int slot) {
return Float.valueOf(values[slot]);
}
}
/** Parses field's values as short (using {@link
* FieldCache#getShorts} and sorts by ascending value */
- public static final class ShortComparator extends NumericComparator<ShortValues> {
+ public static final class ShortComparator extends NumericComparator<ShortValues,Short> {
private short[] docValues;
private final short[] values;
private short bottom;
@@ -516,14 +523,14 @@ public abstract class FieldComparator {
}
@Override
- public Comparable<?> value(int slot) {
+ public Short value(int slot) {
return Short.valueOf(values[slot]);
}
}
/** Parses field's values as int (using {@link
* FieldCache#getInts} and sorts by ascending value */
- public static final class IntComparator extends NumericComparator<IntValues> {
+ public static final class IntComparator extends NumericComparator<IntValues,Integer> {
private int[] docValues;
private final int[] values;
private int bottom; // Value of bottom of queue
@@ -594,13 +601,13 @@ public abstract class FieldComparator {
}
@Override
- public Comparable<?> value(int slot) {
+ public Integer value(int slot) {
return Integer.valueOf(values[slot]);
}
}
/** Loads int index values and sorts by ascending value. */
- public static final class IntDocValuesComparator extends FieldComparator {
+ public static final class IntDocValuesComparator extends FieldComparator<Long> {
private final long[] values;
private Source currentReaderValues;
private final String field;
@@ -660,14 +667,14 @@ public abstract class FieldComparator {
}
@Override
- public Comparable<Long> value(int slot) {
+ public Long value(int slot) {
return Long.valueOf(values[slot]);
}
}
/** Parses field's values as long (using {@link
* FieldCache#getLongs} and sorts by ascending value */
- public static final class LongComparator extends NumericComparator<LongValues> {
+ public static final class LongComparator extends NumericComparator<LongValues,Long> {
private long[] docValues;
private final long[] values;
private long bottom;
@@ -735,7 +742,7 @@ public abstract class FieldComparator {
}
@Override
- public Comparable<?> value(int slot) {
+ public Long value(int slot) {
return Long.valueOf(values[slot]);
}
}
@@ -746,7 +753,7 @@ public abstract class FieldComparator {
* using {@link TopScoreDocCollector} directly (which {@link
* IndexSearcher#search} uses when no {@link Sort} is
* specified). */
- public static final class RelevanceComparator extends FieldComparator {
+ public static final class RelevanceComparator extends FieldComparator<Float> {
private final float[] scores;
private float bottom;
private Scorer scorer;
@@ -791,15 +798,21 @@ public abstract class FieldComparator {
}
@Override
- public Comparable<?> value(int slot) {
+ public Float value(int slot) {
return Float.valueOf(scores[slot]);
}
- }
-
+ // Override because we sort reverse of natural Float order:
+ @Override
+ public int compareValues(Float first, Float second) {
+ // Reversed intentionally because relevance by default
+ // sorts descending:
+ return second.compareTo(first);
+ }
+ }
/** Sorts by ascending docID */
- public static final class DocComparator extends FieldComparator {
+ public static final class DocComparator extends FieldComparator<Integer> {
private final int[] docIDs;
private int docBase;
private int bottom;
@@ -840,7 +853,7 @@ public abstract class FieldComparator {
}
@Override
- public Comparable<?> value(int slot) {
+ public Integer value(int slot) {
return Integer.valueOf(docIDs[slot]);
}
}
@@ -854,7 +867,7 @@ public abstract class FieldComparator {
* to large results, this comparator will be much faster
* than {@link TermValComparator}. For very small
* result sets it may be slower. */
- public static final class TermOrdValComparator extends FieldComparator {
+ public static final class TermOrdValComparator extends FieldComparator<BytesRef> {
/** @lucene.internal */
final int[] ords;
/** @lucene.internal */
@@ -920,7 +933,7 @@ public abstract class FieldComparator {
* the underlying array access when looking up doc->ord
* @lucene.internal
*/
- abstract class PerSegmentComparator extends FieldComparator {
+ abstract class PerSegmentComparator extends FieldComparator<BytesRef> {
@Override
public FieldComparator setNextReader(AtomicReaderContext context) throws IOException {
@@ -938,9 +951,22 @@ public abstract class FieldComparator {
}
@Override
- public Comparable<?> value(int slot) {
+ public BytesRef value(int slot) {
return TermOrdValComparator.this.value(slot);
}
+
+ @Override
+ public int compareValues(BytesRef val1, BytesRef val2) {
+ if (val1 == null) {
+ if (val2 == null) {
+ return 0;
+ }
+ return -1;
+ } else if (val2 == null) {
+ return 1;
+ }
+ return val1.compareTo(val2);
+ }
}
// Used per-segment when bit width of doc->ord is 8:
@@ -1244,7 +1270,7 @@ public abstract class FieldComparator {
}
@Override
- public Comparable<?> value(int slot) {
+ public BytesRef value(int slot) {
return values[slot];
}
}
@@ -1253,7 +1279,7 @@ public abstract class FieldComparator {
* comparisons are done using BytesRef.compareTo, which is
* slow for medium to large result sets but possibly
* very fast for very small results sets. */
- public static final class TermValComparator extends FieldComparator {
+ public static final class TermValComparator extends FieldComparator<BytesRef> {
private BytesRef[] values;
private DocTerms docTerms;
@@ -1316,9 +1342,22 @@ public abstract class FieldComparator {
}
@Override
- public Comparable<?> value(int slot) {
+ public BytesRef value(int slot) {
return values[slot];
}
+
+ @Override
+ public int compareValues(BytesRef val1, BytesRef val2) {
+ if (val1 == null) {
+ if (val2 == null) {
+ return 0;
+ }
+ return -1;
+ } else if (val2 == null) {
+ return 1;
+ }
+ return val1.compareTo(val2);
+ }
}
final protected static int binarySearch(BytesRef br, DocTermsIndex a, BytesRef key) {
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldDoc.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldDoc.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldDoc.java Thu Jun 16 20:33:59 2011
@@ -40,24 +40,31 @@ public class FieldDoc extends ScoreDoc {
/** Expert: The values which are used to sort the referenced document.
* The order of these will match the original sort criteria given by a
- * Sort object. Each Object will be either an Integer, Float or String,
- * depending on the type of values in the terms of the original field.
+ * Sort object. Each Object will have been returned from
+ * the <code>value</code> method corresponding
+ * FieldComparator used to sort this field.
* @see Sort
* @see IndexSearcher#search(Query,Filter,int,Sort)
*/
- public Comparable[] fields;
+ public Object[] fields;
/** Expert: Creates one of these objects with empty sort information. */
- public FieldDoc (int doc, float score) {
+ public FieldDoc(int doc, float score) {
super (doc, score);
}
/** Expert: Creates one of these objects with the given sort information. */
- public FieldDoc (int doc, float score, Comparable[] fields) {
+ public FieldDoc(int doc, float score, Object[] fields) {
super (doc, score);
this.fields = fields;
}
+ /** Expert: Creates one of these objects with the given sort information. */
+ public FieldDoc(int doc, float score, Object[] fields, int shardIndex) {
+ super (doc, score, shardIndex);
+ this.fields = fields;
+ }
+
// A convenience method for debugging.
@Override
public String toString() {
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldValueHitQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldValueHitQueue.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldValueHitQueue.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldValueHitQueue.java Thu Jun 16 20:33:59 2011
@@ -200,7 +200,7 @@ public abstract class FieldValueHitQueue
*/
FieldDoc fillFields(final Entry entry) {
final int n = comparators.length;
- final Comparable<?>[] fields = new Comparable[n];
+ final Object[] fields = new Object[n];
for (int i = 0; i < n; ++i) {
fields[i] = comparators[i].value(entry.slot);
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/IndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/IndexSearcher.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/IndexSearcher.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/IndexSearcher.java Thu Jun 16 20:33:59 2011
@@ -289,7 +289,7 @@ public class IndexSearcher {
*/
public TopDocs search(Query query, Filter filter, int n)
throws IOException {
- return search(createWeight(query), filter, n);
+ return search(createNormalizedWeight(query), filter, n);
}
/** Lower-level search API.
@@ -310,7 +310,7 @@ public class IndexSearcher {
*/
public void search(Query query, Filter filter, Collector results)
throws IOException {
- search(leafContexts, createWeight(query), filter, results);
+ search(leafContexts, createNormalizedWeight(query), filter, results);
}
/** Lower-level search API.
@@ -328,7 +328,7 @@ public class IndexSearcher {
*/
public void search(Query query, Collector results)
throws IOException {
- search(leafContexts, createWeight(query), null, results);
+ search(leafContexts, createNormalizedWeight(query), null, results);
}
/** Search implementation with arbitrary sorting. Finds
@@ -344,7 +344,7 @@ public class IndexSearcher {
*/
public TopFieldDocs search(Query query, Filter filter, int n,
Sort sort) throws IOException {
- return search(createWeight(query), filter, n, sort);
+ return search(createNormalizedWeight(query), filter, n, sort);
}
/**
@@ -357,7 +357,7 @@ public class IndexSearcher {
*/
public TopFieldDocs search(Query query, int n,
Sort sort) throws IOException {
- return search(createWeight(query), null, n, sort);
+ return search(createNormalizedWeight(query), null, n, sort);
}
/** Expert: Low-level search implementation. Finds the top <code>n</code>
@@ -443,7 +443,7 @@ public class IndexSearcher {
* Collector)}.</p>
*/
protected TopFieldDocs search(Weight weight, Filter filter, int nDocs,
- Sort sort, boolean fillFields)
+ Sort sort, boolean fillFields)
throws IOException {
if (sort == null) throw new NullPointerException();
@@ -623,7 +623,7 @@ public class IndexSearcher {
* entire index.
*/
public Explanation explain(Query query, int doc) throws IOException {
- return explain(createWeight(query), doc);
+ return explain(createNormalizedWeight(query), doc);
}
/** Expert: low-level implementation method
@@ -665,13 +665,23 @@ public class IndexSearcher {
}
/**
- * creates a weight for <code>query</code>
- * @return new weight
- */
- protected Weight createWeight(Query query) throws IOException {
- return query.weight(this);
+ * Creates a normalized weight for a top-level {@link Query}.
+ * The query is rewritten by this method and {@link Query#createWeight} called,
+ * afterwards the {@link Weight} is normalized. The returned {@code Weight}
+ * can then directly be used to get a {@link Scorer}.
+ * @lucene.internal
+ */
+ public Weight createNormalizedWeight(Query query) throws IOException {
+ query = rewrite(query);
+ Weight weight = query.createWeight(this);
+ float v = weight.getValueForNormalization();
+ float norm = getSimilarityProvider().queryNorm(v);
+ if (Float.isInfinite(norm) || Float.isNaN(norm))
+ norm = 1.0f;
+ weight.normalize(norm, 1.0f);
+ return weight;
}
-
+
/**
* Returns this searchers the top-level {@link ReaderContext}.
* @see IndexReader#getTopReaderContext()
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/Query.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/Query.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/Query.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/Query.java Thu Jun 16 20:33:59 2011
@@ -91,21 +91,6 @@ public abstract class Query implements C
throw new UnsupportedOperationException();
}
- /**
- * Expert: Constructs and initializes a Weight for a top-level query.
- */
- public Weight weight(IndexSearcher searcher) throws IOException {
- Query query = searcher.rewrite(this);
- Weight weight = query.createWeight(searcher);
- float v = weight.getValueForNormalization();
- float norm = searcher.getSimilarityProvider().queryNorm(v);
- if (Float.isInfinite(norm) || Float.isNaN(norm))
- norm = 1.0f;
- weight.normalize(norm, 1.0f);
- return weight;
- }
-
-
/** Expert: called to re-write queries into primitive queries. For example,
* a PrefixQuery will be rewritten into a BooleanQuery that consists
* of TermQuerys.
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java Thu Jun 16 20:33:59 2011
@@ -52,7 +52,7 @@ public class QueryWrapperFilter extends
// get a private context that is used to rewrite, createWeight and score eventually
assert context.reader.getTopReaderContext().isAtomic;
final AtomicReaderContext privateContext = (AtomicReaderContext) context.reader.getTopReaderContext();
- final Weight weight = query.weight(new IndexSearcher(privateContext));
+ final Weight weight = new IndexSearcher(privateContext).createNormalizedWeight(query);
return new DocIdSet() {
@Override
public DocIdSetIterator iterator() throws IOException {
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/ScoreDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/ScoreDoc.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/ScoreDoc.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/ScoreDoc.java Thu Jun 16 20:33:59 2011
@@ -17,21 +17,30 @@ package org.apache.lucene.search;
* limitations under the License.
*/
-/** Expert: Returned by low-level search implementations.
- * @see TopDocs */
+/** Holds one hit in {@link TopDocs}. */
+
public class ScoreDoc {
- /** Expert: The score of this document for the query. */
+
+ /** The score of this document for the query. */
public float score;
- /** Expert: A hit document's number.
- * @see IndexSearcher#doc(int)
- */
+ /** A hit document's number.
+ * @see IndexSearcher#doc(int) */
public int doc;
- /** Expert: Constructs a ScoreDoc. */
+ /** Only set by {@link TopDocs#merge} */
+ public int shardIndex;
+
+ /** Constructs a ScoreDoc. */
public ScoreDoc(int doc, float score) {
+ this(doc, score, -1);
+ }
+
+ /** Constructs a ScoreDoc. */
+ public ScoreDoc(int doc, float score, int shardIndex) {
this.doc = doc;
this.score = score;
+ this.shardIndex = shardIndex;
}
// A convenience method for debugging.
@@ -39,5 +48,4 @@ public class ScoreDoc {
public String toString() {
return "doc=" + doc + " score=" + score;
}
-
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/SortField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/SortField.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/SortField.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/SortField.java Thu Jun 16 20:33:59 2011
@@ -91,10 +91,10 @@ public class SortField {
public static final int BYTES = 12;
/** Represents sorting by document score (relevance). */
- public static final SortField FIELD_SCORE = new SortField (null, SCORE);
+ public static final SortField FIELD_SCORE = new SortField(null, SCORE);
/** Represents sorting by document number (index order). */
- public static final SortField FIELD_DOC = new SortField (null, DOC);
+ public static final SortField FIELD_DOC = new SortField(null, DOC);
private String field;
private int type; // defaults to determining type dynamically
@@ -111,7 +111,7 @@ public class SortField {
* <code>type</code> is SCORE or DOC.
* @param type Type of values in the terms.
*/
- public SortField (String field, int type) {
+ public SortField(String field, int type) {
initFieldType(field, type);
}
@@ -122,7 +122,7 @@ public class SortField {
* @param type Type of values in the terms.
* @param reverse True if natural order should be reversed.
*/
- public SortField (String field, int type, boolean reverse) {
+ public SortField(String field, int type, boolean reverse) {
initFieldType(field, type);
this.reverse = reverse;
}
@@ -140,7 +140,7 @@ public class SortField {
* @deprecated (4.0) use EntryCreator version
*/
@Deprecated
- public SortField (String field, FieldCache.Parser parser) {
+ public SortField(String field, FieldCache.Parser parser) {
this(field, parser, false);
}
@@ -158,7 +158,7 @@ public class SortField {
* @deprecated (4.0) use EntryCreator version
*/
@Deprecated
- public SortField (String field, FieldCache.Parser parser, boolean reverse) {
+ public SortField(String field, FieldCache.Parser parser, boolean reverse) {
if (field == null) {
throw new IllegalArgumentException("field can only be null when type is SCORE or DOC");
}
@@ -225,7 +225,7 @@ public class SortField {
* @param field Name of field to sort by; cannot be <code>null</code>.
* @param comparator Returns a comparator for sorting hits.
*/
- public SortField (String field, FieldComparatorSource comparator) {
+ public SortField(String field, FieldComparatorSource comparator) {
initFieldType(field, CUSTOM);
this.comparatorSource = comparator;
}
@@ -235,7 +235,7 @@ public class SortField {
* @param comparator Returns a comparator for sorting hits.
* @param reverse True if natural order should be reversed.
*/
- public SortField (String field, FieldComparatorSource comparator, boolean reverse) {
+ public SortField(String field, FieldComparatorSource comparator, boolean reverse) {
initFieldType(field, CUSTOM);
this.reverse = reverse;
this.comparatorSource = comparator;
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/TermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/TermQuery.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/TermQuery.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/TermQuery.java Thu Jun 16 20:33:59 2011
@@ -73,7 +73,7 @@ public class TermQuery extends Query {
public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
final String field = term.field();
final IndexReader reader = context.reader;
- assert termStates.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight is not the same as the current reader's top-reader";
+ assert termStates.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight (" + termStates.topReaderContext + ") is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
final TermState state = termStates
.get(context.ord);
if (state == null) { // term is not present in that reader
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/TopDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/TopDocs.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/TopDocs.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/TopDocs.java Thu Jun 16 20:33:59 2011
@@ -17,15 +17,21 @@ package org.apache.lucene.search;
* limitations under the License.
*/
+import java.io.IOException;
+
+import org.apache.lucene.util.PriorityQueue;
+
/** Represents hits returned by {@link
* IndexSearcher#search(Query,Filter,int)} and {@link
* IndexSearcher#search(Query,int)}. */
public class TopDocs {
- /** The total number of hits for the query.
- */
+
+ /** The total number of hits for the query. */
public int totalHits;
+
/** The top hits for the query. */
public ScoreDoc[] scoreDocs;
+
/** Stores the maximum score value encountered, needed for normalizing. */
private float maxScore;
@@ -34,12 +40,12 @@ public class TopDocs {
* scores are not tracked, this returns {@link Float#NaN}.
*/
public float getMaxScore() {
- return maxScore;
+ return maxScore;
}
/** Sets the maximum score value encountered. */
public void setMaxScore(float maxScore) {
- this.maxScore=maxScore;
+ this.maxScore=maxScore;
}
/** Constructs a TopDocs with a default maxScore=Float.NaN. */
@@ -52,4 +58,199 @@ public class TopDocs {
this.scoreDocs = scoreDocs;
this.maxScore = maxScore;
}
+
+ // Refers to one hit:
+ private static class ShardRef {
+ // Which shard (index into shardHits[]):
+ final int shardIndex;
+
+ // Which hit within the shard:
+ int hitIndex;
+
+ public ShardRef(int shardIndex) {
+ this.shardIndex = shardIndex;
+ }
+
+ @Override
+ public String toString() {
+ return "ShardRef(shardIndex=" + shardIndex + " hitIndex=" + hitIndex + ")";
+ }
+ };
+
+ // Specialized MergeSortQueue that just merges by
+ // relevance score, descending:
+ private static class ScoreMergeSortQueue extends PriorityQueue<ShardRef> {
+ final ScoreDoc[][] shardHits;
+
+ public ScoreMergeSortQueue(TopDocs[] shardHits) {
+ super(shardHits.length);
+ this.shardHits = new ScoreDoc[shardHits.length][];
+ for(int shardIDX=0;shardIDX<shardHits.length;shardIDX++) {
+ this.shardHits[shardIDX] = shardHits[shardIDX].scoreDocs;
+ }
+ }
+
+ // Returns true if first is < second
+ public boolean lessThan(ShardRef first, ShardRef second) {
+ assert first != second;
+ final float firstScore = shardHits[first.shardIndex][first.hitIndex].score;
+ final float secondScore = shardHits[second.shardIndex][second.hitIndex].score;
+
+ if (firstScore < secondScore) {
+ return false;
+ } else if (firstScore > secondScore) {
+ return true;
+ } else {
+ // Tie break: earlier shard wins
+ if (first.shardIndex < second.shardIndex) {
+ return true;
+ } else if (first.shardIndex > second.shardIndex) {
+ return false;
+ } else {
+ // Tie break in same shard: resolve however the
+ // shard had resolved it:
+ assert first.hitIndex != second.hitIndex;
+ return first.hitIndex < second.hitIndex;
+ }
+ }
+ }
+ }
+
+ private static class MergeSortQueue extends PriorityQueue<ShardRef> {
+ // These are really FieldDoc instances:
+ final ScoreDoc[][] shardHits;
+ final FieldComparator[] comparators;
+ final int[] reverseMul;
+
+ public MergeSortQueue(Sort sort, TopDocs[] shardHits) throws IOException {
+ super(shardHits.length);
+ this.shardHits = new ScoreDoc[shardHits.length][];
+ for(int shardIDX=0;shardIDX<shardHits.length;shardIDX++) {
+ final ScoreDoc[] shard = shardHits[shardIDX].scoreDocs;
+ //System.out.println(" init shardIdx=" + shardIDX + " hits=" + shard);
+ if (shard != null) {
+ this.shardHits[shardIDX] = shard;
+ // Fail gracefully if API is misused:
+ for(int hitIDX=0;hitIDX<shard.length;hitIDX++) {
+ final ScoreDoc sd = shard[hitIDX];
+ if (!(sd instanceof FieldDoc)) {
+ throw new IllegalArgumentException("shard " + shardIDX + " was not sorted by the provided Sort (expected FieldDoc but got ScoreDoc)");
+ }
+ final FieldDoc fd = (FieldDoc) sd;
+ if (fd.fields == null) {
+ throw new IllegalArgumentException("shard " + shardIDX + " did not set sort field values (FieldDoc.fields is null); you must pass fillFields=true to IndexSearcher.search on each shard");
+ }
+ }
+ }
+ }
+
+ final SortField[] sortFields = sort.getSort();
+ comparators = new FieldComparator[sortFields.length];
+ reverseMul = new int[sortFields.length];
+ for(int compIDX=0;compIDX<sortFields.length;compIDX++) {
+ final SortField sortField = sortFields[compIDX];
+ comparators[compIDX] = sortField.getComparator(1, compIDX);
+ reverseMul[compIDX] = sortField.getReverse() ? -1 : 1;
+ }
+ }
+
+ // Returns true if first is < second
+ @SuppressWarnings("unchecked")
+ public boolean lessThan(ShardRef first, ShardRef second) {
+ assert first != second;
+ final FieldDoc firstFD = (FieldDoc) shardHits[first.shardIndex][first.hitIndex];
+ final FieldDoc secondFD = (FieldDoc) shardHits[second.shardIndex][second.hitIndex];
+ //System.out.println(" lessThan:\n first=" + first + " doc=" + firstFD.doc + " score=" + firstFD.score + "\n second=" + second + " doc=" + secondFD.doc + " score=" + secondFD.score);
+
+ for(int compIDX=0;compIDX<comparators.length;compIDX++) {
+ final FieldComparator comp = comparators[compIDX];
+ //System.out.println(" cmp idx=" + compIDX + " cmp1=" + firstFD.fields[compIDX] + " cmp2=" + secondFD.fields[compIDX] + " reverse=" + reverseMul[compIDX]);
+
+ final int cmp = reverseMul[compIDX] * comp.compareValues(firstFD.fields[compIDX], secondFD.fields[compIDX]);
+
+ if (cmp != 0) {
+ //System.out.println(" return " + (cmp < 0));
+ return cmp < 0;
+ }
+ }
+
+ // Tie break: earlier shard wins
+ if (first.shardIndex < second.shardIndex) {
+ //System.out.println(" return tb true");
+ return true;
+ } else if (first.shardIndex > second.shardIndex) {
+ //System.out.println(" return tb false");
+ return false;
+ } else {
+ // Tie break in same shard: resolve however the
+ // shard had resolved it:
+ //System.out.println(" return tb " + (first.hitIndex < second.hitIndex));
+ assert first.hitIndex != second.hitIndex;
+ return first.hitIndex < second.hitIndex;
+ }
+ }
+ }
+
+ /** Returns a new TopDocs, containing topN results across
+ * the provided TopDocs, sorting by the specified {@link
+ * Sort}. Each of the TopDocs must have been sorted by
+ * the same Sort, and sort field values must have been
+ * filled (ie, <code>fillFields=true</code> must be
+ * passed to {@link
+ * TopFieldCollector#create}.
+ *
+ * <p>Pass sort=null to merge sort by score descending.
+ *
+ * @lucene.experimental */
+ public static TopDocs merge(Sort sort, int topN, TopDocs[] shardHits) throws IOException {
+
+ final PriorityQueue<ShardRef> queue;
+ if (sort == null) {
+ queue = new ScoreMergeSortQueue(shardHits);
+ } else {
+ queue = new MergeSortQueue(sort, shardHits);
+ }
+
+ int totalHitCount = 0;
+ float maxScore = Float.MIN_VALUE;
+ for(int shardIDX=0;shardIDX<shardHits.length;shardIDX++) {
+ final TopDocs shard = shardHits[shardIDX];
+ if (shard.scoreDocs != null && shard.scoreDocs.length > 0) {
+ totalHitCount += shard.totalHits;
+ queue.add(new ShardRef(shardIDX));
+ maxScore = Math.max(maxScore, shard.getMaxScore());
+ //System.out.println(" maxScore now " + maxScore + " vs " + shard.getMaxScore());
+ }
+ }
+
+ final ScoreDoc[] hits = new ScoreDoc[Math.min(topN, totalHitCount)];
+
+ int hitUpto = 0;
+ while(hitUpto < hits.length) {
+ assert queue.size() > 0;
+ ShardRef ref = queue.pop();
+ final ScoreDoc hit = shardHits[ref.shardIndex].scoreDocs[ref.hitIndex++];
+ if (sort == null) {
+ hits[hitUpto] = new ScoreDoc(hit.doc, hit.score, ref.shardIndex);
+ } else {
+ hits[hitUpto] = new FieldDoc(hit.doc, hit.score, ((FieldDoc) hit).fields, ref.shardIndex);
+ }
+
+ //System.out.println(" hitUpto=" + hitUpto);
+ //System.out.println(" doc=" + hits[hitUpto].doc + " score=" + hits[hitUpto].score);
+
+ hitUpto++;
+
+ if (ref.hitIndex < shardHits[ref.shardIndex].scoreDocs.length) {
+ // Not done with this these TopDocs yet:
+ queue.add(ref);
+ }
+ }
+
+ if (sort == null) {
+ return new TopDocs(totalHitCount, hits, maxScore);
+ } else {
+ return new TopFieldDocs(totalHitCount, hits, sort.getSort(), maxScore);
+ }
+ }
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/store/FSDirectory.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/store/FSDirectory.java Thu Jun 16 20:33:59 2011
@@ -173,7 +173,7 @@ public abstract class FSDirectory extend
/** Just like {@link #open(File)}, but allows you to
* also specify a custom {@link LockFactory}. */
public static FSDirectory open(File path, LockFactory lockFactory) throws IOException {
- if ((Constants.WINDOWS || Constants.SUN_OS)
+ if ((Constants.WINDOWS || Constants.SUN_OS || Constants.LINUX)
&& Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
return new MMapDirectory(path, lockFactory);
} else if (Constants.WINDOWS) {
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/store/MMapDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/store/MMapDirectory.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/store/MMapDirectory.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/store/MMapDirectory.java Thu Jun 16 20:33:59 2011
@@ -79,8 +79,8 @@ import org.apache.lucene.util.Constants;
*/
public class MMapDirectory extends FSDirectory {
private boolean useUnmapHack = UNMAP_SUPPORTED;
- public static final int DEFAULT_MAX_BUFF = Constants.JRE_IS_64BIT ? Integer.MAX_VALUE : (256 * 1024 * 1024);
- private int maxBBuf = DEFAULT_MAX_BUFF;
+ public static final int DEFAULT_MAX_BUFF = Constants.JRE_IS_64BIT ? (1 << 30) : (1 << 28);
+ private int chunkSizePower;
/** Create a new MMapDirectory for the named location.
*
@@ -91,6 +91,7 @@ public class MMapDirectory extends FSDir
*/
public MMapDirectory(File path, LockFactory lockFactory) throws IOException {
super(path, lockFactory);
+ setMaxChunkSize(DEFAULT_MAX_BUFF);
}
/** Create a new MMapDirectory for the named location and {@link NativeFSLockFactory}.
@@ -100,6 +101,7 @@ public class MMapDirectory extends FSDir
*/
public MMapDirectory(File path) throws IOException {
super(path, null);
+ setMaxChunkSize(DEFAULT_MAX_BUFF);
}
/**
@@ -180,23 +182,28 @@ public class MMapDirectory extends FSDir
* Especially on 32 bit platform, the address space can be very fragmented,
* so large index files cannot be mapped.
* Using a lower chunk size makes the directory implementation a little
- * bit slower (as the correct chunk must be resolved on each seek)
+ * bit slower (as the correct chunk may be resolved on lots of seeks)
* but the chance is higher that mmap does not fail. On 64 bit
- * Java platforms, this parameter should always be {@link Integer#MAX_VALUE},
+ * Java platforms, this parameter should always be {@code 1 << 30},
* as the address space is big enough.
+ * <b>Please note:</b> This method always rounds down the chunk size
+ * to a power of 2.
*/
- public void setMaxChunkSize(final int maxBBuf) {
- if (maxBBuf<=0)
+ public final void setMaxChunkSize(final int maxChunkSize) {
+ if (maxChunkSize <= 0)
throw new IllegalArgumentException("Maximum chunk size for mmap must be >0");
- this.maxBBuf=maxBBuf;
+ //System.out.println("Requested chunk size: "+maxChunkSize);
+ this.chunkSizePower = 31 - Integer.numberOfLeadingZeros(maxChunkSize);
+ assert this.chunkSizePower >= 0 && this.chunkSizePower <= 30;
+ //System.out.println("Got chunk size: "+getMaxChunkSize());
}
/**
* Returns the current mmap chunk size.
* @see #setMaxChunkSize
*/
- public int getMaxChunkSize() {
- return maxBBuf;
+ public final int getMaxChunkSize() {
+ return 1 << chunkSizePower;
}
/** Creates an IndexInput for the file with the given name. */
@@ -206,152 +213,55 @@ public class MMapDirectory extends FSDir
File f = new File(getDirectory(), name);
RandomAccessFile raf = new RandomAccessFile(f, "r");
try {
- return (raf.length() <= maxBBuf)
- ? (IndexInput) new MMapIndexInput(raf)
- : (IndexInput) new MultiMMapIndexInput(raf, maxBBuf);
+ return new MMapIndexInput(raf, chunkSizePower);
} finally {
raf.close();
}
}
- private class MMapIndexInput extends IndexInput {
-
- private ByteBuffer buffer;
- private final long length;
- private boolean isClone = false;
-
- private MMapIndexInput(RandomAccessFile raf) throws IOException {
- this.length = raf.length();
- this.buffer = raf.getChannel().map(MapMode.READ_ONLY, 0, length);
- }
-
- @Override
- public byte readByte() throws IOException {
- try {
- return buffer.get();
- } catch (BufferUnderflowException e) {
- throw new IOException("read past EOF");
- }
- }
-
- @Override
- public void readBytes(byte[] b, int offset, int len) throws IOException {
- try {
- buffer.get(b, offset, len);
- } catch (BufferUnderflowException e) {
- throw new IOException("read past EOF");
- }
- }
-
- @Override
- public short readShort() throws IOException {
- try {
- return buffer.getShort();
- } catch (BufferUnderflowException e) {
- throw new IOException("read past EOF");
- }
- }
-
- @Override
- public int readInt() throws IOException {
- try {
- return buffer.getInt();
- } catch (BufferUnderflowException e) {
- throw new IOException("read past EOF");
- }
- }
-
- @Override
- public long readLong() throws IOException {
- try {
- return buffer.getLong();
- } catch (BufferUnderflowException e) {
- throw new IOException("read past EOF");
- }
- }
-
- @Override
- public long getFilePointer() {
- return buffer.position();
- }
-
- @Override
- public void seek(long pos) throws IOException {
- buffer.position((int)pos);
- }
-
- @Override
- public long length() {
- return length;
- }
-
- @Override
- public Object clone() {
- if (buffer == null)
- throw new AlreadyClosedException("MMapIndexInput already closed");
- MMapIndexInput clone = (MMapIndexInput)super.clone();
- clone.isClone = true;
- clone.buffer = buffer.duplicate();
- return clone;
- }
-
- @Override
- public void close() throws IOException {
- // unmap the buffer (if enabled) and at least unset it for GC
- try {
- if (isClone || buffer == null) return;
- cleanMapping(buffer);
- } finally {
- buffer = null;
- }
- }
- }
-
// Because Java's ByteBuffer uses an int to address the
// values, it's necessary to access a file >
// Integer.MAX_VALUE in size using multiple byte buffers.
- private class MultiMMapIndexInput extends IndexInput {
+ private final class MMapIndexInput extends IndexInput {
private ByteBuffer[] buffers;
- private int[] bufSizes; // keep here, ByteBuffer.size() method is optional
- private final long length;
+ private final long length, chunkSizeMask, chunkSize;
+ private final int chunkSizePower;
private int curBufIndex;
- private final int maxBufSize;
private ByteBuffer curBuf; // redundant for speed: buffers[curBufIndex]
private boolean isClone = false;
- public MultiMMapIndexInput(RandomAccessFile raf, int maxBufSize)
- throws IOException {
+ MMapIndexInput(RandomAccessFile raf, int chunkSizePower) throws IOException {
this.length = raf.length();
- this.maxBufSize = maxBufSize;
+ this.chunkSizePower = chunkSizePower;
+ this.chunkSize = 1L << chunkSizePower;
+ this.chunkSizeMask = chunkSize - 1L;
- if (maxBufSize <= 0)
- throw new IllegalArgumentException("Non positive maxBufSize: "
- + maxBufSize);
+ if (chunkSizePower < 0 || chunkSizePower > 30)
+ throw new IllegalArgumentException("Invalid chunkSizePower used for ByteBuffer size: " + chunkSizePower);
- if ((length / maxBufSize) > Integer.MAX_VALUE)
- throw new IllegalArgumentException
- ("RandomAccessFile too big for maximum buffer size: "
- + raf.toString());
+ if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)
+ throw new IllegalArgumentException("RandomAccessFile too big for chunk size: " + raf.toString());
- int nrBuffers = (int) (length / maxBufSize);
- if (((long) nrBuffers * maxBufSize) <= length) nrBuffers++;
+ // we always allocate one more buffer, the last one may be a 0 byte one
+ final int nrBuffers = (int) (length >>> chunkSizePower) + 1;
+
+ //System.out.println("length="+length+", chunkSizePower=" + chunkSizePower + ", chunkSizeMask=" + chunkSizeMask + ", nrBuffers=" + nrBuffers);
this.buffers = new ByteBuffer[nrBuffers];
- this.bufSizes = new int[nrBuffers];
- long bufferStart = 0;
+ long bufferStart = 0L;
FileChannel rafc = raf.getChannel();
for (int bufNr = 0; bufNr < nrBuffers; bufNr++) {
- int bufSize = (length > (bufferStart + maxBufSize))
- ? maxBufSize
- : (int) (length - bufferStart);
- this.buffers[bufNr] = rafc.map(MapMode.READ_ONLY,bufferStart,bufSize);
- this.bufSizes[bufNr] = bufSize;
+ int bufSize = (int) ( (length > (bufferStart + chunkSize))
+ ? chunkSize
+ : (length - bufferStart)
+ );
+ this.buffers[bufNr] = rafc.map(MapMode.READ_ONLY, bufferStart, bufSize);
bufferStart += bufSize;
}
seek(0L);
@@ -362,11 +272,13 @@ public class MMapDirectory extends FSDir
try {
return curBuf.get();
} catch (BufferUnderflowException e) {
- curBufIndex++;
- if (curBufIndex >= buffers.length)
- throw new IOException("read past EOF");
- curBuf = buffers[curBufIndex];
- curBuf.position(0);
+ do {
+ curBufIndex++;
+ if (curBufIndex >= buffers.length)
+ throw new IOException("read past EOF");
+ curBuf = buffers[curBufIndex];
+ curBuf.position(0);
+ } while (!curBuf.hasRemaining());
return curBuf.get();
}
}
@@ -421,15 +333,28 @@ public class MMapDirectory extends FSDir
@Override
public long getFilePointer() {
- return ((long) curBufIndex * maxBufSize) + curBuf.position();
+ return (((long) curBufIndex) << chunkSizePower) + curBuf.position();
}
@Override
public void seek(long pos) throws IOException {
- curBufIndex = (int) (pos / maxBufSize);
- curBuf = buffers[curBufIndex];
- int bufOffset = (int) (pos - ((long) curBufIndex * maxBufSize));
- curBuf.position(bufOffset);
+ // we use >> here to preserve negative, so we will catch AIOOBE:
+ final int bi = (int) (pos >> chunkSizePower);
+ try {
+ final ByteBuffer b = buffers[bi];
+ b.position((int) (pos & chunkSizeMask));
+ // write values, on exception all is unchanged
+ this.curBufIndex = bi;
+ this.curBuf = b;
+ } catch (ArrayIndexOutOfBoundsException aioobe) {
+ if (pos < 0L)
+ throw new IllegalArgumentException("Seeking to negative position");
+ throw new IOException("seek past EOF");
+ } catch (IllegalArgumentException iae) {
+ if (pos < 0L)
+ throw new IllegalArgumentException("Seeking to negative position");
+ throw new IOException("seek past EOF");
+ }
}
@Override
@@ -440,11 +365,10 @@ public class MMapDirectory extends FSDir
@Override
public Object clone() {
if (buffers == null)
- throw new AlreadyClosedException("MultiMMapIndexInput already closed");
- MultiMMapIndexInput clone = (MultiMMapIndexInput)super.clone();
+ throw new AlreadyClosedException("MMapIndexInput already closed");
+ final MMapIndexInput clone = (MMapIndexInput)super.clone();
clone.isClone = true;
clone.buffers = new ByteBuffer[buffers.length];
- // No need to clone bufSizes.
// Since most clones will use only one buffer, duplicate() could also be
// done lazy in clones, e.g. when adapting curBuf.
for (int bufNr = 0; bufNr < buffers.length; bufNr++) {
@@ -453,9 +377,7 @@ public class MMapDirectory extends FSDir
try {
clone.seek(getFilePointer());
} catch(IOException ioe) {
- RuntimeException newException = new RuntimeException(ioe);
- newException.initCause(ioe);
- throw newException;
+ throw new RuntimeException("Should never happen", ioe);
}
return clone;
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/FST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/FST.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/FST.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/FST.java Thu Jun 16 20:33:59 2011
@@ -490,7 +490,7 @@ public class FST<T> {
if (!targetHasArcs(follow)) {
//System.out.println(" end node");
assert follow.isFinal();
- arc.label = -1;
+ arc.label = END_LABEL;
arc.output = follow.nextFinalOutput;
arc.flags = BIT_LAST_ARC;
return arc;
@@ -544,7 +544,7 @@ public class FST<T> {
//System.out.println(" readFirstTarget follow.target=" + follow.target + " isFinal=" + follow.isFinal());
if (follow.isFinal()) {
// Insert "fake" final first arc:
- arc.label = -1;
+ arc.label = END_LABEL;
arc.output = follow.nextFinalOutput;
if (follow.target <= 0) {
arc.flags = BIT_LAST_ARC;
@@ -599,7 +599,7 @@ public class FST<T> {
/** In-place read; returns the arc. */
public Arc<T> readNextArc(Arc<T> arc) throws IOException {
- if (arc.label == -1) {
+ if (arc.label == END_LABEL) {
// This was a fake inserted "final" arc
if (arc.nextArc <= 0) {
// This arc went to virtual final node, ie has no outgoing arcs
Modified: lucene/dev/branches/flexscoring/lucene/src/site/changes/changes2html.pl
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/site/changes/changes2html.pl?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/site/changes/changes2html.pl (original)
+++ lucene/dev/branches/flexscoring/lucene/src/site/changes/changes2html.pl Thu Jun 16 20:33:59 2011
@@ -23,7 +23,6 @@
use strict;
use warnings;
-use JSON;
use LWP::Simple;
my $project_info_url = 'https://issues.apache.org/jira/rest/api/2.0.alpha1/project/LUCENE';
@@ -675,8 +674,8 @@ sub setup_release_dates {
'2.4.0' => '2008-10-06', '2.4.1' => '2009-03-09',
'2.9.0' => '2009-09-23', '2.9.1' => '2009-11-06',
'3.0.0' => '2009-11-25');
- my $project_info_json = get($project_info_url);
- my $project_info = decode_json($project_info_json);
+ my $project_info_json = get($project_info_url) || die "Unable to download $project_info_url";
+ my $project_info = json2perl($project_info_json);
for my $version (@{$project_info->{versions}}) {
if ($version->{releaseDate}) {
my $date = substr($version->{releaseDate}, 0, 10);
@@ -851,5 +850,16 @@ sub setup_bugzilla_jira_map {
36628 => 432);
}
+#
+# json2perl
+#
+# Converts a JSON string to the equivalent Perl data structure
+#
+sub json2perl {
+ my $json_string = shift;
+ $json_string =~ s/(:\s*)(true|false)/$1"$2"/g;
+ $json_string =~ s/":/",/g;
+ return eval $json_string;
+}
1;
Modified: lucene/dev/branches/flexscoring/lucene/src/site/forrest.properties
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/site/forrest.properties?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/site/forrest.properties (original)
+++ lucene/dev/branches/flexscoring/lucene/src/site/forrest.properties Thu Jun 16 20:33:59 2011
@@ -123,7 +123,7 @@ project.configfile=${project.home}/src/d
# a production environment it is recomended that you specify a known working
# version.
# Run "forrest available-plugins" for a list of plug-ins currently available
-project.required.plugins=org.apache.forrest.plugin.output.pdf
+#project.required.plugins=org.apache.forrest.plugin.output.pdf
# Proxy configuration
# proxy.host=
Modified: lucene/dev/branches/flexscoring/lucene/src/site/src/documentation/skinconf.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/site/src/documentation/skinconf.xml?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/site/src/documentation/skinconf.xml (original)
+++ lucene/dev/branches/flexscoring/lucene/src/site/src/documentation/skinconf.xml Thu Jun 16 20:33:59 2011
@@ -35,7 +35,7 @@ which will be used to configure the chos
<!-- Disable the print link? If enabled, invalid HTML 4.0.1 -->
<disable-print-link>true</disable-print-link>
<!-- Disable the PDF link? -->
- <disable-pdf-link>false</disable-pdf-link>
+ <disable-pdf-link>true</disable-pdf-link>
<!-- Disable the POD link? -->
<disable-pod-link>true</disable-pod-link>
<!-- Disable the Text link? FIXME: NOT YET IMPLEMENETED. -->
@@ -364,60 +364,6 @@ which will be used to configure the chos
-->
</colors>
- <!-- Settings specific to PDF output. -->
- <pdf>
- <!--
- Supported page sizes are a0, a1, a2, a3, a4, a5, executive,
- folio, legal, ledger, letter, quarto, tabloid (default letter).
- Supported page orientations are portrait, landscape (default
- portrait).
- Supported text alignments are left, right, justify (default left).
- -->
- <page size="letter" orientation="portrait" text-align="left"/>
-
- <!--
- Pattern of the page numbering in the footer - Default is "Page x".
- first occurrence of '1' digit represents the current page number,
- second occurrence of '1' digit represents the total page number,
- anything else is considered as the static part of the numbering pattern.
- Examples : x is the current page number, y the total page number.
- <page-numbering-format>none</page-numbering-format> Do not displays the page numbering
- <page-numbering-format>1</page-numbering-format> Displays "x"
- <page-numbering-format>p1.</page-numbering-format> Displays "px."
- <page-numbering-format>Page 1/1</page-numbering-format> Displays "Page x/y"
- <page-numbering-format>(1-1)</page-numbering-format> Displays "(x-y)"
- -->
- <page-numbering-format>Page 1</page-numbering-format>
-
- <!--
- Margins can be specified for top, bottom, inner, and outer
- edges. If double-sided="false", the inner edge is always left
- and the outer is always right. If double-sided="true", the
- inner edge will be left on odd pages, right on even pages,
- the outer edge vice versa.
- Specified below are the default settings.
- -->
- <margins double-sided="false">
- <top>1in</top>
- <bottom>1in</bottom>
- <inner>1.25in</inner>
- <outer>1in</outer>
- </margins>
-
- <!--
- Print the URL text next to all links going outside the file
- -->
- <show-external-urls>false</show-external-urls>
-
- <!--
- Disable the copyright footer on each page of the PDF.
- A footer is composed for each page. By default, a "credit" with role=pdf
- will be used, as explained below. Otherwise a copyright statement
- will be generated. This latter can be disabled.
- -->
- <disable-copyright-footer>false</disable-copyright-footer>
- </pdf>
-
<!--
Credits are typically rendered as a set of small clickable
images in the page footer.
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java Thu Jun 16 20:33:59 2011
@@ -308,16 +308,24 @@ public class RandomIndexWriter implement
return getReader(true);
}
+ private boolean doRandomOptimize = true;
+
+ public void setDoRandomOptimize(boolean v) {
+ doRandomOptimize = v;
+ }
+
private void doRandomOptimize() throws IOException {
- final int segCount = w.getSegmentCount();
- if (r.nextBoolean() || segCount == 0) {
- // full optimize
- w.optimize();
- } else {
- // partial optimize
- final int limit = _TestUtil.nextInt(r, 1, segCount);
- w.optimize(limit);
- assert w.getSegmentCount() <= limit: "limit=" + limit + " actual=" + w.getSegmentCount();
+ if (doRandomOptimize) {
+ final int segCount = w.getSegmentCount();
+ if (r.nextBoolean() || segCount == 0) {
+ // full optimize
+ w.optimize();
+ } else {
+ // partial optimize
+ final int limit = _TestUtil.nextInt(r, 1, segCount);
+ w.optimize(limit);
+ assert w.getSegmentCount() <= limit: "limit=" + limit + " actual=" + w.getSegmentCount();
+ }
}
switchDoDocValues();
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java Thu Jun 16 20:33:59 2011
@@ -22,6 +22,8 @@ import java.io.Closeable;
import java.io.IOException;
import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.codecs.preflex.PreFlexCodec;
import org.apache.lucene.index.codecs.preflex.TermInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexOutput;
@@ -102,7 +104,9 @@ final class TermInfosWriter implements C
}
try {
- directory.deleteFile(segment + (isIndex ? ".tii" : ".tis"));
+ directory.deleteFile(IndexFileNames.segmentFileName(segment, "",
+ (isIndex ? PreFlexCodec.TERMS_INDEX_EXTENSION
+ : PreFlexCodec.TERMS_EXTENSION)));
} catch (IOException ignored) {
}
}
@@ -119,7 +123,9 @@ final class TermInfosWriter implements C
indexInterval = interval;
fieldInfos = fis;
isIndex = isi;
- output = directory.createOutput(segment + (isIndex ? ".tii" : ".tis"));
+ output = directory.createOutput(IndexFileNames.segmentFileName(segment, "",
+ (isIndex ? PreFlexCodec.TERMS_INDEX_EXTENSION
+ : PreFlexCodec.TERMS_EXTENSION)));
boolean success = false;
try {
output.writeInt(FORMAT_CURRENT); // write format
@@ -139,7 +145,9 @@ final class TermInfosWriter implements C
}
try {
- directory.deleteFile(segment + (isIndex ? ".tii" : ".tis"));
+ directory.deleteFile(IndexFileNames.segmentFileName(segment, "",
+ (isIndex ? PreFlexCodec.TERMS_INDEX_EXTENSION
+ : PreFlexCodec.TERMS_EXTENSION)));
} catch (IOException ignored) {
}
}
Copied: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java (from r1136649, lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java?p2=lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java&p1=lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java&r1=1136649&r2=1136656&rev=1136656&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java Thu Jun 16 20:33:59 2011
@@ -62,12 +62,7 @@ public class AssertingIndexSearcher exte
}
@Override
- public float getValue() {
- return w.getValue();
- }
-
- @Override
- public void normalize(float norm) {
+ public void normalize(float norm, float topLevelBoost) {
throw new IllegalStateException("Weight already normalized.");
}
@@ -77,7 +72,7 @@ public class AssertingIndexSearcher exte
}
@Override
- public float sumOfSquaredWeights() throws IOException {
+ public float getValueForNormalization() throws IOException {
throw new IllegalStateException("Weight already normalized.");
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java Thu Jun 16 20:33:59 2011
@@ -198,7 +198,7 @@ public class QueryUtils {
public static void checkSkipTo(final Query q, final IndexSearcher s) throws IOException {
//System.out.println("Checking "+q);
final AtomicReaderContext[] readerContextArray = ReaderUtil.leaves(s.getTopReaderContext());
- if (q.weight(s).scoresDocsOutOfOrder()) return; // in this case order of skipTo() might differ from that of next().
+ if (s.createNormalizedWeight(q).scoresDocsOutOfOrder()) return; // in this case order of skipTo() might differ from that of next().
final int skip_op = 0;
final int next_op = 1;
@@ -241,7 +241,7 @@ public class QueryUtils {
lastDoc[0] = doc;
try {
if (scorer == null) {
- Weight w = q.weight(s);
+ Weight w = s.createNormalizedWeight(q);
scorer = w.scorer(readerContextArray[leafPtr], ScorerContext.def());
}
@@ -286,7 +286,7 @@ public class QueryUtils {
if (lastReader[0] != null) {
final IndexReader previousReader = lastReader[0];
IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader);
- Weight w = q.weight(indexSearcher);
+ Weight w = indexSearcher.createNormalizedWeight(q);
Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), ScorerContext.def());
if (scorer != null) {
boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS;
@@ -312,7 +312,7 @@ public class QueryUtils {
// previous reader, hits NO_MORE_DOCS
final IndexReader previousReader = lastReader[0];
IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader, false);
- Weight w = q.weight(indexSearcher);
+ Weight w = indexSearcher.createNormalizedWeight(q);
Scorer scorer = w.scorer((AtomicReaderContext)previousReader.getTopReaderContext(), ScorerContext.def());
if (scorer != null) {
boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS;
@@ -343,7 +343,7 @@ public class QueryUtils {
try {
long startMS = System.currentTimeMillis();
for (int i=lastDoc[0]+1; i<=doc; i++) {
- Weight w = q.weight(s);
+ Weight w = s.createNormalizedWeight(q);
Scorer scorer = w.scorer(context[leafPtr], ScorerContext.def());
Assert.assertTrue("query collected "+doc+" but skipTo("+i+") says no more docs!",scorer.advance(i) != DocIdSetIterator.NO_MORE_DOCS);
Assert.assertEquals("query collected "+doc+" but skipTo("+i+") got to "+scorer.docID(),doc,scorer.docID());
@@ -370,7 +370,7 @@ public class QueryUtils {
if (lastReader[0] != null) {
final IndexReader previousReader = lastReader[0];
IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader);
- Weight w = q.weight(indexSearcher);
+ Weight w = indexSearcher.createNormalizedWeight(q);
Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), ScorerContext.def());
if (scorer != null) {
boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS;
@@ -394,7 +394,7 @@ public class QueryUtils {
// previous reader, hits NO_MORE_DOCS
final IndexReader previousReader = lastReader[0];
IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader);
- Weight w = q.weight(indexSearcher);
+ Weight w = indexSearcher.createNormalizedWeight(q);
Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), ScorerContext.def());
if (scorer != null) {
boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS;
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java Thu Jun 16 20:33:59 2011
@@ -55,6 +55,7 @@ import org.apache.lucene.index.codecs.st
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.FieldCache.CacheEntry;
+import org.apache.lucene.search.AssertingIndexSearcher;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@@ -1231,13 +1232,11 @@ public abstract class LuceneTestCase ext
* with one that returns null for getSequentialSubReaders.
*/
public static IndexSearcher newSearcher(IndexReader r, boolean maybeWrap) throws IOException {
-
if (random.nextBoolean()) {
if (maybeWrap && rarely()) {
- return new IndexSearcher(new SlowMultiReaderWrapper(r));
- } else {
- return new IndexSearcher(r);
+ r = new SlowMultiReaderWrapper(r);
}
+ return random.nextBoolean() ? new AssertingIndexSearcher(r) : new AssertingIndexSearcher(r.getTopReaderContext());
} else {
int threads = 0;
final ExecutorService ex = (random.nextBoolean()) ? null
@@ -1246,20 +1245,31 @@ public abstract class LuceneTestCase ext
if (ex != null && VERBOSE) {
System.out.println("NOTE: newSearcher using ExecutorService with " + threads + " threads");
}
- return new IndexSearcher(r.getTopReaderContext(), ex) {
- @Override
- public void close() throws IOException {
- super.close();
- if (ex != null) {
- ex.shutdown();
- try {
- ex.awaitTermination(1000, TimeUnit.MILLISECONDS);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
+ return random.nextBoolean() ?
+ new AssertingIndexSearcher(r, ex) {
+ @Override
+ public void close() throws IOException {
+ super.close();
+ shutdownExecutorService(ex);
}
- }
- };
+ } : new AssertingIndexSearcher(r.getTopReaderContext(), ex) {
+ @Override
+ public void close() throws IOException {
+ super.close();
+ shutdownExecutorService(ex);
+ }
+ };
+ }
+ }
+
+ static void shutdownExecutorService(ExecutorService ex) {
+ if (ex != null) {
+ ex.shutdown();
+ try {
+ ex.awaitTermination(1000, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
}
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java Thu Jun 16 20:33:59 2011
@@ -27,10 +27,10 @@ import java.io.OutputStream;
import java.io.PrintStream;
import java.lang.reflect.Method;
import java.util.Enumeration;
+import java.util.HashMap;
import java.util.List;
-import java.util.Random;
import java.util.Map;
-import java.util.HashMap;
+import java.util.Random;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
@@ -46,6 +46,9 @@ import org.apache.lucene.index.MergeSche
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.junit.Assert;
@@ -378,6 +381,7 @@ public class _TestUtil {
} else if (mp instanceof TieredMergePolicy) {
TieredMergePolicy tmp = (TieredMergePolicy) mp;
tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
+ tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
}
MergeScheduler ms = w.getConfig().getMergeScheduler();
if (ms instanceof ConcurrentMergeScheduler) {
@@ -467,4 +471,24 @@ public class _TestUtil {
newName.append(suffix);
return new File(directory, newName.toString());
}
+
+ public static void assertEquals(TopDocs expected, TopDocs actual) {
+ Assert.assertEquals("wrong total hits", expected.totalHits, actual.totalHits);
+ Assert.assertEquals("wrong maxScore", expected.getMaxScore(), actual.getMaxScore(), 0.0);
+ Assert.assertEquals("wrong hit count", expected.scoreDocs.length, actual.scoreDocs.length);
+ for(int hitIDX=0;hitIDX<expected.scoreDocs.length;hitIDX++) {
+ final ScoreDoc expectedSD = expected.scoreDocs[hitIDX];
+ final ScoreDoc actualSD = actual.scoreDocs[hitIDX];
+ Assert.assertEquals("wrong hit docID", expectedSD.doc, actualSD.doc);
+ Assert.assertEquals("wrong hit score", expectedSD.score, actualSD.score, 0.0);
+ if (expectedSD instanceof FieldDoc) {
+ Assert.assertTrue(actualSD instanceof FieldDoc);
+ Assert.assertEquals("wrong sort field values",
+ ((FieldDoc) expectedSD).fields,
+ ((FieldDoc) actualSD).fields);
+ } else {
+ Assert.assertFalse(actualSD instanceof FieldDoc);
+ }
+ }
+ }
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Thu Jun 16 20:33:59 2011
@@ -1375,6 +1375,11 @@ public class TestIndexWriter extends Luc
IndexReader r2 = r.reopen();
assertTrue(r != r2);
files = Arrays.asList(dir.listAll());
+
+ // NOTE: here we rely on "Windows" behavior, ie, even
+ // though IW wanted to delete _0.cfs since it was
+ // optimized away, because we have a reader open
+ // against this file, it should still be here:
assertTrue(files.contains("_0.cfs"));
// optimize created this
//assertTrue(files.contains("_2.cfs"));
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java Thu Jun 16 20:33:59 2011
@@ -405,30 +405,30 @@ public class TestNRTThreads extends Luce
for(int thread=0;thread<NUM_SEARCH_THREADS;thread++) {
searchThreads[thread] = new Thread() {
@Override
- public void run() {
+ public void run() {
try {
TermsEnum termsEnum = MultiFields.getTerms(s.getIndexReader(), "body").iterator();
int seenTermCount = 0;
int shift;
int trigger;
- if (totTermCount.get() == 0) {
+ if (totTermCount.get() < 10) {
shift = 0;
trigger = 1;
} else {
- shift = random.nextInt(totTermCount.get()/10);
trigger = totTermCount.get()/10;
+ shift = random.nextInt(trigger);
}
while(System.currentTimeMillis() < searchStopTime) {
BytesRef term = termsEnum.next();
if (term == null) {
- if (seenTermCount == 0) {
+ if (seenTermCount < 10) {
break;
}
totTermCount.set(seenTermCount);
seenTermCount = 0;
trigger = totTermCount.get()/10;
//System.out.println("trigger " + trigger);
- shift = random.nextInt(totTermCount.get()/10);
+ shift = random.nextInt(trigger);
termsEnum.seek(new BytesRef(""));
continue;
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java Thu Jun 16 20:33:59 2011
@@ -390,8 +390,6 @@ public class TestTermVectorsReader exten
fail();
} catch (IOException e) {
// expected exception
- } catch (IllegalArgumentException e) {
- // mmapdir will give us this from java.nio.Buffer.position()
} finally {
reader.close();
}
@@ -402,8 +400,6 @@ public class TestTermVectorsReader exten
fail();
} catch (IOException e) {
// expected exception
- } catch (IllegalArgumentException e) {
- // mmapdir will give us this from java.nio.Buffer.position()
} finally {
reader.close();
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java Thu Jun 16 20:33:59 2011
@@ -342,7 +342,7 @@ public class TestDocValuesIndexing exten
for (ValueType byteIndexValue : byteVariantList) {
List<Closeable> closeables = new ArrayList<Closeable>();
- int bytesSize = 7 + random.nextInt(128);
+ int bytesSize = 1 + random.nextInt(128);
OpenBitSet deleted = indexValues(w, numValues, byteIndexValue,
byteVariantList, withDeletions, bytesSize);
final IndexReader r = IndexReader.open(w, withDeletions);
@@ -493,7 +493,7 @@ public class TestDocValuesIndexing exten
Index.NO };
private OpenBitSet indexValues(IndexWriter w, int numValues, ValueType value,
- List<ValueType> valueVarList, boolean withDeletions, int multOfSeven)
+ List<ValueType> valueVarList, boolean withDeletions, int bytesSize)
throws CorruptIndexException, IOException {
final boolean isNumeric = NUMERICS.contains(value);
OpenBitSet deleted = new OpenBitSet(numValues);
@@ -507,7 +507,7 @@ public class TestDocValuesIndexing exten
final BytesRef bytesRef = new BytesRef();
final String idBase = value.name() + "_";
- final byte[] b = new byte[multOfSeven];
+ final byte[] b = new byte[bytesSize];
if (bytesRef != null) {
bytesRef.bytes = b;
bytesRef.length = b.length;
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java Thu Jun 16 20:33:59 2011
@@ -103,7 +103,7 @@ final class JustCompileSearch {
}
- static final class JustCompileFieldComparator extends FieldComparator {
+ static final class JustCompileFieldComparator extends FieldComparator<Object> {
@Override
public int compare(int slot1, int slot2) {
@@ -132,10 +132,10 @@ final class JustCompileSearch {
}
@Override
- public Comparable<?> value(int slot) {
+ public Object value(int slot) {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
-
+
}
static final class JustCompileFieldComparatorSource extends FieldComparatorSource {
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java Thu Jun 16 20:33:59 2011
@@ -173,7 +173,7 @@ public class TestDisjunctionMaxQuery ext
QueryUtils.check(random, dq, s);
assertTrue(s.getTopReaderContext().isAtomic);
- final Weight dw = dq.weight(s);
+ final Weight dw = s.createNormalizedWeight(dq);
final Scorer ds = dw.scorer((AtomicReaderContext)s.getTopReaderContext(), ScorerContext.def());
final boolean skipOk = ds.advance(3) != DocIdSetIterator.NO_MORE_DOCS;
if (skipOk) {
@@ -188,7 +188,7 @@ public class TestDisjunctionMaxQuery ext
dq.add(tq("dek", "DOES_NOT_EXIST"));
assertTrue(s.getTopReaderContext().isAtomic);
QueryUtils.check(random, dq, s);
- final Weight dw = dq.weight(s);
+ final Weight dw = s.createNormalizedWeight(dq);
final Scorer ds = dw.scorer((AtomicReaderContext)s.getTopReaderContext(), ScorerContext.def());
assertTrue("firsttime skipTo found no match",
ds.advance(3) != DocIdSetIterator.NO_MORE_DOCS);
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestElevationComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestElevationComparator.java?rev=1136656&r1=1136655&r2=1136656&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestElevationComparator.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestElevationComparator.java Thu Jun 16 20:33:59 2011
@@ -139,7 +139,7 @@ class ElevationComparatorSource extends
@Override
public FieldComparator newComparator(final String fieldname, final int numHits, int sortPos, boolean reversed) throws IOException {
- return new FieldComparator() {
+ return new FieldComparator<Integer>() {
FieldCache.DocTermsIndex idIndex;
private final int[] values = new int[numHits];
@@ -184,7 +184,7 @@ class ElevationComparatorSource extends
}
@Override
- public Comparable<?> value(int slot) {
+ public Integer value(int slot) {
return Integer.valueOf(values[slot]);
}
};