You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/06/25 01:07:38 UTC
svn commit: r1139458 [6/24] - in /lucene/dev/branches/solr2193: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/
dev-tools/idea/solr/ dev-tools/idea/solr/contrib/clustering/
dev-tools/maven/ dev-tools/maven/modules/analysis/icu/ dev-t...
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/SortField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/SortField.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/SortField.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/SortField.java Fri Jun 24 23:07:10 2011
@@ -18,10 +18,16 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
+import java.util.Comparator;
import org.apache.lucene.search.cache.*;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.StringHelper;
+// TODO(simonw) -- for cleaner transition, maybe we should make
+// a new SortField that subclasses this one and always uses
+// index values?
+
/**
* Stores information about how to sort documents by terms in an individual
* field. Fields must be indexed in order to sort by them.
@@ -33,62 +39,70 @@ import org.apache.lucene.util.StringHelp
*/
public class SortField {
- /** Sort by document score (relevance). Sort values are Float and higher
- * values are at the front. */
- public static final int SCORE = 0;
-
- /** Sort by document number (index order). Sort values are Integer and lower
- * values are at the front. */
- public static final int DOC = 1;
-
- // reserved, in Lucene 2.9, there was a constant: AUTO = 2;
-
- /** Sort using term values as Strings. Sort values are String and lower
- * values are at the front. */
- public static final int STRING = 3;
-
- /** Sort using term values as encoded Integers. Sort values are Integer and
- * lower values are at the front. */
- public static final int INT = 4;
-
- /** Sort using term values as encoded Floats. Sort values are Float and
- * lower values are at the front. */
- public static final int FLOAT = 5;
-
- /** Sort using term values as encoded Longs. Sort values are Long and
- * lower values are at the front. */
- public static final int LONG = 6;
-
- /** Sort using term values as encoded Doubles. Sort values are Double and
- * lower values are at the front. */
- public static final int DOUBLE = 7;
-
- /** Sort using term values as encoded Shorts. Sort values are Short and
- * lower values are at the front. */
- public static final int SHORT = 8;
-
- /** Sort using a custom Comparator. Sort values are any Comparable and
- * sorting is done according to natural order. */
- public static final int CUSTOM = 9;
-
- /** Sort using term values as encoded Bytes. Sort values are Byte and
- * lower values are at the front. */
- public static final int BYTE = 10;
-
- /** Sort using term values as Strings, but comparing by
- * value (using String.compareTo) for all comparisons.
- * This is typically slower than {@link #STRING}, which
- * uses ordinals to do the sorting. */
- public static final int STRING_VAL = 11;
-
+ public static enum Type {
+
+ /** Sort by document score (relevance). Sort values are Float and higher
+ * values are at the front. */
+ SCORE,
+
+ /** Sort by document number (index order). Sort values are Integer and lower
+ * values are at the front. */
+ DOC,
+
+ /** Sort using term values as Strings. Sort values are String and lower
+ * values are at the front. */
+ STRING,
+
+ /** Sort using term values as encoded Integers. Sort values are Integer and
+ * lower values are at the front. */
+ INT,
+
+ /** Sort using term values as encoded Floats. Sort values are Float and
+ * lower values are at the front. */
+ FLOAT,
+
+ /** Sort using term values as encoded Longs. Sort values are Long and
+ * lower values are at the front. */
+ LONG,
+
+ /** Sort using term values as encoded Doubles. Sort values are Double and
+ * lower values are at the front. */
+ DOUBLE,
+
+ /** Sort using term values as encoded Shorts. Sort values are Short and
+ * lower values are at the front. */
+ SHORT,
+
+ /** Sort using a custom Comparator. Sort values are any Comparable and
+ * sorting is done according to natural order. */
+ CUSTOM,
+
+ /** Sort using term values as encoded Bytes. Sort values are Byte and
+ * lower values are at the front. */
+ BYTE,
+
+ /** Sort using term values as Strings, but comparing by
+ * value (using String.compareTo) for all comparisons.
+ * This is typically slower than {@link #STRING}, which
+ * uses ordinals to do the sorting. */
+ STRING_VAL,
+
+ /** Sort use byte[] index values. */
+ BYTES,
+
+ /** Force rewriting of SortField using {@link SortField#rewrite(IndexSearcher)}
+ * before it can be used for sorting */
+ REWRITEABLE
+ }
+
/** Represents sorting by document score (relevance). */
- public static final SortField FIELD_SCORE = new SortField (null, SCORE);
+ public static final SortField FIELD_SCORE = new SortField(null, Type.SCORE);
/** Represents sorting by document number (index order). */
- public static final SortField FIELD_DOC = new SortField (null, DOC);
+ public static final SortField FIELD_DOC = new SortField(null, Type.DOC);
private String field;
- private int type; // defaults to determining type dynamically
+ private Type type; // defaults to determining type dynamically
boolean reverse = false; // defaults to natural order
private CachedArrayCreator<?> creator;
public Object missingValue = null; // used for 'sortMissingFirst/Last'
@@ -102,7 +116,7 @@ public class SortField {
* <code>type</code> is SCORE or DOC.
* @param type Type of values in the terms.
*/
- public SortField (String field, int type) {
+ public SortField(String field, Type type) {
initFieldType(field, type);
}
@@ -113,7 +127,7 @@ public class SortField {
* @param type Type of values in the terms.
* @param reverse True if natural order should be reversed.
*/
- public SortField (String field, int type, boolean reverse) {
+ public SortField(String field, Type type, boolean reverse) {
initFieldType(field, type);
this.reverse = reverse;
}
@@ -131,7 +145,7 @@ public class SortField {
* @deprecated (4.0) use EntryCreator version
*/
@Deprecated
- public SortField (String field, FieldCache.Parser parser) {
+ public SortField(String field, FieldCache.Parser parser) {
this(field, parser, false);
}
@@ -149,40 +163,35 @@ public class SortField {
* @deprecated (4.0) use EntryCreator version
*/
@Deprecated
- public SortField (String field, FieldCache.Parser parser, boolean reverse) {
+ public SortField(String field, FieldCache.Parser parser, boolean reverse) {
if (field == null) {
throw new IllegalArgumentException("field can only be null when type is SCORE or DOC");
}
- this.field = StringHelper.intern(field);
+ this.field = field;
this.reverse = reverse;
if (parser instanceof FieldCache.IntParser) {
- this.type = INT;
this.creator = new IntValuesCreator( field, (FieldCache.IntParser)parser );
}
else if (parser instanceof FieldCache.FloatParser) {
- this.type = FLOAT;
this.creator = new FloatValuesCreator( field, (FieldCache.FloatParser)parser );
}
else if (parser instanceof FieldCache.ShortParser) {
- this.type = SHORT;
this.creator = new ShortValuesCreator( field, (FieldCache.ShortParser)parser );
}
else if (parser instanceof FieldCache.ByteParser) {
- this.type = BYTE;
this.creator = new ByteValuesCreator( field, (FieldCache.ByteParser)parser );
}
else if (parser instanceof FieldCache.LongParser) {
- this.type = LONG;
this.creator = new LongValuesCreator( field, (FieldCache.LongParser)parser );
}
else if (parser instanceof FieldCache.DoubleParser) {
- this.type = DOUBLE;
this.creator = new DoubleValuesCreator( field, (FieldCache.DoubleParser)parser );
}
else
throw new IllegalArgumentException("Parser instance does not subclass existing numeric parser from FieldCache (got " + parser + ")");
+ this.type = this.creator.getSortType();
}
/**
@@ -192,10 +201,10 @@ public class SortField {
*/
public SortField( CachedArrayCreator<?> creator, boolean reverse )
{
- this.field = StringHelper.intern(creator.field);
+ this.field = creator.field;
this.reverse = reverse;
this.creator = creator;
- this.type = creator.getSortTypeID();
+ this.type = creator.getSortType();
}
public SortField setMissingValue( Object v )
@@ -216,8 +225,8 @@ public class SortField {
* @param field Name of field to sort by; cannot be <code>null</code>.
* @param comparator Returns a comparator for sorting hits.
*/
- public SortField (String field, FieldComparatorSource comparator) {
- initFieldType(field, CUSTOM);
+ public SortField(String field, FieldComparatorSource comparator) {
+ initFieldType(field, Type.CUSTOM);
this.comparatorSource = comparator;
}
@@ -226,21 +235,21 @@ public class SortField {
* @param comparator Returns a comparator for sorting hits.
* @param reverse True if natural order should be reversed.
*/
- public SortField (String field, FieldComparatorSource comparator, boolean reverse) {
- initFieldType(field, CUSTOM);
+ public SortField(String field, FieldComparatorSource comparator, boolean reverse) {
+ initFieldType(field, Type.CUSTOM);
this.reverse = reverse;
this.comparatorSource = comparator;
}
// Sets field & type, and ensures field is not NULL unless
// type is SCORE or DOC
- private void initFieldType(String field, int type) {
+ private void initFieldType(String field, Type type) {
this.type = type;
if (field == null) {
- if (type != SCORE && type != DOC)
+ if (type != Type.SCORE && type != Type.DOC)
throw new IllegalArgumentException("field can only be null when type is SCORE or DOC");
} else {
- this.field = StringHelper.intern(field);
+ this.field = field;
}
if( creator != null ) {
@@ -267,7 +276,7 @@ public class SortField {
/** Returns the type of contents in the field.
* @return One of the constants SCORE, DOC, STRING, INT or FLOAT.
*/
- public int getType() {
+ public Type getType() {
return type;
}
@@ -346,6 +355,10 @@ public class SortField {
case CUSTOM:
buffer.append("<custom:\"").append(field).append("\": ").append(comparatorSource).append('>');
break;
+
+ case REWRITEABLE:
+ buffer.append("<rewriteable: \"").append(field).append("\">");
+ break;
default:
buffer.append("<???: \"").append(field).append("\">");
@@ -368,7 +381,7 @@ public class SortField {
if (!(o instanceof SortField)) return false;
final SortField other = (SortField)o;
return (
- other.field == this.field // field is always interned
+ StringHelper.equals(other.field, this.field)
&& other.type == this.type
&& other.reverse == this.reverse
&& (other.comparatorSource == null ? this.comparatorSource == null : other.comparatorSource.equals(this.comparatorSource))
@@ -383,13 +396,33 @@ public class SortField {
* used). */
@Override
public int hashCode() {
- int hash=type^0x346565dd + Boolean.valueOf(reverse).hashCode()^0xaf5998bb;
+ int hash = type.hashCode() ^ 0x346565dd + Boolean.valueOf(reverse).hashCode() ^ 0xaf5998bb;
if (field != null) hash += field.hashCode()^0xff5685dd;
if (comparatorSource != null) hash += comparatorSource.hashCode();
if (creator != null) hash += creator.hashCode()^0x3aaf56ff;
return hash;
}
+ private boolean useIndexValues;
+
+ public void setUseIndexValues(boolean b) {
+ useIndexValues = b;
+ }
+
+ public boolean getUseIndexValues() {
+ return useIndexValues;
+ }
+
+ private Comparator<BytesRef> bytesComparator = BytesRef.getUTF8SortedAsUnicodeComparator();
+
+ public void setBytesComparator(Comparator<BytesRef> b) {
+ bytesComparator = b;
+ }
+
+ public Comparator<BytesRef> getBytesComparator() {
+ return bytesComparator;
+ }
+
/** Returns the {@link FieldComparator} to use for
* sorting.
*
@@ -405,42 +438,67 @@ public class SortField {
public FieldComparator getComparator(final int numHits, final int sortPos) throws IOException {
switch (type) {
- case SortField.SCORE:
+ case SCORE:
return new FieldComparator.RelevanceComparator(numHits);
- case SortField.DOC:
+ case DOC:
return new FieldComparator.DocComparator(numHits);
- case SortField.INT:
- return new FieldComparator.IntComparator(numHits, (IntValuesCreator)creator, (Integer)missingValue );
+ case INT:
+ if (useIndexValues) {
+ return new FieldComparator.IntDocValuesComparator(numHits, field);
+ } else {
+ return new FieldComparator.IntComparator(numHits, (IntValuesCreator)creator, (Integer) missingValue);
+ }
- case SortField.FLOAT:
- return new FieldComparator.FloatComparator(numHits, (FloatValuesCreator)creator, (Float)missingValue );
+ case FLOAT:
+ if (useIndexValues) {
+ return new FieldComparator.FloatDocValuesComparator(numHits, field);
+ } else {
+ return new FieldComparator.FloatComparator(numHits, (FloatValuesCreator) creator, (Float) missingValue);
+ }
- case SortField.LONG:
+ case LONG:
return new FieldComparator.LongComparator(numHits, (LongValuesCreator)creator, (Long)missingValue );
- case SortField.DOUBLE:
+ case DOUBLE:
return new FieldComparator.DoubleComparator(numHits, (DoubleValuesCreator)creator, (Double)missingValue );
- case SortField.BYTE:
+ case BYTE:
return new FieldComparator.ByteComparator(numHits, (ByteValuesCreator)creator, (Byte)missingValue );
- case SortField.SHORT:
+ case SHORT:
return new FieldComparator.ShortComparator(numHits, (ShortValuesCreator)creator, (Short)missingValue );
- case SortField.CUSTOM:
+ case CUSTOM:
assert comparatorSource != null;
return comparatorSource.newComparator(field, numHits, sortPos, reverse);
- case SortField.STRING:
+ case STRING:
return new FieldComparator.TermOrdValComparator(numHits, field, sortPos, reverse);
- case SortField.STRING_VAL:
+ case STRING_VAL:
return new FieldComparator.TermValComparator(numHits, field);
+
+ case REWRITEABLE:
+ throw new IllegalStateException("SortField needs to be rewritten through Sort.rewrite(..) and SortField.rewrite(..)");
default:
throw new IllegalStateException("Illegal sort type: " + type);
}
}
+
+ /**
+ * Rewrites this SortField, returning a new SortField if a change is made.
+ * Subclasses should override this define their rewriting behavior when this
+ * SortField is of type {@link SortField.Type#REWRITEABLE}
+ *
+ * @param searcher IndexSearcher to use during rewriting
+ * @return New rewritten SortField, or {@code this} if nothing has changed.
+ * @throws IOException Can be thrown by the rewriting
+ * @lucene.experimental
+ */
+ public SortField rewrite(IndexSearcher searcher) throws IOException {
+ return this;
+ }
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TermQuery.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TermQuery.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TermQuery.java Fri Jun 24 23:07:10 2011
@@ -89,7 +89,7 @@ public class TermQuery extends Query {
public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
final String field = term.field();
final IndexReader reader = context.reader;
- assert termStates.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight is not the same as the current reader's top-reader";
+ assert termStates.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight (" + termStates.topReaderContext + ") is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
final TermState state = termStates
.get(context.ord);
if (state == null) { // term is not present in that reader
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TopDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TopDocs.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TopDocs.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TopDocs.java Fri Jun 24 23:07:10 2011
@@ -17,15 +17,21 @@ package org.apache.lucene.search;
* limitations under the License.
*/
+import java.io.IOException;
+
+import org.apache.lucene.util.PriorityQueue;
+
/** Represents hits returned by {@link
* IndexSearcher#search(Query,Filter,int)} and {@link
* IndexSearcher#search(Query,int)}. */
public class TopDocs {
- /** The total number of hits for the query.
- */
+
+ /** The total number of hits for the query. */
public int totalHits;
+
/** The top hits for the query. */
public ScoreDoc[] scoreDocs;
+
/** Stores the maximum score value encountered, needed for normalizing. */
private float maxScore;
@@ -34,12 +40,12 @@ public class TopDocs {
* scores are not tracked, this returns {@link Float#NaN}.
*/
public float getMaxScore() {
- return maxScore;
+ return maxScore;
}
/** Sets the maximum score value encountered. */
public void setMaxScore(float maxScore) {
- this.maxScore=maxScore;
+ this.maxScore=maxScore;
}
/** Constructs a TopDocs with a default maxScore=Float.NaN. */
@@ -52,4 +58,199 @@ public class TopDocs {
this.scoreDocs = scoreDocs;
this.maxScore = maxScore;
}
+
+ // Refers to one hit:
+ private static class ShardRef {
+ // Which shard (index into shardHits[]):
+ final int shardIndex;
+
+ // Which hit within the shard:
+ int hitIndex;
+
+ public ShardRef(int shardIndex) {
+ this.shardIndex = shardIndex;
+ }
+
+ @Override
+ public String toString() {
+ return "ShardRef(shardIndex=" + shardIndex + " hitIndex=" + hitIndex + ")";
+ }
+ };
+
+ // Specialized MergeSortQueue that just merges by
+ // relevance score, descending:
+ private static class ScoreMergeSortQueue extends PriorityQueue<ShardRef> {
+ final ScoreDoc[][] shardHits;
+
+ public ScoreMergeSortQueue(TopDocs[] shardHits) {
+ super(shardHits.length);
+ this.shardHits = new ScoreDoc[shardHits.length][];
+ for(int shardIDX=0;shardIDX<shardHits.length;shardIDX++) {
+ this.shardHits[shardIDX] = shardHits[shardIDX].scoreDocs;
+ }
+ }
+
+ // Returns true if first is < second
+ public boolean lessThan(ShardRef first, ShardRef second) {
+ assert first != second;
+ final float firstScore = shardHits[first.shardIndex][first.hitIndex].score;
+ final float secondScore = shardHits[second.shardIndex][second.hitIndex].score;
+
+ if (firstScore < secondScore) {
+ return false;
+ } else if (firstScore > secondScore) {
+ return true;
+ } else {
+ // Tie break: earlier shard wins
+ if (first.shardIndex < second.shardIndex) {
+ return true;
+ } else if (first.shardIndex > second.shardIndex) {
+ return false;
+ } else {
+ // Tie break in same shard: resolve however the
+ // shard had resolved it:
+ assert first.hitIndex != second.hitIndex;
+ return first.hitIndex < second.hitIndex;
+ }
+ }
+ }
+ }
+
+ private static class MergeSortQueue extends PriorityQueue<ShardRef> {
+ // These are really FieldDoc instances:
+ final ScoreDoc[][] shardHits;
+ final FieldComparator[] comparators;
+ final int[] reverseMul;
+
+ public MergeSortQueue(Sort sort, TopDocs[] shardHits) throws IOException {
+ super(shardHits.length);
+ this.shardHits = new ScoreDoc[shardHits.length][];
+ for(int shardIDX=0;shardIDX<shardHits.length;shardIDX++) {
+ final ScoreDoc[] shard = shardHits[shardIDX].scoreDocs;
+ //System.out.println(" init shardIdx=" + shardIDX + " hits=" + shard);
+ if (shard != null) {
+ this.shardHits[shardIDX] = shard;
+ // Fail gracefully if API is misused:
+ for(int hitIDX=0;hitIDX<shard.length;hitIDX++) {
+ final ScoreDoc sd = shard[hitIDX];
+ if (!(sd instanceof FieldDoc)) {
+ throw new IllegalArgumentException("shard " + shardIDX + " was not sorted by the provided Sort (expected FieldDoc but got ScoreDoc)");
+ }
+ final FieldDoc fd = (FieldDoc) sd;
+ if (fd.fields == null) {
+ throw new IllegalArgumentException("shard " + shardIDX + " did not set sort field values (FieldDoc.fields is null); you must pass fillFields=true to IndexSearcher.search on each shard");
+ }
+ }
+ }
+ }
+
+ final SortField[] sortFields = sort.getSort();
+ comparators = new FieldComparator[sortFields.length];
+ reverseMul = new int[sortFields.length];
+ for(int compIDX=0;compIDX<sortFields.length;compIDX++) {
+ final SortField sortField = sortFields[compIDX];
+ comparators[compIDX] = sortField.getComparator(1, compIDX);
+ reverseMul[compIDX] = sortField.getReverse() ? -1 : 1;
+ }
+ }
+
+ // Returns true if first is < second
+ @SuppressWarnings("unchecked")
+ public boolean lessThan(ShardRef first, ShardRef second) {
+ assert first != second;
+ final FieldDoc firstFD = (FieldDoc) shardHits[first.shardIndex][first.hitIndex];
+ final FieldDoc secondFD = (FieldDoc) shardHits[second.shardIndex][second.hitIndex];
+ //System.out.println(" lessThan:\n first=" + first + " doc=" + firstFD.doc + " score=" + firstFD.score + "\n second=" + second + " doc=" + secondFD.doc + " score=" + secondFD.score);
+
+ for(int compIDX=0;compIDX<comparators.length;compIDX++) {
+ final FieldComparator comp = comparators[compIDX];
+ //System.out.println(" cmp idx=" + compIDX + " cmp1=" + firstFD.fields[compIDX] + " cmp2=" + secondFD.fields[compIDX] + " reverse=" + reverseMul[compIDX]);
+
+ final int cmp = reverseMul[compIDX] * comp.compareValues(firstFD.fields[compIDX], secondFD.fields[compIDX]);
+
+ if (cmp != 0) {
+ //System.out.println(" return " + (cmp < 0));
+ return cmp < 0;
+ }
+ }
+
+ // Tie break: earlier shard wins
+ if (first.shardIndex < second.shardIndex) {
+ //System.out.println(" return tb true");
+ return true;
+ } else if (first.shardIndex > second.shardIndex) {
+ //System.out.println(" return tb false");
+ return false;
+ } else {
+ // Tie break in same shard: resolve however the
+ // shard had resolved it:
+ //System.out.println(" return tb " + (first.hitIndex < second.hitIndex));
+ assert first.hitIndex != second.hitIndex;
+ return first.hitIndex < second.hitIndex;
+ }
+ }
+ }
+
+ /** Returns a new TopDocs, containing topN results across
+ * the provided TopDocs, sorting by the specified {@link
+ * Sort}. Each of the TopDocs must have been sorted by
+ * the same Sort, and sort field values must have been
+ * filled (ie, <code>fillFields=true</code> must be
+ * passed to {@link
+ * TopFieldCollector#create}.
+ *
+ * <p>Pass sort=null to merge sort by score descending.
+ *
+ * @lucene.experimental */
+ public static TopDocs merge(Sort sort, int topN, TopDocs[] shardHits) throws IOException {
+
+ final PriorityQueue<ShardRef> queue;
+ if (sort == null) {
+ queue = new ScoreMergeSortQueue(shardHits);
+ } else {
+ queue = new MergeSortQueue(sort, shardHits);
+ }
+
+ int totalHitCount = 0;
+ float maxScore = Float.MIN_VALUE;
+ for(int shardIDX=0;shardIDX<shardHits.length;shardIDX++) {
+ final TopDocs shard = shardHits[shardIDX];
+ if (shard.scoreDocs != null && shard.scoreDocs.length > 0) {
+ totalHitCount += shard.totalHits;
+ queue.add(new ShardRef(shardIDX));
+ maxScore = Math.max(maxScore, shard.getMaxScore());
+ //System.out.println(" maxScore now " + maxScore + " vs " + shard.getMaxScore());
+ }
+ }
+
+ final ScoreDoc[] hits = new ScoreDoc[Math.min(topN, totalHitCount)];
+
+ int hitUpto = 0;
+ while(hitUpto < hits.length) {
+ assert queue.size() > 0;
+ ShardRef ref = queue.pop();
+ final ScoreDoc hit = shardHits[ref.shardIndex].scoreDocs[ref.hitIndex++];
+ if (sort == null) {
+ hits[hitUpto] = new ScoreDoc(hit.doc, hit.score, ref.shardIndex);
+ } else {
+ hits[hitUpto] = new FieldDoc(hit.doc, hit.score, ((FieldDoc) hit).fields, ref.shardIndex);
+ }
+
+ //System.out.println(" hitUpto=" + hitUpto);
+ //System.out.println(" doc=" + hits[hitUpto].doc + " score=" + hits[hitUpto].score);
+
+ hitUpto++;
+
+ if (ref.hitIndex < shardHits[ref.shardIndex].scoreDocs.length) {
+ // Not done with this these TopDocs yet:
+ queue.add(ref);
+ }
+ }
+
+ if (sort == null) {
+ return new TopDocs(totalHitCount, hits, maxScore);
+ } else {
+ return new TopFieldDocs(totalHitCount, hits, sort.getSort(), maxScore);
+ }
+ }
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java Fri Jun 24 23:07:10 2011
@@ -131,12 +131,11 @@ public abstract class TopTermsRewrite<Q
}
});
- final Term placeholderTerm = new Term(query.field);
final Q q = getTopLevelQuery();
final ScoreTerm[] scoreTerms = stQueue.toArray(new ScoreTerm[stQueue.size()]);
ArrayUtil.mergeSort(scoreTerms, scoreTermSortByTermComp);
for (final ScoreTerm st : scoreTerms) {
- final Term term = placeholderTerm.createTerm(st.bytes);
+ final Term term = new Term(query.field, st.bytes);
assert reader.docFreq(term) == st.termState.docFreq() : "reader DF is " + reader.docFreq(term) + " vs " + st.termState.docFreq();
addClause(q, term, st.termState.docFreq(), query.getBoost() * st.boost, st.termState); // add to query
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java Fri Jun 24 23:07:10 2011
@@ -61,8 +61,8 @@ public class ByteValuesCreator extends C
}
@Override
- public int getSortTypeID() {
- return SortField.BYTE;
+ public SortField.Type getSortType() {
+ return SortField.Type.BYTE;
}
//--------------------------------------------------------------------------------
@@ -143,4 +143,4 @@ public class ByteValuesCreator extends C
vals.valid = new Bits.MatchNoBits( maxDoc );
}
}
-}
\ No newline at end of file
+}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java Fri Jun 24 23:07:10 2011
@@ -26,6 +26,7 @@ import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldCache.Parser;
+import org.apache.lucene.search.SortField;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.OpenBitSet;
@@ -73,7 +74,7 @@ public abstract class CachedArrayCreator
/** Return the type that the array will hold */
public abstract Class getArrayType();
public abstract Parser getParser();
- public abstract int getSortTypeID();
+ public abstract SortField.Type getSortType();
protected void setParserAndResetCounts(T value, Parser parser)
{
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java Fri Jun 24 23:07:10 2011
@@ -29,7 +29,6 @@ import org.apache.lucene.search.FieldCac
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.PagedBytes;
-import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.packed.GrowableWriter;
import org.apache.lucene.util.packed.PackedInts;
@@ -66,7 +65,6 @@ public class DocTermsCreator extends Ent
@Override
public DocTerms create(IndexReader reader) throws IOException {
- String field = StringHelper.intern(this.field); // TODO?? necessary?
Terms terms = MultiFields.getTerms(reader, field);
final boolean fasterButMoreRAM = hasOption( FASTER_BUT_MORE_RAM );
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java Fri Jun 24 23:07:10 2011
@@ -34,7 +34,6 @@ import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.PagedBytes;
-import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.packed.GrowableWriter;
import org.apache.lucene.util.packed.PackedInts;
@@ -70,7 +69,6 @@ public class DocTermsIndexCreator extend
@Override
public DocTermsIndex create(IndexReader reader) throws IOException
{
- String field = StringHelper.intern(this.field); // TODO?? necessary?
Terms terms = MultiFields.getTerms(reader, field);
final boolean fasterButMoreRAM = hasOption(FASTER_BUT_MORE_RAM);
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java Fri Jun 24 23:07:10 2011
@@ -61,8 +61,8 @@ public class DoubleValuesCreator extends
}
@Override
- public int getSortTypeID() {
- return SortField.DOUBLE;
+ public SortField.Type getSortType() {
+ return SortField.Type.DOUBLE;
}
//--------------------------------------------------------------------------------
@@ -161,4 +161,4 @@ public class DoubleValuesCreator extends
vals.valid = new Bits.MatchNoBits( maxDoc );
}
}
-}
\ No newline at end of file
+}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java Fri Jun 24 23:07:10 2011
@@ -61,8 +61,8 @@ public class FloatValuesCreator extends
}
@Override
- public int getSortTypeID() {
- return SortField.FLOAT;
+ public SortField.Type getSortType() {
+ return SortField.Type.FLOAT;
}
@@ -162,4 +162,4 @@ public class FloatValuesCreator extends
vals.valid = new Bits.MatchNoBits( maxDoc );
}
}
-}
\ No newline at end of file
+}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java Fri Jun 24 23:07:10 2011
@@ -61,8 +61,8 @@ public class IntValuesCreator extends Ca
}
@Override
- public int getSortTypeID() {
- return SortField.INT;
+ public SortField.Type getSortType() {
+ return SortField.Type.INT;
}
@@ -162,4 +162,4 @@ public class IntValuesCreator extends Ca
vals.valid = new Bits.MatchNoBits( maxDoc );
}
}
-}
\ No newline at end of file
+}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java Fri Jun 24 23:07:10 2011
@@ -61,8 +61,8 @@ public class LongValuesCreator extends C
}
@Override
- public int getSortTypeID() {
- return SortField.LONG;
+ public SortField.Type getSortType() {
+ return SortField.Type.LONG;
}
@@ -162,4 +162,4 @@ public class LongValuesCreator extends C
vals.valid = new Bits.MatchNoBits( maxDoc );
}
}
-}
\ No newline at end of file
+}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java Fri Jun 24 23:07:10 2011
@@ -61,8 +61,8 @@ public class ShortValuesCreator extends
}
@Override
- public int getSortTypeID() {
- return SortField.SHORT;
+ public SortField.Type getSortType() {
+ return SortField.Type.SHORT;
}
@@ -144,4 +144,4 @@ public class ShortValuesCreator extends
vals.valid = new Bits.MatchNoBits( maxDoc );
}
}
-}
\ No newline at end of file
+}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/function/CustomScoreProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/function/CustomScoreProvider.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/function/CustomScoreProvider.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/function/CustomScoreProvider.java Fri Jun 24 23:07:10 2011
@@ -19,7 +19,8 @@ package org.apache.lucene.search.functio
import java.io.IOException;
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader; // for javadocs
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.FieldCache; // for javadocs
@@ -28,7 +29,7 @@ import org.apache.lucene.search.FieldCac
* {@link CustomScoreQuery#getCustomScoreProvider}, if you want
* to modify the custom score calculation of a {@link CustomScoreQuery}.
* <p>Since Lucene 2.9, queries operate on each segment of an index separately,
- * so the protected {@link #reader} field can be used to resolve doc IDs,
+ * so the protected {@link #context} field can be used to resolve doc IDs,
* as the supplied <code>doc</code> ID is per-segment and without knowledge
* of the IndexReader you cannot access the document or {@link FieldCache}.
*
@@ -37,13 +38,13 @@ import org.apache.lucene.search.FieldCac
*/
public class CustomScoreProvider {
- protected final IndexReader reader;
+ protected final AtomicReaderContext context;
/**
* Creates a new instance of the provider class for the given {@link IndexReader}.
*/
- public CustomScoreProvider(IndexReader reader) {
- this.reader = reader;
+ public CustomScoreProvider(AtomicReaderContext context) {
+ this.context = context;
}
/**
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java Fri Jun 24 23:07:10 2011
@@ -175,8 +175,8 @@ public class CustomScoreQuery extends Qu
* implementation as specified in the docs of {@link CustomScoreProvider}.
* @since 2.9.2
*/
- protected CustomScoreProvider getCustomScoreProvider(IndexReader reader) throws IOException {
- return new CustomScoreProvider(reader);
+ protected CustomScoreProvider getCustomScoreProvider(AtomicReaderContext context) throws IOException {
+ return new CustomScoreProvider(context);
}
//=========================== W E I G H T ============================
@@ -187,7 +187,7 @@ public class CustomScoreQuery extends Qu
boolean qStrict;
public CustomWeight(IndexSearcher searcher) throws IOException {
- this.subQueryWeight = subQuery.weight(searcher);
+ this.subQueryWeight = subQuery.createWeight(searcher);
this.valSrcWeights = new Weight[valSrcQueries.length];
for(int i = 0; i < valSrcQueries.length; i++) {
this.valSrcWeights[i] = valSrcQueries[i].createWeight(searcher);
@@ -251,7 +251,7 @@ public class CustomScoreQuery extends Qu
for(int i = 0; i < valSrcScorers.length; i++) {
valSrcScorers[i] = valSrcWeights[i].scorer(context, scorerContext.scoreDocsInOrder(true));
}
- return new CustomScorer(context.reader, this, subQueryScorer, valSrcScorers);
+ return new CustomScorer(CustomScoreQuery.this.getCustomScoreProvider(context), this, subQueryScorer, valSrcScorers);
}
@Override
@@ -270,7 +270,7 @@ public class CustomScoreQuery extends Qu
for(int i = 0; i < valSrcWeights.length; i++) {
valSrcExpls[i] = valSrcWeights[i].explain(info, doc);
}
- Explanation customExp = CustomScoreQuery.this.getCustomScoreProvider(info.reader).customExplain(doc,subQueryExpl,valSrcExpls);
+ Explanation customExp = CustomScoreQuery.this.getCustomScoreProvider(info).customExplain(doc,subQueryExpl,valSrcExpls);
float sc = getValue() * customExp.getValue();
Explanation res = new ComplexExplanation(
true, sc, CustomScoreQuery.this.toString() + ", product of:");
@@ -300,14 +300,14 @@ public class CustomScoreQuery extends Qu
private float vScores[]; // reused in score() to avoid allocating this array for each doc
// constructor
- private CustomScorer(IndexReader reader, CustomWeight w,
+ private CustomScorer(CustomScoreProvider provider, CustomWeight w,
Scorer subQueryScorer, Scorer[] valSrcScorers) throws IOException {
super(w);
this.qWeight = w.getValue();
this.subQueryScorer = subQueryScorer;
this.valSrcScorers = valSrcScorers;
this.vScores = new float[valSrcScorers.length];
- this.provider = CustomScoreQuery.this.getCustomScoreProvider(reader);
+ this.provider = provider;
}
@Override
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java Fri Jun 24 23:07:10 2011
@@ -17,6 +17,8 @@ package org.apache.lucene.store;
* limitations under the License.
*/
+import org.apache.lucene.util.BytesRef;
+
/** @lucene.experimental */
public final class ByteArrayDataInput extends DataInput {
@@ -25,9 +27,16 @@ public final class ByteArrayDataInput ex
private int pos;
private int limit;
- // TODO: allow BytesRef (slice) too
public ByteArrayDataInput(byte[] bytes) {
- this.bytes = bytes;
+ reset(bytes);
+ }
+
+ public ByteArrayDataInput(byte[] bytes, int offset, int len) {
+ reset(bytes, offset, len);
+ }
+
+ public ByteArrayDataInput() {
+ reset(BytesRef.EMPTY_BYTES);
}
public void reset(byte[] bytes) {
@@ -41,7 +50,7 @@ public final class ByteArrayDataInput ex
public void reset(byte[] bytes, int offset, int len) {
this.bytes = bytes;
pos = offset;
- limit = len;
+ limit = offset + len;
}
public boolean eof() {
@@ -59,12 +68,14 @@ public final class ByteArrayDataInput ex
@Override
public int readInt() {
+ assert pos+4 <= limit;
return ((bytes[pos++] & 0xFF) << 24) | ((bytes[pos++] & 0xFF) << 16)
| ((bytes[pos++] & 0xFF) << 8) | (bytes[pos++] & 0xFF);
}
@Override
public long readLong() {
+ assert pos+8 <= limit;
final int i1 = ((bytes[pos++] & 0xff) << 24) | ((bytes[pos++] & 0xff) << 16) |
((bytes[pos++] & 0xff) << 8) | (bytes[pos++] & 0xff);
final int i2 = ((bytes[pos++] & 0xff) << 24) | ((bytes[pos++] & 0xff) << 16) |
@@ -74,9 +85,11 @@ public final class ByteArrayDataInput ex
@Override
public int readVInt() {
+ checkBounds();
byte b = bytes[pos++];
int i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
+ checkBounds();
b = bytes[pos++];
i |= (b & 0x7F) << shift;
}
@@ -85,9 +98,11 @@ public final class ByteArrayDataInput ex
@Override
public long readVLong() {
+ checkBounds();
byte b = bytes[pos++];
long i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
+ checkBounds();
b = bytes[pos++];
i |= (b & 0x7FL) << shift;
}
@@ -97,7 +112,7 @@ public final class ByteArrayDataInput ex
// NOTE: AIOOBE not EOF if you read too much
@Override
public byte readByte() {
- assert pos < limit;
+ checkBounds();
return bytes[pos++];
}
@@ -108,4 +123,9 @@ public final class ByteArrayDataInput ex
System.arraycopy(bytes, pos, b, offset, len);
pos += len;
}
+
+ private boolean checkBounds() {
+ assert pos < limit;
+ return true;
+ }
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/DataOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/DataOutput.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/DataOutput.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/DataOutput.java Fri Jun 24 23:07:10 2011
@@ -60,6 +60,14 @@ public abstract class DataOutput {
writeByte((byte)(i >> 8));
writeByte((byte) i);
}
+
+ /** Writes a short as two bytes.
+ * @see DataInput#readShort()
+ */
+ public void writeShort(short i) throws IOException {
+ writeByte((byte)(i >> 8));
+ writeByte((byte) i);
+ }
/** Writes an int in a variable-length format. Writes between one and
* five bytes. Smaller values take fewer bytes. Negative numbers are not
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/Directory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/Directory.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/Directory.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/Directory.java Fri Jun 24 23:07:10 2011
@@ -112,11 +112,35 @@ public abstract class Directory implemen
* implementation may ignore the buffer size. Currently
* the only Directory implementations that respect this
* parameter are {@link FSDirectory} and {@link
- * org.apache.lucene.index.CompoundFileReader}.
+ * CompoundFileDirectory}.
*/
public IndexInput openInput(String name, int bufferSize) throws IOException {
return openInput(name);
}
+
+ /**
+ * Returns a {@link CompoundFileDirectory} capable of
+ * reading the Lucene compound file format.
+ * <p>
+ * The default implementation returns
+ * {@link DefaultCompoundFileDirectory}.
+ * @lucene.experimental
+ */
+ public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+ return new DefaultCompoundFileDirectory(this, name, bufferSize, false);
+ }
+
+ /**
+ * Returns a {@link CompoundFileDirectory} capable of
+ * writing the Lucene compound file format.
+ * <p>
+ * The default implementation returns
+ * {@link DefaultCompoundFileDirectory}.
+ * @lucene.experimental
+ */
+ public CompoundFileDirectory createCompoundOutput(String name) throws IOException {
+ return new DefaultCompoundFileDirectory(this, name, 1024, true);
+ }
/** Construct a {@link Lock}.
* @param name the name of the lock file
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/FSDirectory.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/FSDirectory.java Fri Jun 24 23:07:10 2011
@@ -173,7 +173,7 @@ public abstract class FSDirectory extend
/** Just like {@link #open(File)}, but allows you to
* also specify a custom {@link LockFactory}. */
public static FSDirectory open(File path, LockFactory lockFactory) throws IOException {
- if ((Constants.WINDOWS || Constants.SUN_OS)
+ if ((Constants.WINDOWS || Constants.SUN_OS || Constants.LINUX)
&& Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
return new MMapDirectory(path, lockFactory);
} else if (Constants.WINDOWS) {
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java Fri Jun 24 23:07:10 2011
@@ -148,4 +148,14 @@ public class FileSwitchDirectory extends
public IndexInput openInput(String name) throws IOException {
return getDirectory(name).openInput(name);
}
+
+ @Override
+ public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+ return getDirectory(name).openCompoundInput(name, bufferSize);
+ }
+
+ @Override
+ public CompoundFileDirectory createCompoundOutput(String name) throws IOException {
+ return getDirectory(name).createCompoundOutput(name);
+ }
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/IndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/IndexOutput.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/IndexOutput.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/IndexOutput.java Fri Jun 24 23:07:10 2011
@@ -58,4 +58,5 @@ public abstract class IndexOutput extend
* @param length file length
*/
public void setLength(long length) throws IOException {}
+
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/MMapDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/MMapDirectory.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/MMapDirectory.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/MMapDirectory.java Fri Jun 24 23:07:10 2011
@@ -32,6 +32,7 @@ import java.security.PrivilegedActionExc
import java.lang.reflect.Method;
import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.IOUtils;
/** File-based {@link Directory} implementation that uses
* mmap for reading, and {@link
@@ -79,8 +80,8 @@ import org.apache.lucene.util.Constants;
*/
public class MMapDirectory extends FSDirectory {
private boolean useUnmapHack = UNMAP_SUPPORTED;
- public static final int DEFAULT_MAX_BUFF = Constants.JRE_IS_64BIT ? Integer.MAX_VALUE : (256 * 1024 * 1024);
- private int maxBBuf = DEFAULT_MAX_BUFF;
+ public static final int DEFAULT_MAX_BUFF = Constants.JRE_IS_64BIT ? (1 << 30) : (1 << 28);
+ private int chunkSizePower;
/** Create a new MMapDirectory for the named location.
*
@@ -91,6 +92,7 @@ public class MMapDirectory extends FSDir
*/
public MMapDirectory(File path, LockFactory lockFactory) throws IOException {
super(path, lockFactory);
+ setMaxChunkSize(DEFAULT_MAX_BUFF);
}
/** Create a new MMapDirectory for the named location and {@link NativeFSLockFactory}.
@@ -100,6 +102,7 @@ public class MMapDirectory extends FSDir
*/
public MMapDirectory(File path) throws IOException {
super(path, null);
+ setMaxChunkSize(DEFAULT_MAX_BUFF);
}
/**
@@ -180,23 +183,28 @@ public class MMapDirectory extends FSDir
* Especially on 32 bit platform, the address space can be very fragmented,
* so large index files cannot be mapped.
* Using a lower chunk size makes the directory implementation a little
- * bit slower (as the correct chunk must be resolved on each seek)
+ * bit slower (as the correct chunk may be resolved on lots of seeks)
* but the chance is higher that mmap does not fail. On 64 bit
- * Java platforms, this parameter should always be {@link Integer#MAX_VALUE},
+ * Java platforms, this parameter should always be {@code 1 << 30},
* as the address space is big enough.
+ * <b>Please note:</b> This method always rounds down the chunk size
+ * to a power of 2.
*/
- public void setMaxChunkSize(final int maxBBuf) {
- if (maxBBuf<=0)
+ public final void setMaxChunkSize(final int maxChunkSize) {
+ if (maxChunkSize <= 0)
throw new IllegalArgumentException("Maximum chunk size for mmap must be >0");
- this.maxBBuf=maxBBuf;
+ //System.out.println("Requested chunk size: "+maxChunkSize);
+ this.chunkSizePower = 31 - Integer.numberOfLeadingZeros(maxChunkSize);
+ assert this.chunkSizePower >= 0 && this.chunkSizePower <= 30;
+ //System.out.println("Got chunk size: "+getMaxChunkSize());
}
/**
* Returns the current mmap chunk size.
* @see #setMaxChunkSize
*/
- public int getMaxChunkSize() {
- return maxBBuf;
+ public final int getMaxChunkSize() {
+ return 1 << chunkSizePower;
}
/** Creates an IndexInput for the file with the given name. */
@@ -206,103 +214,46 @@ public class MMapDirectory extends FSDir
File f = new File(getDirectory(), name);
RandomAccessFile raf = new RandomAccessFile(f, "r");
try {
- return (raf.length() <= maxBBuf)
- ? (IndexInput) new MMapIndexInput(raf)
- : (IndexInput) new MultiMMapIndexInput(raf, maxBBuf);
+ return new MMapIndexInput(raf, 0, raf.length(), chunkSizePower);
} finally {
raf.close();
}
}
+
+ @Override
+ public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+ return new MMapCompoundFileDirectory(name, bufferSize);
+ }
+
+ private final class MMapCompoundFileDirectory extends CompoundFileDirectory {
+ private RandomAccessFile raf = null;
- private class MMapIndexInput extends IndexInput {
-
- private ByteBuffer buffer;
- private final long length;
- private boolean isClone = false;
-
- private MMapIndexInput(RandomAccessFile raf) throws IOException {
- this.length = raf.length();
- this.buffer = raf.getChannel().map(MapMode.READ_ONLY, 0, length);
- }
-
- @Override
- public byte readByte() throws IOException {
- try {
- return buffer.get();
- } catch (BufferUnderflowException e) {
- throw new IOException("read past EOF");
- }
- }
-
- @Override
- public void readBytes(byte[] b, int offset, int len) throws IOException {
- try {
- buffer.get(b, offset, len);
- } catch (BufferUnderflowException e) {
- throw new IOException("read past EOF");
- }
- }
-
- @Override
- public short readShort() throws IOException {
- try {
- return buffer.getShort();
- } catch (BufferUnderflowException e) {
- throw new IOException("read past EOF");
- }
- }
-
- @Override
- public int readInt() throws IOException {
- try {
- return buffer.getInt();
- } catch (BufferUnderflowException e) {
- throw new IOException("read past EOF");
- }
- }
-
- @Override
- public long readLong() throws IOException {
+ public MMapCompoundFileDirectory(String fileName, int readBufferSize) throws IOException {
+ super(MMapDirectory.this, fileName, readBufferSize);
+ IndexInput stream = null;
try {
- return buffer.getLong();
- } catch (BufferUnderflowException e) {
- throw new IOException("read past EOF");
+ File f = new File(MMapDirectory.this.getDirectory(), fileName);
+ raf = new RandomAccessFile(f, "r");
+ stream = new MMapIndexInput(raf, 0, raf.length(), chunkSizePower);
+ initForRead(CompoundFileDirectory.readEntries(stream, MMapDirectory.this, fileName));
+ stream.close();
+ } catch (IOException e) {
+ // throw our original exception
+ IOUtils.closeSafely(e, raf, stream);
}
}
-
- @Override
- public long getFilePointer() {
- return buffer.position();
- }
-
- @Override
- public void seek(long pos) throws IOException {
- buffer.position((int)pos);
- }
@Override
- public long length() {
- return length;
+ public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException {
+ return new MMapIndexInput(raf, offset, length, chunkSizePower);
}
@Override
- public Object clone() {
- if (buffer == null)
- throw new AlreadyClosedException("MMapIndexInput already closed");
- MMapIndexInput clone = (MMapIndexInput)super.clone();
- clone.isClone = true;
- clone.buffer = buffer.duplicate();
- return clone;
- }
-
- @Override
- public void close() throws IOException {
- // unmap the buffer (if enabled) and at least unset it for GC
+ public synchronized void close() throws IOException {
try {
- if (isClone || buffer == null) return;
- cleanMapping(buffer);
+ raf.close();
} finally {
- buffer = null;
+ super.close();
}
}
}
@@ -310,48 +261,46 @@ public class MMapDirectory extends FSDir
// Because Java's ByteBuffer uses an int to address the
// values, it's necessary to access a file >
// Integer.MAX_VALUE in size using multiple byte buffers.
- private class MultiMMapIndexInput extends IndexInput {
+ private final class MMapIndexInput extends IndexInput {
private ByteBuffer[] buffers;
- private int[] bufSizes; // keep here, ByteBuffer.size() method is optional
- private final long length;
+ private final long length, chunkSizeMask, chunkSize;
+ private final int chunkSizePower;
private int curBufIndex;
- private final int maxBufSize;
private ByteBuffer curBuf; // redundant for speed: buffers[curBufIndex]
private boolean isClone = false;
- public MultiMMapIndexInput(RandomAccessFile raf, int maxBufSize)
- throws IOException {
- this.length = raf.length();
- this.maxBufSize = maxBufSize;
+ MMapIndexInput(RandomAccessFile raf, long offset, long length, int chunkSizePower) throws IOException {
+ this.length = length;
+ this.chunkSizePower = chunkSizePower;
+ this.chunkSize = 1L << chunkSizePower;
+ this.chunkSizeMask = chunkSize - 1L;
+
+ if (chunkSizePower < 0 || chunkSizePower > 30)
+ throw new IllegalArgumentException("Invalid chunkSizePower used for ByteBuffer size: " + chunkSizePower);
- if (maxBufSize <= 0)
- throw new IllegalArgumentException("Non positive maxBufSize: "
- + maxBufSize);
+ if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)
+ throw new IllegalArgumentException("RandomAccessFile too big for chunk size: " + raf.toString());
- if ((length / maxBufSize) > Integer.MAX_VALUE)
- throw new IllegalArgumentException
- ("RandomAccessFile too big for maximum buffer size: "
- + raf.toString());
+ // we always allocate one more buffer, the last one may be a 0 byte one
+ final int nrBuffers = (int) (length >>> chunkSizePower) + 1;
- int nrBuffers = (int) (length / maxBufSize);
- if (((long) nrBuffers * maxBufSize) <= length) nrBuffers++;
+ //System.out.println("length="+length+", chunkSizePower=" + chunkSizePower + ", chunkSizeMask=" + chunkSizeMask + ", nrBuffers=" + nrBuffers);
this.buffers = new ByteBuffer[nrBuffers];
- this.bufSizes = new int[nrBuffers];
- long bufferStart = 0;
+ long bufferStart = 0L;
FileChannel rafc = raf.getChannel();
for (int bufNr = 0; bufNr < nrBuffers; bufNr++) {
- int bufSize = (length > (bufferStart + maxBufSize))
- ? maxBufSize
- : (int) (length - bufferStart);
- this.buffers[bufNr] = rafc.map(MapMode.READ_ONLY,bufferStart,bufSize);
- this.bufSizes[bufNr] = bufSize;
+ int bufSize = (int) ( (length > (bufferStart + chunkSize))
+ ? chunkSize
+ : (length - bufferStart)
+ );
+ this.buffers[bufNr] = rafc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize);
bufferStart += bufSize;
}
seek(0L);
@@ -362,11 +311,13 @@ public class MMapDirectory extends FSDir
try {
return curBuf.get();
} catch (BufferUnderflowException e) {
- curBufIndex++;
- if (curBufIndex >= buffers.length)
- throw new IOException("read past EOF");
- curBuf = buffers[curBufIndex];
- curBuf.position(0);
+ do {
+ curBufIndex++;
+ if (curBufIndex >= buffers.length)
+ throw new IOException("read past EOF");
+ curBuf = buffers[curBufIndex];
+ curBuf.position(0);
+ } while (!curBuf.hasRemaining());
return curBuf.get();
}
}
@@ -421,15 +372,28 @@ public class MMapDirectory extends FSDir
@Override
public long getFilePointer() {
- return ((long) curBufIndex * maxBufSize) + curBuf.position();
+ return (((long) curBufIndex) << chunkSizePower) + curBuf.position();
}
@Override
public void seek(long pos) throws IOException {
- curBufIndex = (int) (pos / maxBufSize);
- curBuf = buffers[curBufIndex];
- int bufOffset = (int) (pos - ((long) curBufIndex * maxBufSize));
- curBuf.position(bufOffset);
+ // we use >> here to preserve negative, so we will catch AIOOBE:
+ final int bi = (int) (pos >> chunkSizePower);
+ try {
+ final ByteBuffer b = buffers[bi];
+ b.position((int) (pos & chunkSizeMask));
+ // write values, on exception all is unchanged
+ this.curBufIndex = bi;
+ this.curBuf = b;
+ } catch (ArrayIndexOutOfBoundsException aioobe) {
+ if (pos < 0L)
+ throw new IllegalArgumentException("Seeking to negative position");
+ throw new IOException("seek past EOF");
+ } catch (IllegalArgumentException iae) {
+ if (pos < 0L)
+ throw new IllegalArgumentException("Seeking to negative position");
+ throw new IOException("seek past EOF");
+ }
}
@Override
@@ -440,11 +404,10 @@ public class MMapDirectory extends FSDir
@Override
public Object clone() {
if (buffers == null)
- throw new AlreadyClosedException("MultiMMapIndexInput already closed");
- MultiMMapIndexInput clone = (MultiMMapIndexInput)super.clone();
+ throw new AlreadyClosedException("MMapIndexInput already closed");
+ final MMapIndexInput clone = (MMapIndexInput)super.clone();
clone.isClone = true;
clone.buffers = new ByteBuffer[buffers.length];
- // No need to clone bufSizes.
// Since most clones will use only one buffer, duplicate() could also be
// done lazy in clones, e.g. when adapting curBuf.
for (int bufNr = 0; bufNr < buffers.length; bufNr++) {
@@ -453,9 +416,7 @@ public class MMapDirectory extends FSDir
try {
clone.seek(getFilePointer());
} catch(IOException ioe) {
- RuntimeException newException = new RuntimeException(ioe);
- newException.initCause(ioe);
- throw newException;
+ throw new RuntimeException("Should never happen", ioe);
}
return clone;
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java Fri Jun 24 23:07:10 2011
@@ -24,6 +24,9 @@ import java.nio.channels.ClosedChannelEx
import java.nio.channels.FileChannel;
import java.util.concurrent.Future; // javadoc
+import org.apache.lucene.store.SimpleFSDirectory.SimpleFSIndexInput;
+import org.apache.lucene.util.IOUtils;
+
/**
* An {@link FSDirectory} implementation that uses java.nio's FileChannel's
* positional read, which allows multiple threads to read from the same file
@@ -77,6 +80,47 @@ public class NIOFSDirectory extends FSDi
ensureOpen();
return new NIOFSIndexInput(new File(getDirectory(), name), bufferSize, getReadChunkSize());
}
+
+ @Override
+ public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+ return new NIOFSCompoundFileDirectory(name, bufferSize);
+ }
+
+ private final class NIOFSCompoundFileDirectory extends CompoundFileDirectory {
+ private SimpleFSIndexInput.Descriptor fd;
+ private FileChannel fc;
+
+ public NIOFSCompoundFileDirectory(String fileName, int readBufferSize) throws IOException {
+ super(NIOFSDirectory.this, fileName, readBufferSize);
+ IndexInput stream = null;
+ try {
+ File f = new File(NIOFSDirectory.this.getDirectory(), fileName);
+ fd = new SimpleFSIndexInput.Descriptor(f, "r");
+ fc = fd.getChannel();
+ stream = new NIOFSIndexInput(fd, fc, 0, fd.length, readBufferSize,
+ getReadChunkSize());
+ initForRead(CompoundFileDirectory.readEntries(stream, NIOFSDirectory.this, fileName));
+ stream.close();
+ } catch (IOException e) {
+ // throw our original exception
+ IOUtils.closeSafely(e, fc, fd, stream);
+ }
+ }
+
+ @Override
+ public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException {
+ return new NIOFSIndexInput(fd, fc, offset, length, readBufferSize, getReadChunkSize());
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ try {
+ IOUtils.closeSafely(false, fc, fd);
+ } finally {
+ super.close();
+ }
+ }
+ }
protected static class NIOFSIndexInput extends SimpleFSDirectory.SimpleFSIndexInput {
@@ -91,6 +135,12 @@ public class NIOFSDirectory extends FSDi
super(path, bufferSize, chunkSize);
channel = file.getChannel();
}
+
+ public NIOFSIndexInput(Descriptor file, FileChannel fc, long off, long length, int bufferSize, int chunkSize) throws IOException {
+ super(file, off, length, bufferSize, chunkSize);
+ channel = fc;
+ isClone = true;
+ }
@Override
protected void newBuffer(byte[] newBuffer) {
@@ -145,7 +195,11 @@ public class NIOFSDirectory extends FSDi
int readLength = bb.limit() - readOffset;
assert readLength == len;
- long pos = getFilePointer();
+ long pos = getFilePointer() + off;
+
+ if (pos + len > end) {
+ throw new IOException("read past EOF");
+ }
try {
while (readLength > 0) {
@@ -159,9 +213,6 @@ public class NIOFSDirectory extends FSDi
}
bb.limit(limit);
int i = channel.read(bb, pos);
- if (i == -1) {
- throw new IOException("read past EOF");
- }
pos += i;
readOffset += i;
readLength -= i;
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/RAMOutputStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/RAMOutputStream.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/RAMOutputStream.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/RAMOutputStream.java Fri Jun 24 23:07:10 2011
@@ -67,6 +67,26 @@ public class RAMOutputStream extends Ind
}
}
+ /** Copy the current contents of this buffer to output
+ * byte array */
+ public void writeTo(byte[] bytes, int offset) throws IOException {
+ flush();
+ final long end = file.length;
+ long pos = 0;
+ int buffer = 0;
+ int bytesUpto = offset;
+ while (pos < end) {
+ int length = BUFFER_SIZE;
+ long nextPos = pos + length;
+ if (nextPos > end) { // at the last buffer
+ length = (int)(end - pos);
+ }
+ System.arraycopy(file.getBuffer(buffer++), 0, bytes, bytesUpto, length);
+ bytesUpto += length;
+ pos = nextPos;
+ }
+ }
+
/** Resets this to an empty file. */
public void reset() {
currentBuffer = null;
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java Fri Jun 24 23:07:10 2011
@@ -21,6 +21,8 @@ import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
+import org.apache.lucene.util.IOUtils;
+
/** A straightforward implementation of {@link FSDirectory}
* using java.io.RandomAccessFile. However, this class has
* poor concurrent performance (multiple threads will
@@ -55,6 +57,45 @@ public class SimpleFSDirectory extends F
ensureOpen();
return new SimpleFSIndexInput(new File(directory, name), bufferSize, getReadChunkSize());
}
+
+ @Override
+ public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+ return new SimpleFSCompoundFileDirectory(name, bufferSize);
+ }
+
+ private final class SimpleFSCompoundFileDirectory extends CompoundFileDirectory {
+ private SimpleFSIndexInput.Descriptor fd;
+
+ public SimpleFSCompoundFileDirectory(String fileName, int readBufferSize) throws IOException {
+ super(SimpleFSDirectory.this, fileName, readBufferSize);
+ IndexInput stream = null;
+ try {
+ final File f = new File(SimpleFSDirectory.this.getDirectory(), fileName);
+ fd = new SimpleFSIndexInput.Descriptor(f, "r");
+ stream = new SimpleFSIndexInput(fd, 0, fd.length, readBufferSize,
+ getReadChunkSize());
+ initForRead(CompoundFileDirectory.readEntries(stream, SimpleFSDirectory.this, fileName));
+ stream.close();
+ } catch (IOException e) {
+ // throw our original exception
+ IOUtils.closeSafely(e, fd, stream);
+ }
+ }
+
+ @Override
+ public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException {
+ return new SimpleFSIndexInput(fd, offset, length, readBufferSize, getReadChunkSize());
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ try {
+ fd.close();
+ } finally {
+ super.close();
+ }
+ }
+ }
protected static class SimpleFSIndexInput extends BufferedIndexInput {
@@ -84,11 +125,24 @@ public class SimpleFSDirectory extends F
boolean isClone;
// LUCENE-1566 - maximum read length on a 32bit JVM to prevent incorrect OOM
protected final int chunkSize;
+ protected final long off;
+ protected final long end;
public SimpleFSIndexInput(File path, int bufferSize, int chunkSize) throws IOException {
super(bufferSize);
- file = new Descriptor(path, "r");
+ this.file = new Descriptor(path, "r");
this.chunkSize = chunkSize;
+ this.off = 0L;
+ this.end = file.length;
+ }
+
+ public SimpleFSIndexInput(Descriptor file, long off, long length, int bufferSize, int chunkSize) throws IOException {
+ super(bufferSize);
+ this.file = file;
+ this.chunkSize = chunkSize;
+ this.off = off;
+ this.end = off + length;
+ this.isClone = true; // well, we are sorta?
}
/** IndexInput methods */
@@ -96,13 +150,17 @@ public class SimpleFSDirectory extends F
protected void readInternal(byte[] b, int offset, int len)
throws IOException {
synchronized (file) {
- long position = getFilePointer();
+ long position = off + getFilePointer();
if (position != file.position) {
file.seek(position);
file.position = position;
}
int total = 0;
+ if (position + len > end) {
+ throw new IOException("read past EOF");
+ }
+
try {
do {
final int readLength;
@@ -113,9 +171,6 @@ public class SimpleFSDirectory extends F
readLength = chunkSize;
}
final int i = file.read(b, offset + total, readLength);
- if (i == -1) {
- throw new IOException("read past EOF");
- }
file.position += i;
total += i;
} while (total < len);
@@ -144,7 +199,7 @@ public class SimpleFSDirectory extends F
@Override
public long length() {
- return file.length;
+ return end - off;
}
@Override
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/ArrayUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/ArrayUtil.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/ArrayUtil.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/ArrayUtil.java Fri Jun 24 23:07:10 2011
@@ -255,6 +255,19 @@ public final class ArrayUtil {
return grow(array, 1 + array.length);
}
+ public static double[] grow(double[] array, int minSize) {
+ if (array.length < minSize) {
+ double[] newArray = new double[oversize(minSize, RamUsageEstimator.NUM_BYTES_DOUBLE)];
+ System.arraycopy(array, 0, newArray, 0, array.length);
+ return newArray;
+ } else
+ return array;
+ }
+
+ public static double[] grow(double[] array) {
+ return grow(array, 1 + array.length);
+ }
+
public static short[] shrink(short[] array, int targetSize) {
final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_SHORT);
if (newSize != array.length) {
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/BitUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/BitUtil.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/BitUtil.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/BitUtil.java Fri Jun 24 23:07:10 2011
@@ -778,6 +778,28 @@ public final class BitUtil {
return n - (y & 1);
}
+ /** table of number of leading zeros in a byte */
+ public static final byte[] nlzTable = {8,7,6,6,5,5,5,5,4,4,4,4,4,4,4,4,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+
+ /** Returns the number of leading zero bits.
+ */
+ public static int nlz(long x) {
+ int n = 0;
+ // do the first step as a long
+ int y = (int)(x>>>32);
+ if (y==0) {n+=32; y = (int)(x); }
+ if ((y & 0xFFFF0000) == 0) { n+=16; y<<=16; }
+ if ((y & 0xFF000000) == 0) { n+=8; y<<=8; }
+ return n + nlzTable[y >>> 24];
+ /* implementation without table:
+ if ((y & 0xF0000000) == 0) { n+=4; y<<=4; }
+ if ((y & 0xC0000000) == 0) { n+=2; y<<=2; }
+ if ((y & 0x80000000) == 0) { n+=1; y<<=1; }
+ if ((y & 0x80000000) == 0) { n+=1;}
+ return n;
+ */
+ }
+
/** returns true if v is a power of two or zero*/
public static boolean isPowerOfTwo(int v) {
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/ByteBlockPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/ByteBlockPool.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/ByteBlockPool.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/ByteBlockPool.java Fri Jun 24 23:07:10 2011
@@ -18,6 +18,8 @@ package org.apache.lucene.util;
*/
import java.util.Arrays;
import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
/**
@@ -78,6 +80,33 @@ public final class ByteBlockPool {
}
}
+
+ public static class DirectTrackingAllocator extends Allocator {
+ private final AtomicLong bytesUsed;
+
+ public DirectTrackingAllocator(AtomicLong bytesUsed) {
+ this(BYTE_BLOCK_SIZE, bytesUsed);
+ }
+
+ public DirectTrackingAllocator(int blockSize, AtomicLong bytesUsed) {
+ super(blockSize);
+ this.bytesUsed = bytesUsed;
+ }
+
+ public byte[] getByteBlock() {
+ bytesUsed.addAndGet(blockSize);
+ return new byte[blockSize];
+ }
+ @Override
+ public void recycleByteBlocks(byte[][] blocks, int start, int end) {
+ bytesUsed.addAndGet(-((end-start)* blockSize));
+ for (int i = start; i < end; i++) {
+ blocks[i] = null;
+ }
+ }
+
+ };
+
public byte[][] buffers = new byte[10][];
@@ -92,6 +121,20 @@ public final class ByteBlockPool {
public ByteBlockPool(Allocator allocator) {
this.allocator = allocator;
}
+
+ public void dropBuffersAndReset() {
+ if (bufferUpto != -1) {
+ // Recycle all but the first buffer
+ allocator.recycleByteBlocks(buffers, 0, 1+bufferUpto);
+
+ // Re-use the first buffer
+ bufferUpto = -1;
+ byteUpto = BYTE_BLOCK_SIZE;
+ byteOffset = -BYTE_BLOCK_SIZE;
+ buffers = new byte[10][];
+ buffer = null;
+ }
+ }
public void reset() {
if (bufferUpto != -1) {
@@ -115,7 +158,7 @@ public final class ByteBlockPool {
buffer = buffers[0];
}
}
-
+
public void nextBuffer() {
if (1+bufferUpto == buffers.length) {
byte[][] newBuffers = new byte[ArrayUtil.oversize(buffers.length+1,
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/BytesRefHash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/BytesRefHash.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/BytesRefHash.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/BytesRefHash.java Fri Jun 24 23:07:10 2011
@@ -227,8 +227,9 @@ public final class BytesRefHash {
public void clear(boolean resetPool) {
lastCount = count;
count = 0;
- if (resetPool)
- pool.reset();
+ if (resetPool) {
+ pool.dropBuffersAndReset();
+ }
bytesStart = bytesStartArray.clear();
if (lastCount != -1 && shrink(lastCount)) {
// shrink clears the hash entries
@@ -240,6 +241,16 @@ public final class BytesRefHash {
public void clear() {
clear(true);
}
+
+ /**
+ * Closes the BytesRefHash and releases all internally used memory
+ */
+ public void close() {
+ clear(true);
+ ords = null;
+ bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT
+ * -hashSize);
+ }
/**
* Adds a new {@link BytesRef}
@@ -332,6 +343,7 @@ public final class BytesRefHash {
// 1 byte to store length
buffer[bufferUpto] = (byte) length;
pool.byteUpto += length + 1;
+ assert length >= 0: "Length must be positive: " + length;
System.arraycopy(bytes.bytes, bytes.offset, buffer, bufferUpto + 1,
length);
} else {
@@ -452,8 +464,14 @@ public final class BytesRefHash {
* effect.
*/
public void reinit() {
- if (bytesStart == null)
+ if (bytesStart == null) {
bytesStart = bytesStartArray.init();
+ }
+
+ if (ords == null) {
+ ords = new int[hashSize];
+ bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT * hashSize);
+ }
}
/**
@@ -514,17 +532,62 @@ public final class BytesRefHash {
*/
public abstract AtomicLong bytesUsed();
}
+
+ /**
+ * A direct {@link BytesStartArray} that tracks all memory allocation using an {@link AtomicLong} instance.
+ */
+ public static class TrackingDirectBytesStartArray extends BytesStartArray {
+ protected final int initSize;
+ private int[] bytesStart;
+ protected final AtomicLong bytesUsed;
+
+ public TrackingDirectBytesStartArray(int initSize, AtomicLong bytesUsed) {
+ this.initSize = initSize;
+ this.bytesUsed = bytesUsed;
+ }
- public static class DirectBytesStartArray extends BytesStartArray {
+ @Override
+ public int[] clear() {
+ if (bytesStart != null) {
+ bytesUsed.addAndGet(-bytesStart.length * RamUsageEstimator.NUM_BYTES_INT);
+ }
+ return bytesStart = null;
+ }
+ @Override
+ public int[] grow() {
+ assert bytesStart != null;
+ final int oldSize = bytesStart.length;
+ bytesStart = ArrayUtil.grow(bytesStart, bytesStart.length + 1);
+ bytesUsed.addAndGet((bytesStart.length - oldSize) * RamUsageEstimator.NUM_BYTES_INT);
+ return bytesStart;
+ }
+
+ @Override
+ public int[] init() {
+ bytesStart = new int[ArrayUtil.oversize(initSize,
+ RamUsageEstimator.NUM_BYTES_INT)];
+ bytesUsed.addAndGet((bytesStart.length) * RamUsageEstimator.NUM_BYTES_INT);
+ return bytesStart;
+ }
+
+ @Override
+ public AtomicLong bytesUsed() {
+ return bytesUsed;
+ }
+ }
+
+ public static class DirectBytesStartArray extends BytesStartArray {
protected final int initSize;
private int[] bytesStart;
- private final AtomicLong bytesUsed = new AtomicLong(0);
-
+ private final AtomicLong bytesUsed;
+
public DirectBytesStartArray(int initSize) {
+ this.bytesUsed = new AtomicLong(0);
this.initSize = initSize;
}
+
@Override
public int[] clear() {
return bytesStart = null;
@@ -546,6 +609,5 @@ public final class BytesRefHash {
public AtomicLong bytesUsed() {
return bytesUsed;
}
-
}
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java Fri Jun 24 23:07:10 2011
@@ -208,7 +208,7 @@ public final class FieldCacheSanityCheck
if (seen.contains(rf)) continue;
- List<Object> kids = getAllDecendentReaderKeys(rf.readerKey);
+ List<Object> kids = getAllDescendentReaderKeys(rf.readerKey);
for (Object kidKey : kids) {
ReaderField kid = new ReaderField(kidKey, rf.fieldName);
@@ -252,7 +252,7 @@ public final class FieldCacheSanityCheck
badness = badEntries.toArray(badness);
insanity.add(new Insanity(InsanityType.SUBREADER,
- "Found caches for decendents of " +
+ "Found caches for descendants of " +
parent.toString(),
badness));
}
@@ -266,7 +266,7 @@ public final class FieldCacheSanityCheck
* the hierarchy of subReaders building up a list of the objects
* returned by obj.getFieldCacheKey()
*/
- private List<Object> getAllDecendentReaderKeys(Object seed) {
+ private List<Object> getAllDescendentReaderKeys(Object seed) {
List<Object> all = new ArrayList<Object>(17); // will grow as we iter
all.add(seed);
for (int i = 0; i < all.size(); i++) {
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/OpenBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/OpenBitSet.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/OpenBitSet.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/OpenBitSet.java Fri Jun 24 23:07:10 2011
@@ -659,7 +659,38 @@ public class OpenBitSet extends DocIdSet
}
+ /** Returns the index of the first set bit starting downwards at
+ * the index specified.
+ * -1 is returned if there are no more set bits.
+ */
+ public int prevSetBit(int index) {
+ int i = index >> 6;
+ final int subIndex;
+ long word;
+ if (i >= wlen) {
+ i = wlen - 1;
+ if (i < 0) return -1;
+ subIndex = 63; // last possible bit
+ word = bits[i];
+ } else {
+ if (i < 0) return -1;
+ subIndex = index & 0x3f; // index within the word
+ word = (bits[i] << (63-subIndex)); // skip all the bits to the left of index
+ }
+
+ if (word != 0) {
+ return (i << 6) + subIndex - Long.numberOfLeadingZeros(word); // See LUCENE-3197
+ }
+
+ while (--i >= 0) {
+ word = bits[i];
+ if (word !=0 ) {
+ return (i << 6) + 63 - Long.numberOfLeadingZeros(word);
+ }
+ }
+ return -1;
+ }
@Override
public Object clone() {