You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/06/25 01:07:38 UTC

svn commit: r1139458 [6/24] - in /lucene/dev/branches/solr2193: ./ dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/ dev-tools/idea/solr/ dev-tools/idea/solr/contrib/clustering/ dev-tools/maven/ dev-tools/maven/modules/analysis/icu/ dev-t...

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/SortField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/SortField.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/SortField.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/SortField.java Fri Jun 24 23:07:10 2011
@@ -18,10 +18,16 @@ package org.apache.lucene.search;
  */
 
 import java.io.IOException;
+import java.util.Comparator;
 
 import org.apache.lucene.search.cache.*;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.StringHelper;
 
+// TODO(simonw) -- for cleaner transition, maybe we should make
+// a new SortField that subclasses this one and always uses
+// index values?
+
 /**
  * Stores information about how to sort documents by terms in an individual
  * field.  Fields must be indexed in order to sort by them.
@@ -33,62 +39,70 @@ import org.apache.lucene.util.StringHelp
  */
 public class SortField {
 
-  /** Sort by document score (relevance).  Sort values are Float and higher
-   * values are at the front. */
-  public static final int SCORE = 0;
-
-  /** Sort by document number (index order).  Sort values are Integer and lower
-   * values are at the front. */
-  public static final int DOC = 1;
-
-  // reserved, in Lucene 2.9, there was a constant: AUTO = 2;
-
-  /** Sort using term values as Strings.  Sort values are String and lower
-   * values are at the front. */
-  public static final int STRING = 3;
-
-  /** Sort using term values as encoded Integers.  Sort values are Integer and
-   * lower values are at the front. */
-  public static final int INT = 4;
-
-  /** Sort using term values as encoded Floats.  Sort values are Float and
-   * lower values are at the front. */
-  public static final int FLOAT = 5;
-
-  /** Sort using term values as encoded Longs.  Sort values are Long and
-   * lower values are at the front. */
-  public static final int LONG = 6;
-
-  /** Sort using term values as encoded Doubles.  Sort values are Double and
-   * lower values are at the front. */
-  public static final int DOUBLE = 7;
-
-  /** Sort using term values as encoded Shorts.  Sort values are Short and
-   * lower values are at the front. */
-  public static final int SHORT = 8;
-
-  /** Sort using a custom Comparator.  Sort values are any Comparable and
-   * sorting is done according to natural order. */
-  public static final int CUSTOM = 9;
-
-  /** Sort using term values as encoded Bytes.  Sort values are Byte and
-   * lower values are at the front. */
-  public static final int BYTE = 10;
-  
-  /** Sort using term values as Strings, but comparing by
-   * value (using String.compareTo) for all comparisons.
-   * This is typically slower than {@link #STRING}, which
-   * uses ordinals to do the sorting. */
-  public static final int STRING_VAL = 11;
-  
+  public static enum Type {
+
+    /** Sort by document score (relevance).  Sort values are Float and higher
+     * values are at the front. */
+    SCORE,
+
+    /** Sort by document number (index order).  Sort values are Integer and lower
+     * values are at the front. */
+    DOC,
+
+    /** Sort using term values as Strings.  Sort values are String and lower
+     * values are at the front. */
+    STRING,
+
+    /** Sort using term values as encoded Integers.  Sort values are Integer and
+     * lower values are at the front. */
+    INT,
+
+    /** Sort using term values as encoded Floats.  Sort values are Float and
+     * lower values are at the front. */
+    FLOAT,
+
+    /** Sort using term values as encoded Longs.  Sort values are Long and
+     * lower values are at the front. */
+    LONG,
+
+    /** Sort using term values as encoded Doubles.  Sort values are Double and
+     * lower values are at the front. */
+    DOUBLE,
+
+    /** Sort using term values as encoded Shorts.  Sort values are Short and
+     * lower values are at the front. */
+    SHORT,
+
+    /** Sort using a custom Comparator.  Sort values are any Comparable and
+     * sorting is done according to natural order. */
+    CUSTOM,
+
+    /** Sort using term values as encoded Bytes.  Sort values are Byte and
+     * lower values are at the front. */
+    BYTE,
+
+    /** Sort using term values as Strings, but comparing by
+     * value (using String.compareTo) for all comparisons.
+     * This is typically slower than {@link #STRING}, which
+     * uses ordinals to do the sorting. */
+    STRING_VAL,
+
+    /** Sort use byte[] index values. */
+    BYTES,
+
+    /** Force rewriting of SortField using {@link SortField#rewrite(IndexSearcher)}
+     * before it can be used for sorting */
+    REWRITEABLE
+  }
+
   /** Represents sorting by document score (relevance). */
-  public static final SortField FIELD_SCORE = new SortField (null, SCORE);
+  public static final SortField FIELD_SCORE = new SortField(null, Type.SCORE);
 
   /** Represents sorting by document number (index order). */
-  public static final SortField FIELD_DOC = new SortField (null, DOC);
+  public static final SortField FIELD_DOC = new SortField(null, Type.DOC);
 
   private String field;
-  private int type;  // defaults to determining type dynamically
+  private Type type;  // defaults to determining type dynamically
   boolean reverse = false;  // defaults to natural order
   private CachedArrayCreator<?> creator;
   public Object missingValue = null; // used for 'sortMissingFirst/Last'
@@ -102,7 +116,7 @@ public class SortField {
    *               <code>type</code> is SCORE or DOC.
    * @param type   Type of values in the terms.
    */
-  public SortField (String field, int type) {
+  public SortField(String field, Type type) {
     initFieldType(field, type);
   }
 
@@ -113,7 +127,7 @@ public class SortField {
    * @param type   Type of values in the terms.
    * @param reverse True if natural order should be reversed.
    */
-  public SortField (String field, int type, boolean reverse) {
+  public SortField(String field, Type type, boolean reverse) {
     initFieldType(field, type);
     this.reverse = reverse;
   }
@@ -131,7 +145,7 @@ public class SortField {
    *  @deprecated (4.0) use EntryCreator version
    */
   @Deprecated
-  public SortField (String field, FieldCache.Parser parser) {
+  public SortField(String field, FieldCache.Parser parser) {
     this(field, parser, false);
   }
 
@@ -149,40 +163,35 @@ public class SortField {
    *  @deprecated (4.0) use EntryCreator version
    */
   @Deprecated
-  public SortField (String field, FieldCache.Parser parser, boolean reverse) {
+  public SortField(String field, FieldCache.Parser parser, boolean reverse) {
     if (field == null) {
       throw new IllegalArgumentException("field can only be null when type is SCORE or DOC");
     } 
-    this.field = StringHelper.intern(field);
+    this.field = field;
     this.reverse = reverse;
     
     if (parser instanceof FieldCache.IntParser) {
-      this.type = INT;
       this.creator = new IntValuesCreator( field, (FieldCache.IntParser)parser );
     }
     else if (parser instanceof FieldCache.FloatParser) {
-      this.type = FLOAT;
       this.creator = new FloatValuesCreator( field, (FieldCache.FloatParser)parser );
     }
     else if (parser instanceof FieldCache.ShortParser) {
-      this.type = SHORT;
       this.creator = new ShortValuesCreator( field, (FieldCache.ShortParser)parser );
     }
     else if (parser instanceof FieldCache.ByteParser) {
-      this.type = BYTE;
       this.creator = new ByteValuesCreator( field, (FieldCache.ByteParser)parser );
     }
     else if (parser instanceof FieldCache.LongParser) {
-      this.type = LONG;
       this.creator = new LongValuesCreator( field, (FieldCache.LongParser)parser );
     }
     else if (parser instanceof FieldCache.DoubleParser) {
-      this.type = DOUBLE;
       this.creator = new DoubleValuesCreator( field, (FieldCache.DoubleParser)parser );
     }
     else
       throw new IllegalArgumentException("Parser instance does not subclass existing numeric parser from FieldCache (got " + parser + ")");
 
+    this.type = this.creator.getSortType();
   }
   
   /**
@@ -192,10 +201,10 @@ public class SortField {
    */
   public SortField( CachedArrayCreator<?> creator, boolean reverse ) 
   {
-    this.field = StringHelper.intern(creator.field);
+    this.field = creator.field;
     this.reverse = reverse;
     this.creator = creator;
-    this.type = creator.getSortTypeID();
+    this.type = creator.getSortType();
   }
   
   public SortField setMissingValue( Object v )
@@ -216,8 +225,8 @@ public class SortField {
    * @param field Name of field to sort by; cannot be <code>null</code>.
    * @param comparator Returns a comparator for sorting hits.
    */
-  public SortField (String field, FieldComparatorSource comparator) {
-    initFieldType(field, CUSTOM);
+  public SortField(String field, FieldComparatorSource comparator) {
+    initFieldType(field, Type.CUSTOM);
     this.comparatorSource = comparator;
   }
 
@@ -226,21 +235,21 @@ public class SortField {
    * @param comparator Returns a comparator for sorting hits.
    * @param reverse True if natural order should be reversed.
    */
-  public SortField (String field, FieldComparatorSource comparator, boolean reverse) {
-    initFieldType(field, CUSTOM);
+  public SortField(String field, FieldComparatorSource comparator, boolean reverse) {
+    initFieldType(field, Type.CUSTOM);
     this.reverse = reverse;
     this.comparatorSource = comparator;
   }
 
   // Sets field & type, and ensures field is not NULL unless
   // type is SCORE or DOC
-  private void initFieldType(String field, int type) {
+  private void initFieldType(String field, Type type) {
     this.type = type;
     if (field == null) {
-      if (type != SCORE && type != DOC)
+      if (type != Type.SCORE && type != Type.DOC)
         throw new IllegalArgumentException("field can only be null when type is SCORE or DOC");
     } else {
-      this.field = StringHelper.intern(field);
+      this.field = field;
     }
     
     if( creator != null ) {
@@ -267,7 +276,7 @@ public class SortField {
   /** Returns the type of contents in the field.
    * @return One of the constants SCORE, DOC, STRING, INT or FLOAT.
    */
-  public int getType() {
+  public Type getType() {
     return type;
   }
 
@@ -346,6 +355,10 @@ public class SortField {
       case CUSTOM:
         buffer.append("<custom:\"").append(field).append("\": ").append(comparatorSource).append('>');
         break;
+      
+      case REWRITEABLE:
+        buffer.append("<rewriteable: \"").append(field).append("\">");
+        break;
 
       default:
         buffer.append("<???: \"").append(field).append("\">");
@@ -368,7 +381,7 @@ public class SortField {
     if (!(o instanceof SortField)) return false;
     final SortField other = (SortField)o;
     return (
-      other.field == this.field // field is always interned
+      StringHelper.equals(other.field, this.field)
       && other.type == this.type
       && other.reverse == this.reverse
       && (other.comparatorSource == null ? this.comparatorSource == null : other.comparatorSource.equals(this.comparatorSource))
@@ -383,13 +396,33 @@ public class SortField {
    *  used). */
   @Override
   public int hashCode() {
-    int hash=type^0x346565dd + Boolean.valueOf(reverse).hashCode()^0xaf5998bb;
+    int hash = type.hashCode() ^ 0x346565dd + Boolean.valueOf(reverse).hashCode() ^ 0xaf5998bb;
     if (field != null) hash += field.hashCode()^0xff5685dd;
     if (comparatorSource != null) hash += comparatorSource.hashCode();
     if (creator != null) hash += creator.hashCode()^0x3aaf56ff;
     return hash;
   }
 
+  private boolean useIndexValues;
+
+  public void setUseIndexValues(boolean b) {
+    useIndexValues = b;
+  }
+
+  public boolean getUseIndexValues() {
+    return useIndexValues;
+  }
+
+  private Comparator<BytesRef> bytesComparator = BytesRef.getUTF8SortedAsUnicodeComparator();
+
+  public void setBytesComparator(Comparator<BytesRef> b) {
+    bytesComparator = b;
+  }
+
+  public Comparator<BytesRef> getBytesComparator() {
+    return bytesComparator;
+  }
+
   /** Returns the {@link FieldComparator} to use for
    * sorting.
    *
@@ -405,42 +438,67 @@ public class SortField {
   public FieldComparator getComparator(final int numHits, final int sortPos) throws IOException {
 
     switch (type) {
-    case SortField.SCORE:
+    case SCORE:
       return new FieldComparator.RelevanceComparator(numHits);
 
-    case SortField.DOC:
+    case DOC:
       return new FieldComparator.DocComparator(numHits);
 
-    case SortField.INT:
-      return new FieldComparator.IntComparator(numHits, (IntValuesCreator)creator, (Integer)missingValue );
+    case INT:
+      if (useIndexValues) {
+        return new FieldComparator.IntDocValuesComparator(numHits, field);
+      } else {
+        return new FieldComparator.IntComparator(numHits, (IntValuesCreator)creator, (Integer) missingValue);
+      }
 
-    case SortField.FLOAT:
-      return new FieldComparator.FloatComparator(numHits, (FloatValuesCreator)creator, (Float)missingValue );
+    case FLOAT:
+      if (useIndexValues) {
+        return new FieldComparator.FloatDocValuesComparator(numHits, field);
+      } else {
+        return new FieldComparator.FloatComparator(numHits, (FloatValuesCreator) creator, (Float) missingValue);
+      }
 
-    case SortField.LONG:
+    case LONG:
       return new FieldComparator.LongComparator(numHits, (LongValuesCreator)creator, (Long)missingValue );
 
-    case SortField.DOUBLE:
+    case DOUBLE:
       return new FieldComparator.DoubleComparator(numHits, (DoubleValuesCreator)creator, (Double)missingValue );
 
-    case SortField.BYTE:
+    case BYTE:
       return new FieldComparator.ByteComparator(numHits, (ByteValuesCreator)creator, (Byte)missingValue );
 
-    case SortField.SHORT:
+    case SHORT:
       return new FieldComparator.ShortComparator(numHits, (ShortValuesCreator)creator, (Short)missingValue );
 
-    case SortField.CUSTOM:
+    case CUSTOM:
       assert comparatorSource != null;
       return comparatorSource.newComparator(field, numHits, sortPos, reverse);
 
-    case SortField.STRING:
+    case STRING:
       return new FieldComparator.TermOrdValComparator(numHits, field, sortPos, reverse);
 
-    case SortField.STRING_VAL:
+    case STRING_VAL:
       return new FieldComparator.TermValComparator(numHits, field);
+
+    case REWRITEABLE:
+      throw new IllegalStateException("SortField needs to be rewritten through Sort.rewrite(..) and SortField.rewrite(..)");
         
     default:
       throw new IllegalStateException("Illegal sort type: " + type);
     }
   }
+
+  /**
+   * Rewrites this SortField, returning a new SortField if a change is made.
+   * Subclasses should override this define their rewriting behavior when this
+   * SortField is of type {@link SortField.Type#REWRITEABLE}
+   *
+   * @param searcher IndexSearcher to use during rewriting
+   * @return New rewritten SortField, or {@code this} if nothing has changed.
+   * @throws IOException Can be thrown by the rewriting
+   * @lucene.experimental
+   */
+  public SortField rewrite(IndexSearcher searcher) throws IOException {
+    return this;
+  }
 }

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TermQuery.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TermQuery.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TermQuery.java Fri Jun 24 23:07:10 2011
@@ -89,7 +89,7 @@ public class TermQuery extends Query {
     public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
       final String field = term.field();
       final IndexReader reader = context.reader;
-      assert termStates.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight is not the same as the current reader's top-reader";
+      assert termStates.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight (" + termStates.topReaderContext + ") is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
       final TermState state = termStates
           .get(context.ord);
       if (state == null) { // term is not present in that reader

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TopDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TopDocs.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TopDocs.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TopDocs.java Fri Jun 24 23:07:10 2011
@@ -17,15 +17,21 @@ package org.apache.lucene.search;
  * limitations under the License.
  */
 
+import java.io.IOException;
+
+import org.apache.lucene.util.PriorityQueue;
+
 /** Represents hits returned by {@link
  * IndexSearcher#search(Query,Filter,int)} and {@link
  * IndexSearcher#search(Query,int)}. */
 public class TopDocs {
-  /** The total number of hits for the query.
-  */
+
+  /** The total number of hits for the query. */
   public int totalHits;
+
   /** The top hits for the query. */
   public ScoreDoc[] scoreDocs;
+
   /** Stores the maximum score value encountered, needed for normalizing. */
   private float maxScore;
   
@@ -34,12 +40,12 @@ public class TopDocs {
    * scores are not tracked, this returns {@link Float#NaN}.
    */
   public float getMaxScore() {
-      return maxScore;
+    return maxScore;
   }
   
   /** Sets the maximum score value encountered. */
   public void setMaxScore(float maxScore) {
-      this.maxScore=maxScore;
+    this.maxScore=maxScore;
   }
 
   /** Constructs a TopDocs with a default maxScore=Float.NaN. */
@@ -52,4 +58,199 @@ public class TopDocs {
     this.scoreDocs = scoreDocs;
     this.maxScore = maxScore;
   }
+
+  // Refers to one hit:
+  private static class ShardRef {
+    // Which shard (index into shardHits[]):
+    final int shardIndex;
+
+    // Which hit within the shard:
+    int hitIndex;
+
+    public ShardRef(int shardIndex) {
+      this.shardIndex = shardIndex;
+    }
+
+    @Override
+    public String toString() {
+      return "ShardRef(shardIndex=" + shardIndex + " hitIndex=" + hitIndex + ")";
+    }
+  };
+
+  // Specialized MergeSortQueue that just merges by
+  // relevance score, descending:
+  private static class ScoreMergeSortQueue extends PriorityQueue<ShardRef> {
+    final ScoreDoc[][] shardHits;
+
+    public ScoreMergeSortQueue(TopDocs[] shardHits) {
+      super(shardHits.length);
+      this.shardHits = new ScoreDoc[shardHits.length][];
+      for(int shardIDX=0;shardIDX<shardHits.length;shardIDX++) {
+        this.shardHits[shardIDX] = shardHits[shardIDX].scoreDocs;
+      }
+    }
+
+    // Returns true if first is < second
+    public boolean lessThan(ShardRef first, ShardRef second) {
+      assert first != second;
+      final float firstScore = shardHits[first.shardIndex][first.hitIndex].score;
+      final float secondScore = shardHits[second.shardIndex][second.hitIndex].score;
+
+      if (firstScore < secondScore) {
+        return false;
+      } else if (firstScore > secondScore) {
+        return true;
+      } else {
+        // Tie break: earlier shard wins
+        if (first.shardIndex < second.shardIndex) {
+          return true;
+        } else if (first.shardIndex > second.shardIndex) {
+          return false;
+        } else {
+          // Tie break in same shard: resolve however the
+          // shard had resolved it:
+          assert first.hitIndex != second.hitIndex;
+          return first.hitIndex < second.hitIndex;
+        }
+      }
+    }
+  }
+
+  private static class MergeSortQueue extends PriorityQueue<ShardRef> {
+    // These are really FieldDoc instances:
+    final ScoreDoc[][] shardHits;
+    final FieldComparator[] comparators;
+    final int[] reverseMul;
+
+    public MergeSortQueue(Sort sort, TopDocs[] shardHits) throws IOException {
+      super(shardHits.length);
+      this.shardHits = new ScoreDoc[shardHits.length][];
+      for(int shardIDX=0;shardIDX<shardHits.length;shardIDX++) {
+        final ScoreDoc[] shard = shardHits[shardIDX].scoreDocs;
+        //System.out.println("  init shardIdx=" + shardIDX + " hits=" + shard);
+        if (shard != null) {
+          this.shardHits[shardIDX] = shard;
+          // Fail gracefully if API is misused:
+          for(int hitIDX=0;hitIDX<shard.length;hitIDX++) {
+            final ScoreDoc sd = shard[hitIDX];
+            if (!(sd instanceof FieldDoc)) {
+              throw new IllegalArgumentException("shard " + shardIDX + " was not sorted by the provided Sort (expected FieldDoc but got ScoreDoc)");
+            }
+            final FieldDoc fd = (FieldDoc) sd;
+            if (fd.fields == null) {
+              throw new IllegalArgumentException("shard " + shardIDX + " did not set sort field values (FieldDoc.fields is null); you must pass fillFields=true to IndexSearcher.search on each shard");
+            }
+          }
+        }
+      }
+
+      final SortField[] sortFields = sort.getSort();
+      comparators = new FieldComparator[sortFields.length];
+      reverseMul = new int[sortFields.length];
+      for(int compIDX=0;compIDX<sortFields.length;compIDX++) {
+        final SortField sortField = sortFields[compIDX];
+        comparators[compIDX] = sortField.getComparator(1, compIDX);
+        reverseMul[compIDX] = sortField.getReverse() ? -1 : 1;
+      }
+    }
+
+    // Returns true if first is < second
+    @SuppressWarnings("unchecked")
+    public boolean lessThan(ShardRef first, ShardRef second) {
+      assert first != second;
+      final FieldDoc firstFD = (FieldDoc) shardHits[first.shardIndex][first.hitIndex];
+      final FieldDoc secondFD = (FieldDoc) shardHits[second.shardIndex][second.hitIndex];
+      //System.out.println("  lessThan:\n     first=" + first + " doc=" + firstFD.doc + " score=" + firstFD.score + "\n    second=" + second + " doc=" + secondFD.doc + " score=" + secondFD.score);
+
+      for(int compIDX=0;compIDX<comparators.length;compIDX++) {
+        final FieldComparator comp = comparators[compIDX];
+        //System.out.println("    cmp idx=" + compIDX + " cmp1=" + firstFD.fields[compIDX] + " cmp2=" + secondFD.fields[compIDX] + " reverse=" + reverseMul[compIDX]);
+
+        final int cmp = reverseMul[compIDX] * comp.compareValues(firstFD.fields[compIDX], secondFD.fields[compIDX]);
+        
+        if (cmp != 0) {
+          //System.out.println("    return " + (cmp < 0));
+          return cmp < 0;
+        }
+      }
+
+      // Tie break: earlier shard wins
+      if (first.shardIndex < second.shardIndex) {
+        //System.out.println("    return tb true");
+        return true;
+      } else if (first.shardIndex > second.shardIndex) {
+        //System.out.println("    return tb false");
+        return false;
+      } else {
+        // Tie break in same shard: resolve however the
+        // shard had resolved it:
+        //System.out.println("    return tb " + (first.hitIndex < second.hitIndex));
+        assert first.hitIndex != second.hitIndex;
+        return first.hitIndex < second.hitIndex;
+      }
+    }
+  }
+
+  /** Returns a new TopDocs, containing topN results across
+   *  the provided TopDocs, sorting by the specified {@link
+   *  Sort}.  Each of the TopDocs must have been sorted by
+   *  the same Sort, and sort field values must have been
+   *  filled (ie, <code>fillFields=true</code> must be
+   *  passed to {@link
+   *  TopFieldCollector#create}.
+   *
+   * <p>Pass sort=null to merge sort by score descending.
+   *
+   * @lucene.experimental */
+  public static TopDocs merge(Sort sort, int topN, TopDocs[] shardHits) throws IOException {
+
+    final PriorityQueue<ShardRef> queue;
+    if (sort == null) {
+      queue = new ScoreMergeSortQueue(shardHits);
+    } else {
+      queue = new MergeSortQueue(sort, shardHits);
+    }
+
+    int totalHitCount = 0;
+    float maxScore = Float.MIN_VALUE;
+    for(int shardIDX=0;shardIDX<shardHits.length;shardIDX++) {
+      final TopDocs shard = shardHits[shardIDX];
+      if (shard.scoreDocs != null && shard.scoreDocs.length > 0) {
+        totalHitCount += shard.totalHits;
+        queue.add(new ShardRef(shardIDX));
+        maxScore = Math.max(maxScore, shard.getMaxScore());
+        //System.out.println("  maxScore now " + maxScore + " vs " + shard.getMaxScore());
+      }
+    }
+
+    final ScoreDoc[] hits = new ScoreDoc[Math.min(topN, totalHitCount)];
+
+    int hitUpto = 0;
+    while(hitUpto < hits.length) {
+      assert queue.size() > 0;
+      ShardRef ref = queue.pop();
+      final ScoreDoc hit = shardHits[ref.shardIndex].scoreDocs[ref.hitIndex++];
+      if (sort == null) {
+        hits[hitUpto] = new ScoreDoc(hit.doc, hit.score, ref.shardIndex);
+      } else {
+        hits[hitUpto] = new FieldDoc(hit.doc, hit.score, ((FieldDoc) hit).fields, ref.shardIndex);
+      }
+
+      //System.out.println("  hitUpto=" + hitUpto);
+      //System.out.println("    doc=" + hits[hitUpto].doc + " score=" + hits[hitUpto].score);
+
+      hitUpto++;
+
+      if (ref.hitIndex < shardHits[ref.shardIndex].scoreDocs.length) {
+        // Not done with this these TopDocs yet:
+        queue.add(ref);
+      }
+    }
+
+    if (sort == null) {
+      return new TopDocs(totalHitCount, hits, maxScore);
+    } else {
+      return new TopFieldDocs(totalHitCount, hits, sort.getSort(), maxScore);
+    }
+  }
 }

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java Fri Jun 24 23:07:10 2011
@@ -131,12 +131,11 @@ public abstract class TopTermsRewrite<Q 
       }
     });
     
-    final Term placeholderTerm = new Term(query.field);
     final Q q = getTopLevelQuery();
     final ScoreTerm[] scoreTerms = stQueue.toArray(new ScoreTerm[stQueue.size()]);
     ArrayUtil.mergeSort(scoreTerms, scoreTermSortByTermComp);
     for (final ScoreTerm st : scoreTerms) {
-      final Term term = placeholderTerm.createTerm(st.bytes);
+      final Term term = new Term(query.field, st.bytes);
       assert reader.docFreq(term) == st.termState.docFreq() : "reader DF is " + reader.docFreq(term) + " vs " + st.termState.docFreq();
       addClause(q, term, st.termState.docFreq(), query.getBoost() * st.boost, st.termState); // add to query
     }

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java Fri Jun 24 23:07:10 2011
@@ -61,8 +61,8 @@ public class ByteValuesCreator extends C
   }
   
   @Override
-  public int getSortTypeID() {
-    return SortField.BYTE;
+  public SortField.Type getSortType() {
+    return SortField.Type.BYTE;
   }
 
   //--------------------------------------------------------------------------------
@@ -143,4 +143,4 @@ public class ByteValuesCreator extends C
       vals.valid = new Bits.MatchNoBits( maxDoc );
     }
   }
-}
\ No newline at end of file
+}

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java Fri Jun 24 23:07:10 2011
@@ -26,6 +26,7 @@ import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.FieldCache.Parser;
+import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.OpenBitSet;
@@ -73,7 +74,7 @@ public abstract class CachedArrayCreator
   /** Return the type that the array will hold */
   public abstract Class getArrayType();
   public abstract Parser getParser();
-  public abstract int getSortTypeID();
+  public abstract SortField.Type getSortType();
 
   protected void setParserAndResetCounts(T value, Parser parser)
   {

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java Fri Jun 24 23:07:10 2011
@@ -29,7 +29,6 @@ import org.apache.lucene.search.FieldCac
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.PagedBytes;
-import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.packed.GrowableWriter;
 import org.apache.lucene.util.packed.PackedInts;
 
@@ -66,7 +65,6 @@ public class DocTermsCreator extends Ent
   @Override
   public DocTerms create(IndexReader reader) throws IOException {
 
-    String field = StringHelper.intern(this.field); // TODO?? necessary?
     Terms terms = MultiFields.getTerms(reader, field);
 
     final boolean fasterButMoreRAM = hasOption( FASTER_BUT_MORE_RAM );

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java Fri Jun 24 23:07:10 2011
@@ -34,7 +34,6 @@ import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.PagedBytes;
-import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.packed.GrowableWriter;
 import org.apache.lucene.util.packed.PackedInts;
 
@@ -70,7 +69,6 @@ public class DocTermsIndexCreator extend
   @Override
   public DocTermsIndex create(IndexReader reader) throws IOException
   {
-    String field = StringHelper.intern(this.field); // TODO?? necessary?
     Terms terms = MultiFields.getTerms(reader, field);
 
     final boolean fasterButMoreRAM = hasOption(FASTER_BUT_MORE_RAM);

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java Fri Jun 24 23:07:10 2011
@@ -61,8 +61,8 @@ public class DoubleValuesCreator extends
   }
   
   @Override
-  public int getSortTypeID() {
-    return SortField.DOUBLE;
+  public SortField.Type getSortType() {
+    return SortField.Type.DOUBLE;
   }
 
   //--------------------------------------------------------------------------------
@@ -161,4 +161,4 @@ public class DoubleValuesCreator extends
       vals.valid = new Bits.MatchNoBits( maxDoc );
     }
   }
-}
\ No newline at end of file
+}

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java Fri Jun 24 23:07:10 2011
@@ -61,8 +61,8 @@ public class FloatValuesCreator extends 
   }
   
   @Override
-  public int getSortTypeID() {
-    return SortField.FLOAT;
+  public SortField.Type getSortType() {
+    return SortField.Type.FLOAT;
   }
 
 
@@ -162,4 +162,4 @@ public class FloatValuesCreator extends 
       vals.valid = new Bits.MatchNoBits( maxDoc );
     }
   }
-}
\ No newline at end of file
+}

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java Fri Jun 24 23:07:10 2011
@@ -61,8 +61,8 @@ public class IntValuesCreator extends Ca
   }
   
   @Override
-  public int getSortTypeID() {
-    return SortField.INT;
+  public SortField.Type getSortType() {
+    return SortField.Type.INT;
   }
 
 
@@ -162,4 +162,4 @@ public class IntValuesCreator extends Ca
       vals.valid = new Bits.MatchNoBits( maxDoc );
     }
   }
-}
\ No newline at end of file
+}

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java Fri Jun 24 23:07:10 2011
@@ -61,8 +61,8 @@ public class LongValuesCreator extends C
   }
   
   @Override
-  public int getSortTypeID() {
-    return SortField.LONG;
+  public SortField.Type getSortType() {
+    return SortField.Type.LONG;
   }
 
 
@@ -162,4 +162,4 @@ public class LongValuesCreator extends C
       vals.valid = new Bits.MatchNoBits( maxDoc );
     }
   }
-}
\ No newline at end of file
+}

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java Fri Jun 24 23:07:10 2011
@@ -61,8 +61,8 @@ public class ShortValuesCreator extends 
   }
   
   @Override
-  public int getSortTypeID() {
-    return SortField.SHORT;
+  public SortField.Type getSortType() {
+    return SortField.Type.SHORT;
   }
 
 
@@ -144,4 +144,4 @@ public class ShortValuesCreator extends 
       vals.valid = new Bits.MatchNoBits( maxDoc );
     }
   }
-}
\ No newline at end of file
+}

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/function/CustomScoreProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/function/CustomScoreProvider.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/function/CustomScoreProvider.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/function/CustomScoreProvider.java Fri Jun 24 23:07:10 2011
@@ -19,7 +19,8 @@ package org.apache.lucene.search.functio
 
 import java.io.IOException;
 
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader; // for javadocs
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.FieldCache; // for javadocs
 
@@ -28,7 +29,7 @@ import org.apache.lucene.search.FieldCac
  * {@link CustomScoreQuery#getCustomScoreProvider}, if you want
  * to modify the custom score calculation of a {@link CustomScoreQuery}.
  * <p>Since Lucene 2.9, queries operate on each segment of an index separately,
- * so the protected {@link #reader} field can be used to resolve doc IDs,
+ * so the protected {@link #context} field can be used to resolve doc IDs,
  * as the supplied <code>doc</code> ID is per-segment and without knowledge
  * of the IndexReader you cannot access the document or {@link FieldCache}.
  * 
@@ -37,13 +38,13 @@ import org.apache.lucene.search.FieldCac
  */
 public class CustomScoreProvider {
 
-  protected final IndexReader reader;
+  protected final AtomicReaderContext context;
 
   /**
    * Creates a new instance of the provider class for the given {@link IndexReader}.
    */
-  public CustomScoreProvider(IndexReader reader) {
-    this.reader = reader;
+  public CustomScoreProvider(AtomicReaderContext context) {
+    this.context = context;
   }
 
   /**

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java Fri Jun 24 23:07:10 2011
@@ -175,8 +175,8 @@ public class CustomScoreQuery extends Qu
    * implementation as specified in the docs of {@link CustomScoreProvider}.
    * @since 2.9.2
    */
-  protected CustomScoreProvider getCustomScoreProvider(IndexReader reader) throws IOException {
-    return new CustomScoreProvider(reader);
+  protected CustomScoreProvider getCustomScoreProvider(AtomicReaderContext context) throws IOException {
+    return new CustomScoreProvider(context);
   }
 
   //=========================== W E I G H T ============================
@@ -187,7 +187,7 @@ public class CustomScoreQuery extends Qu
     boolean qStrict;
 
     public CustomWeight(IndexSearcher searcher) throws IOException {
-      this.subQueryWeight = subQuery.weight(searcher);
+      this.subQueryWeight = subQuery.createWeight(searcher);
       this.valSrcWeights = new Weight[valSrcQueries.length];
       for(int i = 0; i < valSrcQueries.length; i++) {
         this.valSrcWeights[i] = valSrcQueries[i].createWeight(searcher);
@@ -251,7 +251,7 @@ public class CustomScoreQuery extends Qu
       for(int i = 0; i < valSrcScorers.length; i++) {
          valSrcScorers[i] = valSrcWeights[i].scorer(context, scorerContext.scoreDocsInOrder(true));
       }
-      return new CustomScorer(context.reader, this, subQueryScorer, valSrcScorers);
+      return new CustomScorer(CustomScoreQuery.this.getCustomScoreProvider(context), this, subQueryScorer, valSrcScorers);
     }
 
     @Override
@@ -270,7 +270,7 @@ public class CustomScoreQuery extends Qu
       for(int i = 0; i < valSrcWeights.length; i++) {
         valSrcExpls[i] = valSrcWeights[i].explain(info, doc);
       }
-      Explanation customExp = CustomScoreQuery.this.getCustomScoreProvider(info.reader).customExplain(doc,subQueryExpl,valSrcExpls);
+      Explanation customExp = CustomScoreQuery.this.getCustomScoreProvider(info).customExplain(doc,subQueryExpl,valSrcExpls);
       float sc = getValue() * customExp.getValue();
       Explanation res = new ComplexExplanation(
         true, sc, CustomScoreQuery.this.toString() + ", product of:");
@@ -300,14 +300,14 @@ public class CustomScoreQuery extends Qu
     private float vScores[]; // reused in score() to avoid allocating this array for each doc 
 
     // constructor
-    private CustomScorer(IndexReader reader, CustomWeight w,
+    private CustomScorer(CustomScoreProvider provider, CustomWeight w,
         Scorer subQueryScorer, Scorer[] valSrcScorers) throws IOException {
       super(w);
       this.qWeight = w.getValue();
       this.subQueryScorer = subQueryScorer;
       this.valSrcScorers = valSrcScorers;
       this.vScores = new float[valSrcScorers.length];
-      this.provider = CustomScoreQuery.this.getCustomScoreProvider(reader);
+      this.provider = provider;
     }
 
     @Override

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java Fri Jun 24 23:07:10 2011
@@ -17,6 +17,8 @@ package org.apache.lucene.store;
  * limitations under the License.
  */
 
+import org.apache.lucene.util.BytesRef;
+
 /** @lucene.experimental */
 public final class ByteArrayDataInput extends DataInput {
 
@@ -25,9 +27,16 @@ public final class ByteArrayDataInput ex
   private int pos;
   private int limit;
 
-  // TODO: allow BytesRef (slice) too
   public ByteArrayDataInput(byte[] bytes) {
-    this.bytes = bytes;
+    reset(bytes);
+  }
+
+  public ByteArrayDataInput(byte[] bytes, int offset, int len) {
+    reset(bytes, offset, len);
+  }
+
+  public ByteArrayDataInput() {
+    reset(BytesRef.EMPTY_BYTES);
   }
 
   public void reset(byte[] bytes) {
@@ -41,7 +50,7 @@ public final class ByteArrayDataInput ex
   public void reset(byte[] bytes, int offset, int len) {
     this.bytes = bytes;
     pos = offset;
-    limit = len;
+    limit = offset + len;
   }
 
   public boolean eof() {
@@ -59,12 +68,14 @@ public final class ByteArrayDataInput ex
  
   @Override
   public int readInt() {
+    assert pos+4 <= limit;
     return ((bytes[pos++] & 0xFF) << 24) | ((bytes[pos++] & 0xFF) << 16)
       | ((bytes[pos++] & 0xFF) <<  8) |  (bytes[pos++] & 0xFF);
   }
  
   @Override
   public long readLong() {
+    assert pos+8 <= limit;
     final int i1 = ((bytes[pos++] & 0xff) << 24) | ((bytes[pos++] & 0xff) << 16) |
       ((bytes[pos++] & 0xff) << 8) | (bytes[pos++] & 0xff);
     final int i2 = ((bytes[pos++] & 0xff) << 24) | ((bytes[pos++] & 0xff) << 16) |
@@ -74,9 +85,11 @@ public final class ByteArrayDataInput ex
 
   @Override
   public int readVInt() {
+    checkBounds();
     byte b = bytes[pos++];
     int i = b & 0x7F;
     for (int shift = 7; (b & 0x80) != 0; shift += 7) {
+      checkBounds();
       b = bytes[pos++];
       i |= (b & 0x7F) << shift;
     }
@@ -85,9 +98,11 @@ public final class ByteArrayDataInput ex
  
   @Override
   public long readVLong() {
+    checkBounds();
     byte b = bytes[pos++];
     long i = b & 0x7F;
     for (int shift = 7; (b & 0x80) != 0; shift += 7) {
+      checkBounds();
       b = bytes[pos++];
       i |= (b & 0x7FL) << shift;
     }
@@ -97,7 +112,7 @@ public final class ByteArrayDataInput ex
   // NOTE: AIOOBE not EOF if you read too much
   @Override
   public byte readByte() {
-    assert pos < limit;
+    checkBounds();
     return bytes[pos++];
   }
 
@@ -108,4 +123,9 @@ public final class ByteArrayDataInput ex
     System.arraycopy(bytes, pos, b, offset, len);
     pos += len;
   }
+
+  private boolean checkBounds() {
+    assert pos < limit;
+    return true;
+  }
 }

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/DataOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/DataOutput.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/DataOutput.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/DataOutput.java Fri Jun 24 23:07:10 2011
@@ -60,6 +60,14 @@ public abstract class DataOutput {
     writeByte((byte)(i >>  8));
     writeByte((byte) i);
   }
+  
+  /** Writes a short as two bytes.
+   * @see DataInput#readShort()
+   */
+  public void writeShort(short i) throws IOException {
+    writeByte((byte)(i >>  8));
+    writeByte((byte) i);
+  }
 
   /** Writes an int in a variable-length format.  Writes between one and
    * five bytes.  Smaller values take fewer bytes.  Negative numbers are not

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/Directory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/Directory.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/Directory.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/Directory.java Fri Jun 24 23:07:10 2011
@@ -112,11 +112,35 @@ public abstract class Directory implemen
    * implementation may ignore the buffer size.  Currently
    * the only Directory implementations that respect this
    * parameter are {@link FSDirectory} and {@link
-   * org.apache.lucene.index.CompoundFileReader}.
+   * CompoundFileDirectory}.
   */
   public IndexInput openInput(String name, int bufferSize) throws IOException {
     return openInput(name);
   }
+  
+  /** 
+   * Returns a {@link CompoundFileDirectory} capable of
+   * reading the Lucene compound file format.  
+   * <p>
+   * The default implementation returns 
+   * {@link DefaultCompoundFileDirectory}.
+   * @lucene.experimental
+   */
+  public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+    return new DefaultCompoundFileDirectory(this, name, bufferSize, false);
+  }
+  
+  /** 
+   * Returns a {@link CompoundFileDirectory} capable of
+   * writing the Lucene compound file format.  
+   * <p>
+   * The default implementation returns 
+   * {@link DefaultCompoundFileDirectory}.
+   * @lucene.experimental
+   */
+  public CompoundFileDirectory createCompoundOutput(String name) throws IOException {
+    return new DefaultCompoundFileDirectory(this, name, 1024, true);
+  }
 
   /** Construct a {@link Lock}.
    * @param name the name of the lock file

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/FSDirectory.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/FSDirectory.java Fri Jun 24 23:07:10 2011
@@ -173,7 +173,7 @@ public abstract class FSDirectory extend
   /** Just like {@link #open(File)}, but allows you to
    *  also specify a custom {@link LockFactory}. */
   public static FSDirectory open(File path, LockFactory lockFactory) throws IOException {
-    if ((Constants.WINDOWS || Constants.SUN_OS)
+    if ((Constants.WINDOWS || Constants.SUN_OS || Constants.LINUX)
           && Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
       return new MMapDirectory(path, lockFactory);
     } else if (Constants.WINDOWS) {

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java Fri Jun 24 23:07:10 2011
@@ -148,4 +148,14 @@ public class FileSwitchDirectory extends
   public IndexInput openInput(String name) throws IOException {
     return getDirectory(name).openInput(name);
   }
+
+  @Override
+  public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+    return getDirectory(name).openCompoundInput(name, bufferSize);
+  }
+  
+  @Override
+  public CompoundFileDirectory createCompoundOutput(String name) throws IOException {
+    return getDirectory(name).createCompoundOutput(name);
+  }
 }

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/IndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/IndexOutput.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/IndexOutput.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/IndexOutput.java Fri Jun 24 23:07:10 2011
@@ -58,4 +58,5 @@ public abstract class IndexOutput extend
    * @param length file length
    */
   public void setLength(long length) throws IOException {}
+
 }

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/MMapDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/MMapDirectory.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/MMapDirectory.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/MMapDirectory.java Fri Jun 24 23:07:10 2011
@@ -32,6 +32,7 @@ import java.security.PrivilegedActionExc
 import java.lang.reflect.Method;
 
 import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.IOUtils;
 
 /** File-based {@link Directory} implementation that uses
  *  mmap for reading, and {@link
@@ -79,8 +80,8 @@ import org.apache.lucene.util.Constants;
  */
 public class MMapDirectory extends FSDirectory {
   private boolean useUnmapHack = UNMAP_SUPPORTED;
-  public static final int DEFAULT_MAX_BUFF = Constants.JRE_IS_64BIT ? Integer.MAX_VALUE : (256 * 1024 * 1024);
-  private int maxBBuf = DEFAULT_MAX_BUFF;
+  public static final int DEFAULT_MAX_BUFF = Constants.JRE_IS_64BIT ? (1 << 30) : (1 << 28);
+  private int chunkSizePower;
 
   /** Create a new MMapDirectory for the named location.
    *
@@ -91,6 +92,7 @@ public class MMapDirectory extends FSDir
    */
   public MMapDirectory(File path, LockFactory lockFactory) throws IOException {
     super(path, lockFactory);
+    setMaxChunkSize(DEFAULT_MAX_BUFF);
   }
 
   /** Create a new MMapDirectory for the named location and {@link NativeFSLockFactory}.
@@ -100,6 +102,7 @@ public class MMapDirectory extends FSDir
    */
   public MMapDirectory(File path) throws IOException {
     super(path, null);
+    setMaxChunkSize(DEFAULT_MAX_BUFF);
   }
 
   /**
@@ -180,23 +183,28 @@ public class MMapDirectory extends FSDir
    * Especially on 32 bit platform, the address space can be very fragmented,
    * so large index files cannot be mapped.
    * Using a lower chunk size makes the directory implementation a little
-   * bit slower (as the correct chunk must be resolved on each seek)
+   * bit slower (as the correct chunk may be resolved on lots of seeks)
    * but the chance is higher that mmap does not fail. On 64 bit
-   * Java platforms, this parameter should always be {@link Integer#MAX_VALUE},
+   * Java platforms, this parameter should always be {@code 1 << 30},
    * as the address space is big enough.
+   * <b>Please note:</b> This method always rounds down the chunk size
+   * to a power of 2.
    */
-  public void setMaxChunkSize(final int maxBBuf) {
-    if (maxBBuf<=0)
+  public final void setMaxChunkSize(final int maxChunkSize) {
+    if (maxChunkSize <= 0)
       throw new IllegalArgumentException("Maximum chunk size for mmap must be >0");
-    this.maxBBuf=maxBBuf;
+    //System.out.println("Requested chunk size: "+maxChunkSize);
+    this.chunkSizePower = 31 - Integer.numberOfLeadingZeros(maxChunkSize);
+    assert this.chunkSizePower >= 0 && this.chunkSizePower <= 30;
+    //System.out.println("Got chunk size: "+getMaxChunkSize());
   }
   
   /**
    * Returns the current mmap chunk size.
    * @see #setMaxChunkSize
    */
-  public int getMaxChunkSize() {
-    return maxBBuf;
+  public final int getMaxChunkSize() {
+    return 1 << chunkSizePower;
   }
 
   /** Creates an IndexInput for the file with the given name. */
@@ -206,103 +214,46 @@ public class MMapDirectory extends FSDir
     File f = new File(getDirectory(), name);
     RandomAccessFile raf = new RandomAccessFile(f, "r");
     try {
-      return (raf.length() <= maxBBuf)
-             ? (IndexInput) new MMapIndexInput(raf)
-             : (IndexInput) new MultiMMapIndexInput(raf, maxBBuf);
+      return new MMapIndexInput(raf, 0, raf.length(), chunkSizePower);
     } finally {
       raf.close();
     }
   }
+  
+  @Override
+  public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+    return new MMapCompoundFileDirectory(name, bufferSize);
+  }
+  
+  private final class MMapCompoundFileDirectory extends CompoundFileDirectory {
+    private RandomAccessFile raf = null;
 
-  private class MMapIndexInput extends IndexInput {
-
-    private ByteBuffer buffer;
-    private final long length;
-    private boolean isClone = false;
-
-    private MMapIndexInput(RandomAccessFile raf) throws IOException {
-        this.length = raf.length();
-        this.buffer = raf.getChannel().map(MapMode.READ_ONLY, 0, length);
-    }
-
-    @Override
-    public byte readByte() throws IOException {
-      try {
-        return buffer.get();
-      } catch (BufferUnderflowException e) {
-        throw new IOException("read past EOF");
-      }
-    }
-
-    @Override
-    public void readBytes(byte[] b, int offset, int len) throws IOException {
-      try {
-        buffer.get(b, offset, len);
-      } catch (BufferUnderflowException e) {
-        throw new IOException("read past EOF");
-      }
-    }
-    
-    @Override
-    public short readShort() throws IOException {
-      try {
-        return buffer.getShort();
-      } catch (BufferUnderflowException e) {
-        throw new IOException("read past EOF");
-      }
-    }
-
-    @Override
-    public int readInt() throws IOException {
-      try {
-        return buffer.getInt();
-      } catch (BufferUnderflowException e) {
-        throw new IOException("read past EOF");
-      }
-    }
-
-    @Override
-    public long readLong() throws IOException {
+    public MMapCompoundFileDirectory(String fileName, int readBufferSize) throws IOException {
+      super(MMapDirectory.this, fileName, readBufferSize);
+      IndexInput stream = null;
       try {
-        return buffer.getLong();
-      } catch (BufferUnderflowException e) {
-        throw new IOException("read past EOF");
+        File f = new File(MMapDirectory.this.getDirectory(), fileName);
+        raf = new RandomAccessFile(f, "r");
+        stream = new MMapIndexInput(raf, 0, raf.length(), chunkSizePower);
+        initForRead(CompoundFileDirectory.readEntries(stream, MMapDirectory.this, fileName));
+        stream.close();
+      } catch (IOException e) {
+        // throw our original exception
+        IOUtils.closeSafely(e, raf, stream);
       }
     }
-    
-    @Override
-    public long getFilePointer() {
-      return buffer.position();
-    }
-
-    @Override
-    public void seek(long pos) throws IOException {
-      buffer.position((int)pos);
-    }
 
     @Override
-    public long length() {
-      return length;
+    public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException {
+      return new MMapIndexInput(raf, offset, length, chunkSizePower);
     }
 
     @Override
-    public Object clone() {
-      if (buffer == null)
-        throw new AlreadyClosedException("MMapIndexInput already closed");
-      MMapIndexInput clone = (MMapIndexInput)super.clone();
-      clone.isClone = true;
-      clone.buffer = buffer.duplicate();
-      return clone;
-    }
-
-    @Override
-    public void close() throws IOException {
-      // unmap the buffer (if enabled) and at least unset it for GC
+    public synchronized void close() throws IOException {
       try {
-        if (isClone || buffer == null) return;
-        cleanMapping(buffer);
+        raf.close();
       } finally {
-        buffer = null;
+        super.close();
       }
     }
   }
@@ -310,48 +261,46 @@ public class MMapDirectory extends FSDir
   // Because Java's ByteBuffer uses an int to address the
   // values, it's necessary to access a file >
   // Integer.MAX_VALUE in size using multiple byte buffers.
-  private class MultiMMapIndexInput extends IndexInput {
+  private final class MMapIndexInput extends IndexInput {
   
     private ByteBuffer[] buffers;
-    private int[] bufSizes; // keep here, ByteBuffer.size() method is optional
   
-    private final long length;
+    private final long length, chunkSizeMask, chunkSize;
+    private final int chunkSizePower;
   
     private int curBufIndex;
-    private final int maxBufSize;
   
     private ByteBuffer curBuf; // redundant for speed: buffers[curBufIndex]
   
     private boolean isClone = false;
     
-    public MultiMMapIndexInput(RandomAccessFile raf, int maxBufSize)
-      throws IOException {
-      this.length = raf.length();
-      this.maxBufSize = maxBufSize;
+    MMapIndexInput(RandomAccessFile raf, long offset, long length, int chunkSizePower) throws IOException {
+      this.length = length;
+      this.chunkSizePower = chunkSizePower;
+      this.chunkSize = 1L << chunkSizePower;
+      this.chunkSizeMask = chunkSize - 1L;
+      
+      if (chunkSizePower < 0 || chunkSizePower > 30)
+        throw new IllegalArgumentException("Invalid chunkSizePower used for ByteBuffer size: " + chunkSizePower);
       
-      if (maxBufSize <= 0)
-        throw new IllegalArgumentException("Non positive maxBufSize: "
-                                           + maxBufSize);
+      if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)
+        throw new IllegalArgumentException("RandomAccessFile too big for chunk size: " + raf.toString());
       
-      if ((length / maxBufSize) > Integer.MAX_VALUE)
-        throw new IllegalArgumentException
-          ("RandomAccessFile too big for maximum buffer size: "
-           + raf.toString());
+      // we always allocate one more buffer, the last one may be a 0 byte one
+      final int nrBuffers = (int) (length >>> chunkSizePower) + 1;
       
-      int nrBuffers = (int) (length / maxBufSize);
-      if (((long) nrBuffers * maxBufSize) <= length) nrBuffers++;
+      //System.out.println("length="+length+", chunkSizePower=" + chunkSizePower + ", chunkSizeMask=" + chunkSizeMask + ", nrBuffers=" + nrBuffers);
       
       this.buffers = new ByteBuffer[nrBuffers];
-      this.bufSizes = new int[nrBuffers];
       
-      long bufferStart = 0;
+      long bufferStart = 0L;
       FileChannel rafc = raf.getChannel();
       for (int bufNr = 0; bufNr < nrBuffers; bufNr++) { 
-        int bufSize = (length > (bufferStart + maxBufSize))
-          ? maxBufSize
-          : (int) (length - bufferStart);
-        this.buffers[bufNr] = rafc.map(MapMode.READ_ONLY,bufferStart,bufSize);
-        this.bufSizes[bufNr] = bufSize;
+        int bufSize = (int) ( (length > (bufferStart + chunkSize))
+          ? chunkSize
+          : (length - bufferStart)
+        );
+        this.buffers[bufNr] = rafc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize);
         bufferStart += bufSize;
       }
       seek(0L);
@@ -362,11 +311,13 @@ public class MMapDirectory extends FSDir
       try {
         return curBuf.get();
       } catch (BufferUnderflowException e) {
-        curBufIndex++;
-        if (curBufIndex >= buffers.length)
-          throw new IOException("read past EOF");
-        curBuf = buffers[curBufIndex];
-        curBuf.position(0);
+        do {
+          curBufIndex++;
+          if (curBufIndex >= buffers.length)
+            throw new IOException("read past EOF");
+          curBuf = buffers[curBufIndex];
+          curBuf.position(0);
+        } while (!curBuf.hasRemaining());
         return curBuf.get();
       }
     }
@@ -421,15 +372,28 @@ public class MMapDirectory extends FSDir
     
     @Override
     public long getFilePointer() {
-      return ((long) curBufIndex * maxBufSize) + curBuf.position();
+      return (((long) curBufIndex) << chunkSizePower) + curBuf.position();
     }
   
     @Override
     public void seek(long pos) throws IOException {
-      curBufIndex = (int) (pos / maxBufSize);
-      curBuf = buffers[curBufIndex];
-      int bufOffset = (int) (pos - ((long) curBufIndex * maxBufSize));
-      curBuf.position(bufOffset);
+      // we use >> here to preserve negative, so we will catch AIOOBE:
+      final int bi = (int) (pos >> chunkSizePower);
+      try {
+        final ByteBuffer b = buffers[bi];
+        b.position((int) (pos & chunkSizeMask));
+        // write values, on exception all is unchanged
+        this.curBufIndex = bi;
+        this.curBuf = b;
+      } catch (ArrayIndexOutOfBoundsException aioobe) {
+        if (pos < 0L)
+          throw new IllegalArgumentException("Seeking to negative position");
+        throw new IOException("seek past EOF");
+      } catch (IllegalArgumentException iae) {
+        if (pos < 0L)
+          throw new IllegalArgumentException("Seeking to negative position");
+        throw new IOException("seek past EOF");
+      }
     }
   
     @Override
@@ -440,11 +404,10 @@ public class MMapDirectory extends FSDir
     @Override
     public Object clone() {
       if (buffers == null)
-        throw new AlreadyClosedException("MultiMMapIndexInput already closed");
-      MultiMMapIndexInput clone = (MultiMMapIndexInput)super.clone();
+        throw new AlreadyClosedException("MMapIndexInput already closed");
+      final MMapIndexInput clone = (MMapIndexInput)super.clone();
       clone.isClone = true;
       clone.buffers = new ByteBuffer[buffers.length];
-      // No need to clone bufSizes.
       // Since most clones will use only one buffer, duplicate() could also be
       // done lazy in clones, e.g. when adapting curBuf.
       for (int bufNr = 0; bufNr < buffers.length; bufNr++) {
@@ -453,9 +416,7 @@ public class MMapDirectory extends FSDir
       try {
         clone.seek(getFilePointer());
       } catch(IOException ioe) {
-        RuntimeException newException = new RuntimeException(ioe);
-        newException.initCause(ioe);
-        throw newException;
+        throw new RuntimeException("Should never happen", ioe);
       }
       return clone;
     }

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java Fri Jun 24 23:07:10 2011
@@ -24,6 +24,9 @@ import java.nio.channels.ClosedChannelEx
 import java.nio.channels.FileChannel;
 import java.util.concurrent.Future; // javadoc
 
+import org.apache.lucene.store.SimpleFSDirectory.SimpleFSIndexInput;
+import org.apache.lucene.util.IOUtils;
+
 /**
  * An {@link FSDirectory} implementation that uses java.nio's FileChannel's
  * positional read, which allows multiple threads to read from the same file
@@ -77,6 +80,47 @@ public class NIOFSDirectory extends FSDi
     ensureOpen();
     return new NIOFSIndexInput(new File(getDirectory(), name), bufferSize, getReadChunkSize());
   }
+  
+  @Override
+  public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+    return new NIOFSCompoundFileDirectory(name, bufferSize);
+  }
+
+  private final class NIOFSCompoundFileDirectory extends CompoundFileDirectory {
+    private SimpleFSIndexInput.Descriptor fd;
+    private FileChannel fc;
+
+    public NIOFSCompoundFileDirectory(String fileName, int readBufferSize) throws IOException {
+      super(NIOFSDirectory.this, fileName, readBufferSize);
+      IndexInput stream = null;
+      try {
+        File f = new File(NIOFSDirectory.this.getDirectory(), fileName);
+        fd = new SimpleFSIndexInput.Descriptor(f, "r");
+        fc = fd.getChannel();
+        stream = new NIOFSIndexInput(fd, fc, 0, fd.length, readBufferSize,
+            getReadChunkSize());
+        initForRead(CompoundFileDirectory.readEntries(stream, NIOFSDirectory.this, fileName));
+        stream.close();
+      } catch (IOException e) {
+        // throw our original exception
+        IOUtils.closeSafely(e, fc, fd, stream);
+      }
+    }
+    
+    @Override
+    public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException {
+      return new NIOFSIndexInput(fd, fc, offset, length, readBufferSize, getReadChunkSize());
+    }
+
+    @Override
+    public synchronized void close() throws IOException {
+      try {
+        IOUtils.closeSafely(false, fc, fd);
+      } finally {
+        super.close();
+      }
+    }
+  }
 
   protected static class NIOFSIndexInput extends SimpleFSDirectory.SimpleFSIndexInput {
 
@@ -91,6 +135,12 @@ public class NIOFSDirectory extends FSDi
       super(path, bufferSize, chunkSize);
       channel = file.getChannel();
     }
+    
+    public NIOFSIndexInput(Descriptor file, FileChannel fc, long off, long length, int bufferSize, int chunkSize) throws IOException {
+      super(file, off, length, bufferSize, chunkSize);
+      channel = fc;
+      isClone = true;
+    }
 
     @Override
     protected void newBuffer(byte[] newBuffer) {
@@ -145,7 +195,11 @@ public class NIOFSDirectory extends FSDi
       int readLength = bb.limit() - readOffset;
       assert readLength == len;
 
-      long pos = getFilePointer();
+      long pos = getFilePointer() + off;
+      
+      if (pos + len > end) {
+        throw new IOException("read past EOF");
+      }
 
       try {
         while (readLength > 0) {
@@ -159,9 +213,6 @@ public class NIOFSDirectory extends FSDi
           }
           bb.limit(limit);
           int i = channel.read(bb, pos);
-          if (i == -1) {
-            throw new IOException("read past EOF");
-          }
           pos += i;
           readOffset += i;
           readLength -= i;

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/RAMOutputStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/RAMOutputStream.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/RAMOutputStream.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/RAMOutputStream.java Fri Jun 24 23:07:10 2011
@@ -67,6 +67,26 @@ public class RAMOutputStream extends Ind
     }
   }
 
+  /** Copy the current contents of this buffer to output
+   *  byte array */
+  public void writeTo(byte[] bytes, int offset) throws IOException {
+    flush();
+    final long end = file.length;
+    long pos = 0;
+    int buffer = 0;
+    int bytesUpto = offset;
+    while (pos < end) {
+      int length = BUFFER_SIZE;
+      long nextPos = pos + length;
+      if (nextPos > end) {                        // at the last buffer
+        length = (int)(end - pos);
+      }
+      System.arraycopy(file.getBuffer(buffer++), 0, bytes, bytesUpto, length);
+      bytesUpto += length;
+      pos = nextPos;
+    }
+  }
+
   /** Resets this to an empty file. */
   public void reset() {
     currentBuffer = null;

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java Fri Jun 24 23:07:10 2011
@@ -21,6 +21,8 @@ import java.io.File;
 import java.io.IOException;
 import java.io.RandomAccessFile;
 
+import org.apache.lucene.util.IOUtils;
+
 /** A straightforward implementation of {@link FSDirectory}
  *  using java.io.RandomAccessFile.  However, this class has
  *  poor concurrent performance (multiple threads will
@@ -55,6 +57,45 @@ public class SimpleFSDirectory extends F
     ensureOpen();
     return new SimpleFSIndexInput(new File(directory, name), bufferSize, getReadChunkSize());
   }
+  
+  @Override
+  public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+    return new SimpleFSCompoundFileDirectory(name, bufferSize);
+  }
+
+  private final class SimpleFSCompoundFileDirectory extends CompoundFileDirectory {
+    private SimpleFSIndexInput.Descriptor fd;
+
+    public SimpleFSCompoundFileDirectory(String fileName, int readBufferSize) throws IOException {
+      super(SimpleFSDirectory.this, fileName, readBufferSize);
+      IndexInput stream = null;
+      try {
+        final File f = new File(SimpleFSDirectory.this.getDirectory(), fileName);
+        fd = new SimpleFSIndexInput.Descriptor(f, "r");
+        stream = new SimpleFSIndexInput(fd, 0, fd.length, readBufferSize,
+            getReadChunkSize());
+        initForRead(CompoundFileDirectory.readEntries(stream, SimpleFSDirectory.this, fileName));
+        stream.close();
+      } catch (IOException e) {
+        // throw our original exception
+        IOUtils.closeSafely(e, fd, stream);
+      }
+    }
+
+    @Override
+    public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException {
+      return new SimpleFSIndexInput(fd, offset, length, readBufferSize, getReadChunkSize());
+    }
+
+    @Override
+    public synchronized void close() throws IOException {
+      try {
+        fd.close();
+      } finally {
+        super.close();
+      }
+    }
+  }
 
   protected static class SimpleFSIndexInput extends BufferedIndexInput {
   
@@ -84,11 +125,24 @@ public class SimpleFSDirectory extends F
     boolean isClone;
     //  LUCENE-1566 - maximum read length on a 32bit JVM to prevent incorrect OOM 
     protected final int chunkSize;
+    protected final long off;
+    protected final long end;
     
     public SimpleFSIndexInput(File path, int bufferSize, int chunkSize) throws IOException {
       super(bufferSize);
-      file = new Descriptor(path, "r");
+      this.file = new Descriptor(path, "r"); 
       this.chunkSize = chunkSize;
+      this.off = 0L;
+      this.end = file.length;
+    }
+    
+    public SimpleFSIndexInput(Descriptor file, long off, long length, int bufferSize, int chunkSize) throws IOException {
+      super(bufferSize);
+      this.file = file;
+      this.chunkSize = chunkSize;
+      this.off = off;
+      this.end = off + length;
+      this.isClone = true; // well, we are sorta?
     }
   
     /** IndexInput methods */
@@ -96,13 +150,17 @@ public class SimpleFSDirectory extends F
     protected void readInternal(byte[] b, int offset, int len)
          throws IOException {
       synchronized (file) {
-        long position = getFilePointer();
+        long position = off + getFilePointer();
         if (position != file.position) {
           file.seek(position);
           file.position = position;
         }
         int total = 0;
 
+        if (position + len > end) {
+          throw new IOException("read past EOF");
+        }
+
         try {
           do {
             final int readLength;
@@ -113,9 +171,6 @@ public class SimpleFSDirectory extends F
               readLength = chunkSize;
             }
             final int i = file.read(b, offset + total, readLength);
-            if (i == -1) {
-              throw new IOException("read past EOF");
-            }
             file.position += i;
             total += i;
           } while (total < len);
@@ -144,7 +199,7 @@ public class SimpleFSDirectory extends F
   
     @Override
     public long length() {
-      return file.length;
+      return end - off;
     }
   
     @Override

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/ArrayUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/ArrayUtil.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/ArrayUtil.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/ArrayUtil.java Fri Jun 24 23:07:10 2011
@@ -255,6 +255,19 @@ public final class ArrayUtil {
     return grow(array, 1 + array.length);
   }
 
+  public static double[] grow(double[] array, int minSize) {
+    if (array.length < minSize) {
+      double[] newArray = new double[oversize(minSize, RamUsageEstimator.NUM_BYTES_DOUBLE)];
+      System.arraycopy(array, 0, newArray, 0, array.length);
+      return newArray;
+    } else
+      return array;
+  }
+
+  public static double[] grow(double[] array) {
+    return grow(array, 1 + array.length);
+  }
+
   public static short[] shrink(short[] array, int targetSize) {
     final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_SHORT);
     if (newSize != array.length) {

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/BitUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/BitUtil.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/BitUtil.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/BitUtil.java Fri Jun 24 23:07:10 2011
@@ -778,6 +778,28 @@ public final class BitUtil {
    return n - (y & 1);
   }
 
+  /** table of number of leading zeros in a byte */
+  public static final byte[] nlzTable = {8,7,6,6,5,5,5,5,4,4,4,4,4,4,4,4,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+
+  /** Returns the number of leading zero bits.
+   */
+  public static int nlz(long x) {
+   int n = 0;
+   // do the first step as a long
+   int y = (int)(x>>>32);
+   if (y==0) {n+=32; y = (int)(x); }
+   if ((y & 0xFFFF0000) == 0) { n+=16; y<<=16; }
+   if ((y & 0xFF000000) == 0) { n+=8; y<<=8; }
+   return n + nlzTable[y >>> 24];
+ /* implementation without table:
+   if ((y & 0xF0000000) == 0) { n+=4; y<<=4; }
+   if ((y & 0xC0000000) == 0) { n+=2; y<<=2; }
+   if ((y & 0x80000000) == 0) { n+=1; y<<=1; }
+   if ((y & 0x80000000) == 0) { n+=1;}
+   return n;
+  */
+  }
+
 
   /** returns true if v is a power of two or zero*/
   public static boolean isPowerOfTwo(int v) {

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/ByteBlockPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/ByteBlockPool.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/ByteBlockPool.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/ByteBlockPool.java Fri Jun 24 23:07:10 2011
@@ -18,6 +18,8 @@ package org.apache.lucene.util;
  */
 import java.util.Arrays;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
 import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
 
 /** 
@@ -78,6 +80,33 @@ public final class ByteBlockPool {
     }
     
   }
+  
+  public static class DirectTrackingAllocator extends Allocator {
+    private final AtomicLong bytesUsed;
+    
+    public DirectTrackingAllocator(AtomicLong bytesUsed) {
+      this(BYTE_BLOCK_SIZE, bytesUsed);
+    }
+
+    public DirectTrackingAllocator(int blockSize, AtomicLong bytesUsed) {
+      super(blockSize);
+      this.bytesUsed = bytesUsed;
+    }
+
+    public byte[] getByteBlock() {
+      bytesUsed.addAndGet(blockSize);
+      return new byte[blockSize];
+    }
+    @Override
+    public void recycleByteBlocks(byte[][] blocks, int start, int end) {
+      bytesUsed.addAndGet(-((end-start)* blockSize));
+      for (int i = start; i < end; i++) {
+        blocks[i] = null;
+      }
+    }
+    
+  };
+
 
   public byte[][] buffers = new byte[10][];
 
@@ -92,6 +121,20 @@ public final class ByteBlockPool {
   public ByteBlockPool(Allocator allocator) {
     this.allocator = allocator;
   }
+  
+  public void dropBuffersAndReset() {
+    if (bufferUpto != -1) {
+      // Recycle all but the first buffer
+      allocator.recycleByteBlocks(buffers, 0, 1+bufferUpto);
+
+      // Re-use the first buffer
+      bufferUpto = -1;
+      byteUpto = BYTE_BLOCK_SIZE;
+      byteOffset = -BYTE_BLOCK_SIZE;
+      buffers = new byte[10][];
+      buffer = null;
+    }
+  }
 
   public void reset() {
     if (bufferUpto != -1) {
@@ -115,7 +158,7 @@ public final class ByteBlockPool {
       buffer = buffers[0];
     }
   }
-
+  
   public void nextBuffer() {
     if (1+bufferUpto == buffers.length) {
       byte[][] newBuffers = new byte[ArrayUtil.oversize(buffers.length+1,

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/BytesRefHash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/BytesRefHash.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/BytesRefHash.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/BytesRefHash.java Fri Jun 24 23:07:10 2011
@@ -227,8 +227,9 @@ public final class BytesRefHash {
   public void clear(boolean resetPool) {
     lastCount = count;
     count = 0;
-    if (resetPool)
-      pool.reset();
+    if (resetPool) {
+      pool.dropBuffersAndReset();
+    }
     bytesStart = bytesStartArray.clear();
     if (lastCount != -1 && shrink(lastCount)) {
       // shrink clears the hash entries
@@ -240,6 +241,16 @@ public final class BytesRefHash {
   public void clear() {
     clear(true);
   }
+  
+  /**
+   * Closes the BytesRefHash and releases all internally used memory
+   */
+  public void close() {
+    clear(true);
+    ords = null;
+    bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT
+        * -hashSize);
+  }
 
   /**
    * Adds a new {@link BytesRef}
@@ -332,6 +343,7 @@ public final class BytesRefHash {
         // 1 byte to store length
         buffer[bufferUpto] = (byte) length;
         pool.byteUpto += length + 1;
+        assert length >= 0: "Length must be positive: " + length;
         System.arraycopy(bytes.bytes, bytes.offset, buffer, bufferUpto + 1,
             length);
       } else {
@@ -452,8 +464,14 @@ public final class BytesRefHash {
    * effect.
    */
   public void reinit() {
-    if (bytesStart == null)
+    if (bytesStart == null) {
       bytesStart = bytesStartArray.init();
+    }
+    
+    if (ords == null) {
+      ords = new int[hashSize];
+      bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT * hashSize);
+    }
   }
 
   /**
@@ -514,17 +532,62 @@ public final class BytesRefHash {
      */
     public abstract AtomicLong bytesUsed();
   }
+  
+  /**
+   * A direct {@link BytesStartArray} that tracks all memory allocation using an {@link AtomicLong} instance.
+   */
+  public static class TrackingDirectBytesStartArray extends BytesStartArray {
+    protected final int initSize;
+    private int[] bytesStart;
+    protected final AtomicLong bytesUsed;
+    
+    public TrackingDirectBytesStartArray(int initSize, AtomicLong bytesUsed) {
+      this.initSize = initSize;
+      this.bytesUsed = bytesUsed;
+    }
 
-  public static class DirectBytesStartArray extends BytesStartArray {
+    @Override
+    public int[] clear() {
+      if (bytesStart != null) {
+        bytesUsed.addAndGet(-bytesStart.length * RamUsageEstimator.NUM_BYTES_INT);
+      }
+      return bytesStart = null;
+    }
 
+    @Override
+    public int[] grow() {
+      assert bytesStart != null;
+      final int oldSize = bytesStart.length;
+      bytesStart = ArrayUtil.grow(bytesStart, bytesStart.length + 1);
+      bytesUsed.addAndGet((bytesStart.length - oldSize) * RamUsageEstimator.NUM_BYTES_INT);
+      return bytesStart;
+    }
+
+    @Override
+    public int[] init() {
+      bytesStart = new int[ArrayUtil.oversize(initSize,
+          RamUsageEstimator.NUM_BYTES_INT)];
+      bytesUsed.addAndGet((bytesStart.length) * RamUsageEstimator.NUM_BYTES_INT);
+      return bytesStart;
+    }
+
+    @Override
+    public AtomicLong bytesUsed() {
+      return bytesUsed;
+    }
+  }
+
+  public static class DirectBytesStartArray extends BytesStartArray {
     protected final int initSize;
     private int[] bytesStart;
-    private final AtomicLong bytesUsed = new AtomicLong(0);
-
+    private final AtomicLong bytesUsed;
+    
     public DirectBytesStartArray(int initSize) {
+      this.bytesUsed = new AtomicLong(0);
       this.initSize = initSize;
     }
 
+
     @Override
     public int[] clear() {
       return bytesStart = null;
@@ -546,6 +609,5 @@ public final class BytesRefHash {
     public AtomicLong bytesUsed() {
       return bytesUsed;
     }
-
   }
 }

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java Fri Jun 24 23:07:10 2011
@@ -208,7 +208,7 @@ public final class FieldCacheSanityCheck
       
       if (seen.contains(rf)) continue;
 
-      List<Object> kids = getAllDecendentReaderKeys(rf.readerKey);
+      List<Object> kids = getAllDescendentReaderKeys(rf.readerKey);
       for (Object kidKey : kids) {
         ReaderField kid = new ReaderField(kidKey, rf.fieldName);
         
@@ -252,7 +252,7 @@ public final class FieldCacheSanityCheck
       badness = badEntries.toArray(badness);
 
       insanity.add(new Insanity(InsanityType.SUBREADER,
-                                "Found caches for decendents of " + 
+                                "Found caches for descendants of " + 
                                 parent.toString(),
                                 badness));
     }
@@ -266,7 +266,7 @@ public final class FieldCacheSanityCheck
    * the hierarchy of subReaders building up a list of the objects 
    * returned by obj.getFieldCacheKey()
    */
-  private List<Object> getAllDecendentReaderKeys(Object seed) {
+  private List<Object> getAllDescendentReaderKeys(Object seed) {
     List<Object> all = new ArrayList<Object>(17); // will grow as we iter
     all.add(seed);
     for (int i = 0; i < all.size(); i++) {

Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/OpenBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/OpenBitSet.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/OpenBitSet.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/util/OpenBitSet.java Fri Jun 24 23:07:10 2011
@@ -659,7 +659,38 @@ public class OpenBitSet extends DocIdSet
   }
 
 
+  /** Returns the index of the first set bit starting downwards at
+   *  the index specified.
+   *  -1 is returned if there are no more set bits.
+   */
+  public int prevSetBit(int index) {
+    int i = index >> 6;
+    final int subIndex;
+    long word;
+    if (i >= wlen) {
+      i = wlen - 1;
+      if (i < 0) return -1;
+      subIndex = 63;  // last possible bit
+      word = bits[i];
+    } else {
+      if (i < 0) return -1;
+      subIndex = index & 0x3f;  // index within the word
+      word = (bits[i] << (63-subIndex));  // skip all the bits to the left of index
+    }
+
+    if (word != 0) {
+      return (i << 6) + subIndex - Long.numberOfLeadingZeros(word); // See LUCENE-3197
+    }
+
+    while (--i >= 0) {
+      word = bits[i];
+      if (word !=0 ) {
+        return (i << 6) + 63 - Long.numberOfLeadingZeros(word);
+      }
+    }
 
+    return -1;
+  }
 
   @Override
   public Object clone() {