You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/06/10 15:18:57 UTC

svn commit: r1134311 [2/3] - in /lucene/dev/trunk: ./ dev-tools/idea/lucene/contrib/ lucene/ lucene/backwards/ lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/ lucene/contrib/memory/src/java/org/apache/lucene/index/memory/ luc...

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReader.java Fri Jun 10 13:18:54 2011
@@ -29,6 +29,8 @@ import java.util.concurrent.atomic.Atomi
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.index.codecs.PerDocValues;
+import org.apache.lucene.index.values.IndexDocValues;
 import org.apache.lucene.store.BufferedIndexInput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
@@ -839,4 +841,9 @@ public class SegmentReader extends Index
     // longer used (all SegmentReaders sharing it have been
     // closed).
   }
+  
+  @Override
+  public PerDocValues perDocValues() throws IOException {
+    return core.perDocProducer;
+  }
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java Fri Jun 10 13:18:54 2011
@@ -43,7 +43,7 @@ public class SegmentWriteState {
   public BitVector deletedDocs;
 
   final SegmentCodecs segmentCodecs;
-  public final String codecId;
+  public final int codecId;
 
   /** Expert: The fraction of terms in the "dictionary" which should be stored
    * in RAM.  Smaller values use more memory, but make searching slightly
@@ -53,7 +53,7 @@ public class SegmentWriteState {
   public int termIndexInterval;                   // TODO: this should be private to the codec, not settable here or in IWC
 
   public SegmentWriteState(PrintStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos,
-                           int numDocs, int termIndexInterval, SegmentCodecs segmentCodecs, BufferedDeletes segDeletes) {
+      int numDocs, int termIndexInterval, SegmentCodecs segmentCodecs, BufferedDeletes segDeletes) {
     this.infoStream = infoStream;
     this.segDeletes = segDeletes;
     this.directory = directory;
@@ -62,13 +62,13 @@ public class SegmentWriteState {
     this.numDocs = numDocs;
     this.termIndexInterval = termIndexInterval;
     this.segmentCodecs = segmentCodecs;
-    codecId = "";
+    codecId = -1;
   }
   
   /**
    * Create a shallow {@link SegmentWriteState} copy final a codec ID
    */
-  SegmentWriteState(SegmentWriteState state, String codecId) {
+  SegmentWriteState(SegmentWriteState state, int codecId) {
     infoStream = state.infoStream;
     directory = state.directory;
     segmentName = state.segmentName;

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java Fri Jun 10 13:18:54 2011
@@ -26,6 +26,7 @@ import org.apache.lucene.util.ReaderUtil
 
 import org.apache.lucene.index.DirectoryReader; // javadoc
 import org.apache.lucene.index.MultiReader; // javadoc
+import org.apache.lucene.index.codecs.PerDocValues;
 
 /**
  * This class forces a composite reader (eg a {@link
@@ -65,6 +66,11 @@ public final class SlowMultiReaderWrappe
   }
 
   @Override
+  public PerDocValues perDocValues() throws IOException {
+    return MultiPerDocValues.getPerDocs(in);
+  }
+
+  @Override
   public Bits getDeletedDocs() {
     return MultiFields.getDeletedDocs(in);
   }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java Fri Jun 10 13:18:54 2011
@@ -108,7 +108,7 @@ public class BlockTermsReader extends Fi
   //private String segment;
   
   public BlockTermsReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, int readBufferSize,
-                          int termsCacheSize, String codecId)
+                          int termsCacheSize, int codecId)
     throws IOException {
     
     this.postingsReader = postingsReader;
@@ -196,7 +196,7 @@ public class BlockTermsReader extends Fi
     }
   }
 
-  public static void files(Directory dir, SegmentInfo segmentInfo, String id, Collection<String> files) {
+  public static void files(Directory dir, SegmentInfo segmentInfo, int id, Collection<String> files) {
     files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, BlockTermsWriter.TERMS_EXTENSION));
   }
 

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/Codec.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/Codec.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/Codec.java Fri Jun 10 13:18:54 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs;
 import java.io.IOException;
 import java.util.Set;
 
+import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SegmentReadState;
@@ -50,6 +51,10 @@ public abstract class Codec {
    *  returns, it must hold open any files it will need to
    *  use; else, those files may be deleted. */
   public abstract FieldsProducer fieldsProducer(SegmentReadState state) throws IOException;
+  
+  public abstract PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException;
+  
+  public abstract PerDocValues docsProducer(SegmentReadState state) throws IOException;
 
   /**
    * Gathers files associated with this segment
@@ -59,7 +64,7 @@ public abstract class Codec {
    * @param id the codec id within this segment
    * @param files the of files to add the codec files to.
    */
-  public abstract void files(Directory dir, SegmentInfo segmentInfo, String id, Set<String> files) throws IOException;
+  public abstract void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException;
 
   /** Records all file extensions this codec uses */
   public abstract void getExtensions(Set<String> extensions);

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java Fri Jun 10 13:18:54 2011
@@ -22,6 +22,7 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.Map.Entry;
 
 /** Holds a set of codecs, keyed by name.  You subclass
  *  this, instantiate it, and register your codecs, then

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java Fri Jun 10 13:18:54 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.TermsEnum;
 
 import java.io.IOException;
 import java.io.Closeable;
@@ -35,7 +36,7 @@ public abstract class FieldsConsumer imp
 
   /** Add a new field */
   public abstract TermsConsumer addField(FieldInfo field) throws IOException;
-
+  
   /** Called when we are done adding everything. */
   public abstract void close() throws IOException;
 
@@ -45,8 +46,12 @@ public abstract class FieldsConsumer imp
     String field;
     while((field = fieldsEnum.next()) != null) {
       mergeState.fieldInfo = mergeState.fieldInfos.fieldInfo(field);
-      final TermsConsumer termsConsumer = addField(mergeState.fieldInfo);
-      termsConsumer.merge(mergeState, fieldsEnum.terms());
+      assert mergeState.fieldInfo != null : "FieldInfo for field is null: "+ field;
+      TermsEnum terms = fieldsEnum.terms();
+      if (terms != null) {
+        final TermsConsumer termsConsumer = addField(mergeState.fieldInfo);
+        termsConsumer.merge(mergeState, terms);
+      }
     }
   }
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java Fri Jun 10 13:18:54 2011
@@ -17,10 +17,12 @@ package org.apache.lucene.index.codecs;
  * limitations under the License.
  */
 
-import org.apache.lucene.index.Fields;
-
-import java.io.IOException;
 import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.Terms;
 
 /** Abstract API that consumes terms, doc, freq, prox and
  *  payloads postings.  Concrete implementations of this
@@ -33,4 +35,28 @@ import java.io.Closeable;
 public abstract class FieldsProducer extends Fields implements Closeable {
   public abstract void close() throws IOException;
   public abstract void loadTermsIndex(int indexDivisor) throws IOException;
+
+  public static final FieldsProducer EMPTY = new FieldsProducer() {
+    
+    @Override
+    public Terms terms(String field) throws IOException {
+      return null;
+    }
+    
+    @Override
+    public FieldsEnum iterator() throws IOException {
+      return FieldsEnum.EMPTY;
+    }
+    
+    @Override
+    public void loadTermsIndex(int indexDivisor) throws IOException {
+      
+    }
+    
+    @Override
+    public void close() throws IOException {
+      
+    }
+  };
+  
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java Fri Jun 10 13:18:54 2011
@@ -68,7 +68,7 @@ public class FixedGapTermsIndexReader ex
   // start of the field info data
   protected long dirOffset;
 
-  public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator<BytesRef> termComp, String codecId)
+  public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator<BytesRef> termComp, int codecId)
     throws IOException {
 
     this.termComp = termComp;
@@ -406,7 +406,7 @@ public class FixedGapTermsIndexReader ex
     }
   }
 
-  public static void files(Directory dir, SegmentInfo info, String id, Collection<String> files) {
+  public static void files(Directory dir, SegmentInfo info, int id, Collection<String> files) {
     files.add(IndexFileNames.segmentFileName(info.name, id, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION));
   }
 

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java Fri Jun 10 13:18:54 2011
@@ -57,8 +57,7 @@ public class VariableGapTermsIndexReader
   protected long dirOffset;
 
   final String segment;
-
-  public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, String codecId)
+  public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, int codecId)
     throws IOException {
 
     in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION));
@@ -159,15 +158,11 @@ public class VariableGapTermsIndexReader
 
   private final class FieldIndexData {
 
-    private final FieldInfo fieldInfo;
     private final long indexStart;
-
     // Set only if terms index is loaded:
     private volatile FST<Long> fst;
 
     public FieldIndexData(FieldInfo fieldInfo, long indexStart) throws IOException {
-
-      this.fieldInfo = fieldInfo;
       this.indexStart = indexStart;
 
       if (indexDivisor > 0) {
@@ -241,7 +236,7 @@ public class VariableGapTermsIndexReader
     }
   }
 
-  public static void files(Directory dir, SegmentInfo info, String id, Collection<String> files) {
+  public static void files(Directory dir, SegmentInfo info, int id, Collection<String> files) {
     files.add(IndexFileNames.segmentFileName(info.name, id, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION));
   }
 

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java Fri Jun 10 13:18:54 2011
@@ -22,11 +22,14 @@ import java.io.IOException;
 
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 
 /** Codec that reads the pre-flex-indexing postings
  *  format.  It does not provide a writer because newly
@@ -66,7 +69,7 @@ public class PreFlexCodec extends Codec 
   }
 
   @Override
-  public void files(Directory dir, SegmentInfo info, String id, Set<String> files) throws IOException {
+  public void files(Directory dir, SegmentInfo info, int id, Set<String> files) throws IOException {
     // preflex fields have no codec ID - we ignore it here
     PreFlexFields.files(dir, info, files);
   }
@@ -78,4 +81,14 @@ public class PreFlexCodec extends Codec 
     extensions.add(TERMS_EXTENSION);
     extensions.add(TERMS_INDEX_EXTENSION);
   }
+
+  @Override
+  public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+    throw new UnsupportedOperationException("PerDocConsumer is not supported by Preflex codec");
+  }
+
+  @Override
+  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+    throw new UnsupportedOperationException("PerDocValues is not supported by Preflex codec");
+  }
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java Fri Jun 10 13:18:54 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs.p
 import java.io.IOException;
 import java.util.Set;
 
+import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SegmentReadState;
@@ -28,8 +29,12 @@ import org.apache.lucene.index.codecs.Po
 import org.apache.lucene.index.codecs.standard.StandardPostingsWriter;
 import org.apache.lucene.index.codecs.PostingsReaderBase;
 import org.apache.lucene.index.codecs.standard.StandardPostingsReader;
+import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.VariableGapTermsIndexReader;
 import org.apache.lucene.index.codecs.VariableGapTermsIndexWriter;
 import org.apache.lucene.index.codecs.BlockTermsReader;
@@ -38,6 +43,7 @@ import org.apache.lucene.index.codecs.Te
 import org.apache.lucene.index.codecs.TermsIndexWriterBase;
 import org.apache.lucene.index.codecs.standard.StandardCodec;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 
 /** This codec "inlines" the postings for terms that have
@@ -147,14 +153,26 @@ public class PulsingCodec extends Codec 
   }
 
   @Override
-  public void files(Directory dir, SegmentInfo segmentInfo, String id, Set<String> files) throws IOException {
+  public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
     StandardPostingsReader.files(dir, segmentInfo, id, files);
     BlockTermsReader.files(dir, segmentInfo, id, files);
     VariableGapTermsIndexReader.files(dir, segmentInfo, id, files);
+    DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     StandardCodec.getStandardExtensions(extensions);
+    DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
+  }
+  
+  @Override
+  public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+    return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+  }
+
+  @Override
+  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
   }
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java Fri Jun 10 13:18:54 2011
@@ -58,8 +58,7 @@ public class SepPostingsReaderImpl exten
   int maxSkipLevels;
   int skipMinimum;
 
-  public SepPostingsReaderImpl(Directory dir, SegmentInfo segmentInfo, int readBufferSize, IntStreamFactory intFactory, String codecId) throws IOException {
-
+  public SepPostingsReaderImpl(Directory dir, SegmentInfo segmentInfo, int readBufferSize, IntStreamFactory intFactory, int codecId) throws IOException {
     boolean success = false;
     try {
 
@@ -85,7 +84,7 @@ public class SepPostingsReaderImpl exten
     }
   }
 
-  public static void files(SegmentInfo segmentInfo, String codecId, Collection<String> files) throws IOException {
+  public static void files(SegmentInfo segmentInfo, int codecId, Collection<String> files) throws IOException {
     files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.DOC_EXTENSION));
     files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.SKIP_EXTENSION));
 

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java Fri Jun 10 13:18:54 2011
@@ -20,14 +20,20 @@ package org.apache.lucene.index.codecs.s
 import java.io.IOException;
 import java.util.Set;
 
+import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 
 /** For debugging, curiosity, transparency only!!  Do not
  *  use this codec in production.
@@ -56,17 +62,30 @@ public class SimpleTextCodec extends Cod
   /** Extension of freq postings file */
   static final String POSTINGS_EXTENSION = "pst";
 
-  static String getPostingsFileName(String segment, String id) {
+  static String getPostingsFileName(String segment, int id) {
     return IndexFileNames.segmentFileName(segment, id, POSTINGS_EXTENSION);
   }
 
   @Override
-  public void files(Directory dir, SegmentInfo segmentInfo, String id, Set<String> files) throws IOException {
+  public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
     files.add(getPostingsFileName(segmentInfo.name, id));
+    DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     extensions.add(POSTINGS_EXTENSION);
+    DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
+  }
+  
+  // TODO: would be great if these used a plain text impl
+  @Override
+  public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+    return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+  }
+
+  @Override
+  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
   }
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java Fri Jun 10 13:18:54 2011
@@ -58,7 +58,7 @@ class SimpleTextFieldsReader extends Fie
   final static BytesRef PAYLOAD = SimpleTextFieldsWriter.PAYLOAD;
 
   public SimpleTextFieldsReader(SegmentReadState state) throws IOException {
-    in = state.dir.openInput(SimpleTextCodec.getPostingsFileName(state.segmentInfo.name, ""+state.codecId));
+    in = state.dir.openInput(SimpleTextCodec.getPostingsFileName(state.segmentInfo.name, state.codecId));
    
     fieldInfos = state.fieldInfos;
   }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java Fri Jun 10 13:18:54 2011
@@ -20,12 +20,16 @@ package org.apache.lucene.index.codecs.s
 import java.io.IOException;
 import java.util.Set;
 
+import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.PostingsWriterBase;
 import org.apache.lucene.index.codecs.PostingsReaderBase;
 import org.apache.lucene.index.codecs.TermsIndexWriterBase;
@@ -34,7 +38,9 @@ import org.apache.lucene.index.codecs.Va
 import org.apache.lucene.index.codecs.VariableGapTermsIndexReader;
 import org.apache.lucene.index.codecs.BlockTermsWriter;
 import org.apache.lucene.index.codecs.BlockTermsReader;
+import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 
 /** Default codec. 
  *  @lucene.experimental */
@@ -130,15 +136,17 @@ public class StandardCodec extends Codec
   static final String PROX_EXTENSION = "prx";
 
   @Override
-  public void files(Directory dir, SegmentInfo segmentInfo, String id, Set<String> files) throws IOException {
+  public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
     StandardPostingsReader.files(dir, segmentInfo, id, files);
     BlockTermsReader.files(dir, segmentInfo, id, files);
     VariableGapTermsIndexReader.files(dir, segmentInfo, id, files);
+    DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     getStandardExtensions(extensions);
+    DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
   }
 
   public static void getStandardExtensions(Set<String> extensions) {
@@ -147,4 +155,14 @@ public class StandardCodec extends Codec
     BlockTermsReader.getExtensions(extensions);
     VariableGapTermsIndexReader.getIndexExtensions(extensions);
   }
+
+  @Override
+  public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+    return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+  }
+
+  @Override
+  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
+  }
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java Fri Jun 10 13:18:54 2011
@@ -51,7 +51,7 @@ public class StandardPostingsReader exte
 
   //private String segment;
 
-  public StandardPostingsReader(Directory dir, SegmentInfo segmentInfo, int readBufferSize, String codecId) throws IOException {
+  public StandardPostingsReader(Directory dir, SegmentInfo segmentInfo, int readBufferSize, int codecId) throws IOException {
     freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, StandardCodec.FREQ_EXTENSION),
                            readBufferSize);
     //this.segment = segmentInfo.name;
@@ -71,7 +71,7 @@ public class StandardPostingsReader exte
     }
   }
 
-  public static void files(Directory dir, SegmentInfo segmentInfo, String id, Collection<String> files) throws IOException {
+  public static void files(Directory dir, SegmentInfo segmentInfo, int id, Collection<String> files) throws IOException {
     files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, StandardCodec.FREQ_EXTENSION));
     if (segmentInfo.getHasProx()) {
       files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, StandardCodec.PROX_EXTENSION));

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldComparator.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldComparator.java Fri Jun 10 13:18:54 2011
@@ -20,8 +20,10 @@ package org.apache.lucene.search;
 import java.io.IOException;
 
 import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-import org.apache.lucene.search.FieldCache.DocTermsIndex;
+import org.apache.lucene.index.values.IndexDocValues;
+import org.apache.lucene.index.values.IndexDocValues.Source;
 import org.apache.lucene.search.FieldCache.DocTerms;
+import org.apache.lucene.search.FieldCache.DocTermsIndex;
 import org.apache.lucene.search.cache.ByteValuesCreator;
 import org.apache.lucene.search.cache.CachedArray;
 import org.apache.lucene.search.cache.CachedArrayCreator;
@@ -38,9 +40,9 @@ import org.apache.lucene.search.cache.Ca
 import org.apache.lucene.search.cache.CachedArray.ShortValues;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.packed.Direct8;
 import org.apache.lucene.util.packed.Direct16;
 import org.apache.lucene.util.packed.Direct32;
+import org.apache.lucene.util.packed.Direct8;
 import org.apache.lucene.util.packed.PackedInts;
 
 /**
@@ -157,7 +159,6 @@ public abstract class FieldComparator {
    *   comparators can just return "this" to reuse the same
    *   comparator across segments
    * @throws IOException
-   * @throws IOException
    */
   public abstract FieldComparator setNextReader(AtomicReaderContext context) throws IOException;
 
@@ -328,6 +329,68 @@ public abstract class FieldComparator {
     }
   }
 
+  /** Uses float index values to sort by ascending value */
+  public static final class FloatDocValuesComparator extends FieldComparator {
+    private final double[] values;
+    private Source currentReaderValues;
+    private final String field;
+    private double bottom;
+
+    FloatDocValuesComparator(int numHits, String field) {
+      values = new double[numHits];
+      this.field = field;
+    }
+
+    @Override
+    public int compare(int slot1, int slot2) {
+      final double v1 = values[slot1];
+      final double v2 = values[slot2];
+      if (v1 > v2) {
+        return 1;
+      } else if (v1 < v2) {
+        return -1;
+      } else {
+        return 0;
+      }
+    }
+
+    @Override
+    public int compareBottom(int doc) {
+      final double v2 = currentReaderValues.getFloat(doc);
+      if (bottom > v2) {
+        return 1;
+      } else if (bottom < v2) {
+        return -1;
+      } else {
+        return 0;
+      }
+    }
+
+    @Override
+    public void copy(int slot, int doc) {
+      values[slot] = currentReaderValues.getFloat(doc); 
+    }
+
+    @Override
+    public FieldComparator setNextReader(AtomicReaderContext context) throws IOException {
+      final IndexDocValues docValues = context.reader.docValues(field);
+      if (docValues != null) {
+        currentReaderValues = docValues.getSource(); 
+      }
+      return this;
+    }
+    
+    @Override
+    public void setBottom(final int bottom) {
+      this.bottom = values[bottom];
+    }
+
+    @Override
+    public Comparable<Double> value(int slot) {
+      return Double.valueOf(values[slot]);
+    }
+  }
+
   /** Parses field's values as float (using {@link
    *  FieldCache#getFloats} and sorts by ascending value */
   public static final class FloatComparator extends NumericComparator<FloatValues> {
@@ -536,6 +599,72 @@ public abstract class FieldComparator {
     }
   }
 
+  /** Loads int index values and sorts by ascending value. */
+  public static final class IntDocValuesComparator extends FieldComparator {
+    private final long[] values;
+    private Source currentReaderValues;
+    private final String field;
+    private long bottom;
+
+    IntDocValuesComparator(int numHits, String field) {
+      values = new long[numHits];
+      this.field = field;
+    }
+
+    @Override
+    public int compare(int slot1, int slot2) {
+      // TODO: there are sneaky non-branch ways to compute
+      // -1/+1/0 sign
+      final long v1 = values[slot1];
+      final long v2 = values[slot2];
+      if (v1 > v2) {
+        return 1;
+      } else if (v1 < v2) {
+        return -1;
+      } else {
+        return 0;
+      }
+    }
+
+    @Override
+    public int compareBottom(int doc) {
+      // TODO: there are sneaky non-branch ways to compute
+      // -1/+1/0 sign
+      final long v2 = currentReaderValues.getInt(doc);
+      if (bottom > v2) {
+        return 1;
+      } else if (bottom < v2) {
+        return -1;
+      } else {
+        return 0;
+      }
+    }
+
+    @Override
+    public void copy(int slot, int doc) {
+      values[slot] = currentReaderValues.getInt(doc); 
+    }
+
+    @Override
+    public FieldComparator setNextReader(AtomicReaderContext context) throws IOException {
+      IndexDocValues docValues = context.reader.docValues(field);
+      if (docValues != null) {
+        currentReaderValues = docValues.getSource();
+      }
+      return this;
+    }
+    
+    @Override
+    public void setBottom(final int bottom) {
+      this.bottom = values[bottom];
+    }
+
+    @Override
+    public Comparable<Long> value(int slot) {
+      return Long.valueOf(values[slot]);
+    }
+  }
+
   /** Parses field's values as long (using {@link
    *  FieldCache#getLongs} and sorts by ascending value */
   public static final class LongComparator extends NumericComparator<LongValues> {

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ReqExclScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ReqExclScorer.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ReqExclScorer.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ReqExclScorer.java Fri Jun 10 13:18:54 2011
@@ -23,7 +23,7 @@ import java.io.IOException;
 /** A Scorer for queries with a required subscorer
  * and an excluding (prohibited) sub DocIdSetIterator.
  * <br>
- * This <code>Scorer</code> implements {@link Scorer#skipTo(int)},
+ * This <code>Scorer</code> implements {@link Scorer#advance(int)},
  * and it uses the skipTo() on the given scorers.
  */
 class ReqExclScorer extends Scorer {

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ReqOptSumScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ReqOptSumScorer.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ReqOptSumScorer.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ReqOptSumScorer.java Fri Jun 10 13:18:54 2011
@@ -21,7 +21,7 @@ import java.io.IOException;
 /** A Scorer for queries with a required part and an optional part.
  * Delays skipTo() on the optional part until a score() is needed.
  * <br>
- * This <code>Scorer</code> implements {@link Scorer#skipTo(int)}.
+ * This <code>Scorer</code> implements {@link Scorer#advance(int)}.
  */
 class ReqOptSumScorer extends Scorer {
   /** The scorers passed from the constructor.

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/SortField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/SortField.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/SortField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/SortField.java Fri Jun 10 13:18:54 2011
@@ -18,9 +18,15 @@ package org.apache.lucene.search;
  */
 
 import java.io.IOException;
+import java.util.Comparator;
 
 import org.apache.lucene.search.cache.*;
 import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.BytesRef;
+
+// TODO(simonw) -- for cleaner transition, maybe we should make
+// a new SortField that subclasses this one and always uses
+// index values?
 
 /**
  * Stores information about how to sort documents by terms in an individual
@@ -81,6 +87,9 @@ public class SortField {
    * uses ordinals to do the sorting. */
   public static final int STRING_VAL = 11;
   
+  /** Sort use byte[] index values. */
+  public static final int BYTES = 12;
+  
   /** Represents sorting by document score (relevance). */
   public static final SortField FIELD_SCORE = new SortField (null, SCORE);
 
@@ -390,6 +399,26 @@ public class SortField {
     return hash;
   }
 
+  private boolean useIndexValues;
+
+  public void setUseIndexValues(boolean b) {
+    useIndexValues = b;
+  }
+
+  public boolean getUseIndexValues() {
+    return useIndexValues;
+  }
+
+  private Comparator<BytesRef> bytesComparator = BytesRef.getUTF8SortedAsUnicodeComparator();
+
+  public void setBytesComparator(Comparator<BytesRef> b) {
+    bytesComparator = b;
+  }
+
+  public Comparator<BytesRef> getBytesComparator() {
+    return bytesComparator;
+  }
+
   /** Returns the {@link FieldComparator} to use for
    * sorting.
    *
@@ -412,10 +441,18 @@ public class SortField {
       return new FieldComparator.DocComparator(numHits);
 
     case SortField.INT:
-      return new FieldComparator.IntComparator(numHits, (IntValuesCreator)creator, (Integer)missingValue );
+      if (useIndexValues) {
+        return new FieldComparator.IntDocValuesComparator(numHits, field);
+      } else {
+        return new FieldComparator.IntComparator(numHits, (IntValuesCreator)creator, (Integer) missingValue);
+      }
 
     case SortField.FLOAT:
-      return new FieldComparator.FloatComparator(numHits, (FloatValuesCreator)creator, (Float)missingValue );
+      if (useIndexValues) {
+        return new FieldComparator.FloatDocValuesComparator(numHits, field);
+      } else {
+        return new FieldComparator.FloatComparator(numHits, (FloatValuesCreator) creator, (Float) missingValue);
+      }
 
     case SortField.LONG:
       return new FieldComparator.LongComparator(numHits, (LongValuesCreator)creator, (Long)missingValue );

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/ArrayUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/ArrayUtil.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/ArrayUtil.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/ArrayUtil.java Fri Jun 10 13:18:54 2011
@@ -255,6 +255,19 @@ public final class ArrayUtil {
     return grow(array, 1 + array.length);
   }
 
+  public static double[] grow(double[] array, int minSize) {
+    if (array.length < minSize) {
+      double[] newArray = new double[oversize(minSize, RamUsageEstimator.NUM_BYTES_DOUBLE)];
+      System.arraycopy(array, 0, newArray, 0, array.length);
+      return newArray;
+    } else
+      return array;
+  }
+
+  public static double[] grow(double[] array) {
+    return grow(array, 1 + array.length);
+  }
+
   public static short[] shrink(short[] array, int targetSize) {
     final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_SHORT);
     if (newSize != array.length) {

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/ByteBlockPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/ByteBlockPool.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/ByteBlockPool.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/ByteBlockPool.java Fri Jun 10 13:18:54 2011
@@ -18,6 +18,8 @@ package org.apache.lucene.util;
  */
 import java.util.Arrays;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
 import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
 
 /** 
@@ -78,6 +80,33 @@ public final class ByteBlockPool {
     }
     
   }
+  
+  public static class DirectTrackingAllocator extends Allocator {
+    private final AtomicLong bytesUsed;
+    
+    public DirectTrackingAllocator(AtomicLong bytesUsed) {
+      this(BYTE_BLOCK_SIZE, bytesUsed);
+    }
+
+    public DirectTrackingAllocator(int blockSize, AtomicLong bytesUsed) {
+      super(blockSize);
+      this.bytesUsed = bytesUsed;
+    }
+
+    public byte[] getByteBlock() {
+      bytesUsed.addAndGet(blockSize);
+      return new byte[blockSize];
+    }
+    @Override
+    public void recycleByteBlocks(byte[][] blocks, int start, int end) {
+      bytesUsed.addAndGet(-((end-start)* blockSize));
+      for (int i = start; i < end; i++) {
+        blocks[i] = null;
+      }
+    }
+    
+  };
+
 
   public byte[][] buffers = new byte[10][];
 
@@ -92,6 +121,20 @@ public final class ByteBlockPool {
   public ByteBlockPool(Allocator allocator) {
     this.allocator = allocator;
   }
+  
+  public void dropBuffersAndReset() {
+    if (bufferUpto != -1) {
+      // Recycle all but the first buffer
+      allocator.recycleByteBlocks(buffers, 0, 1+bufferUpto);
+
+      // Re-use the first buffer
+      bufferUpto = -1;
+      byteUpto = BYTE_BLOCK_SIZE;
+      byteOffset = -BYTE_BLOCK_SIZE;
+      buffers = new byte[10][];
+      buffer = null;
+    }
+  }
 
   public void reset() {
     if (bufferUpto != -1) {
@@ -115,7 +158,7 @@ public final class ByteBlockPool {
       buffer = buffers[0];
     }
   }
-
+  
   public void nextBuffer() {
     if (1+bufferUpto == buffers.length) {
       byte[][] newBuffers = new byte[ArrayUtil.oversize(buffers.length+1,

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/BytesRefHash.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/BytesRefHash.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/BytesRefHash.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/BytesRefHash.java Fri Jun 10 13:18:54 2011
@@ -227,8 +227,9 @@ public final class BytesRefHash {
   public void clear(boolean resetPool) {
     lastCount = count;
     count = 0;
-    if (resetPool)
-      pool.reset();
+    if (resetPool) {
+      pool.dropBuffersAndReset();
+    }
     bytesStart = bytesStartArray.clear();
     if (lastCount != -1 && shrink(lastCount)) {
       // shrink clears the hash entries
@@ -240,6 +241,16 @@ public final class BytesRefHash {
   public void clear() {
     clear(true);
   }
+  
+  /**
+   * Closes the BytesRefHash and releases all internally used memory
+   */
+  public void close() {
+    clear(true);
+    ords = null;
+    bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT
+        * -hashSize);
+  }
 
   /**
    * Adds a new {@link BytesRef}
@@ -332,6 +343,7 @@ public final class BytesRefHash {
         // 1 byte to store length
         buffer[bufferUpto] = (byte) length;
         pool.byteUpto += length + 1;
+        assert length >= 0: "Length must be positive: " + length;
         System.arraycopy(bytes.bytes, bytes.offset, buffer, bufferUpto + 1,
             length);
       } else {
@@ -452,8 +464,14 @@ public final class BytesRefHash {
    * effect.
    */
   public void reinit() {
-    if (bytesStart == null)
+    if (bytesStart == null) {
       bytesStart = bytesStartArray.init();
+    }
+    
+    if (ords == null) {
+      ords = new int[hashSize];
+      bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT * hashSize);
+    }
   }
 
   /**
@@ -514,17 +532,62 @@ public final class BytesRefHash {
      */
     public abstract AtomicLong bytesUsed();
   }
+  
+  /**
+   * A direct {@link BytesStartArray} that tracks all memory allocation using an {@link AtomicLong} instance.
+   */
+  public static class TrackingDirectBytesStartArray extends BytesStartArray {
+    protected final int initSize;
+    private int[] bytesStart;
+    protected final AtomicLong bytesUsed;
+    
+    public TrackingDirectBytesStartArray(int initSize, AtomicLong bytesUsed) {
+      this.initSize = initSize;
+      this.bytesUsed = bytesUsed;
+    }
 
-  public static class DirectBytesStartArray extends BytesStartArray {
+    @Override
+    public int[] clear() {
+      if (bytesStart != null) {
+        bytesUsed.addAndGet(-bytesStart.length * RamUsageEstimator.NUM_BYTES_INT);
+      }
+      return bytesStart = null;
+    }
 
+    @Override
+    public int[] grow() {
+      assert bytesStart != null;
+      final int oldSize = bytesStart.length;
+      bytesStart = ArrayUtil.grow(bytesStart, bytesStart.length + 1);
+      bytesUsed.addAndGet((bytesStart.length - oldSize) * RamUsageEstimator.NUM_BYTES_INT);
+      return bytesStart;
+    }
+
+    @Override
+    public int[] init() {
+      bytesStart = new int[ArrayUtil.oversize(initSize,
+          RamUsageEstimator.NUM_BYTES_INT)];
+      bytesUsed.addAndGet((bytesStart.length) * RamUsageEstimator.NUM_BYTES_INT);
+      return bytesStart;
+    }
+
+    @Override
+    public AtomicLong bytesUsed() {
+      return bytesUsed;
+    }
+  }
+
+  public static class DirectBytesStartArray extends BytesStartArray {
     protected final int initSize;
     private int[] bytesStart;
-    private final AtomicLong bytesUsed = new AtomicLong(0);
-
+    private final AtomicLong bytesUsed;
+    
     public DirectBytesStartArray(int initSize) {
+      this.bytesUsed = new AtomicLong(0);
       this.initSize = initSize;
     }
 
+
     @Override
     public int[] clear() {
       return bytesStart = null;
@@ -546,6 +609,5 @@ public final class BytesRefHash {
     public AtomicLong bytesUsed() {
       return bytesUsed;
     }
-
   }
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java Fri Jun 10 13:18:54 2011
@@ -99,7 +99,7 @@ public final class PagedBytes {
       }
       return b;
     }
-
+    
     /**
      * Reads length as 1 or 2 byte vInt prefix, starting at <i>start</i>.
      * <p>
@@ -184,6 +184,55 @@ public final class PagedBytes {
       }
       return start;
     }
+    
+  
+    /**
+     * Gets a slice out of {@link PagedBytes} starting at <i>start</i>, the
+     * length is read as 1 or 2 byte vInt prefix. Iff the slice spans across a
+     * block border this method will allocate sufficient resources and copy the
+     * paged data.
+     * <p>
+     * Slices spanning more than one block are not supported.
+     * </p>
+     * 
+     * @lucene.internal
+     **/
+    public BytesRef fillSliceWithPrefix(BytesRef b, long start) {
+      final int index = (int) (start >> blockBits);
+      int offset = (int) (start & blockMask);
+      final byte[] block = blocks[index];
+      final int length;
+      if ((block[offset] & 128) == 0) {
+        length = block[offset];
+        offset = offset+1;
+      } else {
+        length = ((block[offset] & 0x7f) << 8) | (block[1+offset] & 0xff);
+        offset = offset+2;
+        assert length > 0;
+      }
+      assert length >= 0: "length=" + length;
+      b.length = length;
+      if (blockSize - offset >= length) {
+        // Within block
+        b.offset = offset;
+        b.bytes = blocks[index];
+      } else {
+        // Split
+        byte[] buffer = threadBuffers.get();
+        if (buffer == null) {
+          buffer = new byte[length];
+          threadBuffers.set(buffer);
+        } else if (buffer.length < length) {
+          buffer = ArrayUtil.grow(buffer, length);
+          threadBuffers.set(buffer);
+        }
+        b.bytes = buffer;
+        b.offset = 0;
+        System.arraycopy(blocks[index], offset, buffer, 0, blockSize-offset);
+        System.arraycopy(blocks[1+index], 0, buffer, blockSize-offset, length-(blockSize-offset));
+      }
+      return b;
+    }
 
     /** @lucene.internal */
     public byte[][] getBlocks() {

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Packed64.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Packed64.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Packed64.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Packed64.java Fri Jun 10 13:18:54 2011
@@ -182,7 +182,7 @@ class Packed64 extends PackedInts.Reader
     final int bitPos =     (int)(majorBitPos & MOD_MASK); // % BLOCK_SIZE);
 
     final int base = bitPos * FAC_BITPOS;
-
+    assert elementPos < blocks.length : "elementPos: " + elementPos + "; blocks.len: " + blocks.length;
     return ((blocks[elementPos] << shifts[base]) >>> shifts[base+1]) |
             ((blocks[elementPos+1] >>> shifts[base+2]) & readMasks[bitPos]);
   }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java Fri Jun 10 13:18:54 2011
@@ -93,12 +93,12 @@ final class PackedReaderIterator impleme
     final long bits = (long) bitsPerValue;
     final int posToSkip = ord - 1 - position;
     final long bitsToSkip = (bits * (long)posToSkip);
-    if(bitsToSkip < pendingBitsLeft ){ // enough bits left - no seek required
+    if (bitsToSkip < pendingBitsLeft) { // enough bits left - no seek required
       pendingBitsLeft -= bitsToSkip;
-    }else {
+    } else {
       final long skip = bitsToSkip-pendingBitsLeft;
       final long closestByte = (skip >> 6) << 3;
-      if(closestByte != 0) { // need to seek 
+      if (closestByte != 0) { // need to seek 
         final long filePointer = in.getFilePointer();
         in.seek(filePointer + closestByte);
       }

Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java Fri Jun 10 13:18:54 2011
@@ -24,9 +24,13 @@ import java.util.Random;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.IndexDocValuesField;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexWriter; // javadoc
+import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.index.values.ValueType;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.Version;
 import org.apache.lucene.util._TestUtil;
@@ -45,6 +49,10 @@ public class RandomIndexWriter implement
   int flushAt;
   private double flushAtFactor = 1.0;
   private boolean getReaderCalled;
+  private final int fixedBytesLength;
+  private final long docValuesFieldPrefix;
+  private volatile boolean doDocValues;
+  private CodecProvider codecProvider;
 
   // Randomly calls Thread.yield so we mixup thread scheduling
   private static final class MockIndexWriter extends IndexWriter {
@@ -92,13 +100,32 @@ public class RandomIndexWriter implement
       System.out.println("codec default=" + w.getConfig().getCodecProvider().getDefaultFieldCodec());
       w.setInfoStream(System.out);
     }
+    /* TODO: find some what to make that random...
+     * This must be fixed across all fixed bytes 
+     * fields in one index. so if you open another writer
+     * this might change if I use r.nextInt(x)
+     * maybe we can peek at the existing files here? 
+     */
+    fixedBytesLength = 37; 
+    docValuesFieldPrefix = r.nextLong();
+    codecProvider =  w.getConfig().getCodecProvider();
+    switchDoDocValues();
   } 
 
+  private void switchDoDocValues() {
+    // randomly enable / disable docValues 
+    doDocValues = r.nextInt(10) != 0;
+  }
+  
   /**
    * Adds a Document.
    * @see IndexWriter#addDocument(Document)
    */
   public void addDocument(final Document doc) throws IOException {
+    if (doDocValues) {
+      randomPerDocFieldValues(r, doc);
+    }
+
     if (r.nextInt(5) == 3) {
       // TODO: maybe, we should simply buffer up added docs
       // (but we need to clone them), and only when
@@ -135,8 +162,53 @@ public class RandomIndexWriter implement
     } else {
       w.addDocument(doc);
     }
+    
     maybeCommit();
   }
+  
+  private void randomPerDocFieldValues(Random random, Document doc) {
+    
+    ValueType[] values = ValueType.values();
+    ValueType type = values[random.nextInt(values.length)];
+    String name = "random_" + type.name() + "" + docValuesFieldPrefix;
+    if ("PreFlex".equals(codecProvider.getFieldCodec(name)) || doc.getFieldable(name) != null)
+        return;
+    IndexDocValuesField docValuesField = new IndexDocValuesField(name);
+    switch (type) {
+    case BYTES_FIXED_DEREF:
+    case BYTES_FIXED_SORTED:
+    case BYTES_FIXED_STRAIGHT:
+      final String randomUnicodeString = _TestUtil.randomUnicodeString(random, fixedBytesLength);
+      BytesRef fixedRef = new BytesRef(randomUnicodeString);
+      if (fixedRef.length > fixedBytesLength) {
+        fixedRef = new BytesRef(fixedRef.bytes, 0, fixedBytesLength);
+      } else {
+        fixedRef.grow(fixedBytesLength);
+        fixedRef.length = fixedBytesLength;
+      }
+      docValuesField.setBytes(fixedRef, type);
+      break;
+    case BYTES_VAR_DEREF:
+    case BYTES_VAR_SORTED:
+    case BYTES_VAR_STRAIGHT:
+      BytesRef ref = new BytesRef(_TestUtil.randomUnicodeString(random, 200));
+      docValuesField.setBytes(ref, type);
+      break;
+    case FLOAT_32:
+      docValuesField.setFloat(random.nextFloat());
+      break;
+    case FLOAT_64:
+      docValuesField.setFloat(random.nextDouble());
+      break;
+    case INTS:
+      docValuesField.setInt(random.nextInt());
+      break;
+    default:
+      throw new IllegalArgumentException("no such type: " + type);
+    }
+
+    doc.add(docValuesField);
+  }
 
   private void maybeCommit() throws IOException {
     if (docCount++ == flushAt) {
@@ -149,6 +221,7 @@ public class RandomIndexWriter implement
         // gradually but exponentially increase time b/w flushes
         flushAtFactor *= 1.05;
       }
+      switchDoDocValues();
     }
   }
   
@@ -166,7 +239,11 @@ public class RandomIndexWriter implement
    * Updates a document.
    * @see IndexWriter#updateDocument(Term, Document)
    */
-  public void updateDocument(Term t, final Document doc) throws IOException {
+  public void updateDocument(final Term t, final Document doc) throws IOException {
+    if (doDocValues) {
+      randomPerDocFieldValues(r, doc);
+    }
+    
     if (r.nextInt(5) == 3) {
       w.updateDocuments(t, new Iterable<Document>() {
 
@@ -212,6 +289,7 @@ public class RandomIndexWriter implement
   
   public void commit() throws CorruptIndexException, IOException {
     w.commit();
+    switchDoDocValues();
   }
   
   public int numDocs() throws IOException {
@@ -241,6 +319,7 @@ public class RandomIndexWriter implement
       w.optimize(limit);
       assert w.getSegmentCount() <= limit: "limit=" + limit + " actual=" + w.getSegmentCount();
     }
+    switchDoDocValues();
   }
 
   public IndexReader getReader(boolean applyDeletions) throws IOException {
@@ -261,6 +340,7 @@ public class RandomIndexWriter implement
         System.out.println("RIW.getReader: open new reader");
       }
       w.commit();
+      switchDoDocValues();
       return IndexReader.open(w.getDirectory(), new KeepOnlyLastCommitDeletionPolicy(), r.nextBoolean(), _TestUtil.nextInt(r, 1, 10), w.getConfig().getCodecProvider());
     }
   }

Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java Fri Jun 10 13:18:54 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs.m
 import java.io.IOException;
 import java.util.Set;
 
+import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SegmentReadState;
@@ -33,8 +34,12 @@ import org.apache.lucene.index.codecs.se
 import org.apache.lucene.index.codecs.sep.SepPostingsWriterImpl;
 import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput;
 import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexOutput;
+import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.PostingsWriterBase;
 import org.apache.lucene.index.codecs.PostingsReaderBase;
 import org.apache.lucene.index.codecs.BlockTermsReader;
@@ -197,10 +202,11 @@ public class MockFixedIntBlockCodec exte
   }
 
   @Override
-  public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set<String> files) throws IOException {
+  public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
     SepPostingsReaderImpl.files(segmentInfo, codecId, files);
     BlockTermsReader.files(dir, segmentInfo, codecId, files);
     FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
+    DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
   }
 
   @Override
@@ -208,5 +214,16 @@ public class MockFixedIntBlockCodec exte
     SepPostingsWriterImpl.getExtensions(extensions);
     BlockTermsReader.getExtensions(extensions);
     FixedGapTermsIndexReader.getIndexExtensions(extensions);
+    DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
+  }
+  
+  @Override
+  public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+    return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+  }
+
+  @Override
+  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
   }
 }

Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java Fri Jun 10 13:18:54 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs.m
 import java.io.IOException;
 import java.util.Set;
 
+import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SegmentReadState;
@@ -33,8 +34,12 @@ import org.apache.lucene.index.codecs.se
 import org.apache.lucene.index.codecs.sep.SepPostingsWriterImpl;
 import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexInput;
 import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexOutput;
+import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.PostingsWriterBase;
 import org.apache.lucene.index.codecs.PostingsReaderBase;
 import org.apache.lucene.index.codecs.BlockTermsReader;
@@ -220,10 +225,11 @@ public class MockVariableIntBlockCodec e
   }
 
   @Override
-  public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set<String> files) throws IOException {
+  public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
     SepPostingsReaderImpl.files(segmentInfo, codecId, files);
     BlockTermsReader.files(dir, segmentInfo, codecId, files);
     FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
+    DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
   }
 
   @Override
@@ -231,5 +237,16 @@ public class MockVariableIntBlockCodec e
     SepPostingsWriterImpl.getExtensions(extensions);
     BlockTermsReader.getExtensions(extensions);
     FixedGapTermsIndexReader.getIndexExtensions(extensions);
+    DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
+  }
+  
+  @Override
+  public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+    return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+  }
+
+  @Override
+  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
   }
 }

Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java Fri Jun 10 13:18:54 2011
@@ -26,16 +26,21 @@ import java.util.Set;
 
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.codecs.BlockTermsReader;
 import org.apache.lucene.index.codecs.BlockTermsWriter;
 import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.PostingsReaderBase;
 import org.apache.lucene.index.codecs.PostingsWriterBase;
 import org.apache.lucene.index.codecs.TermStats;
@@ -341,7 +346,7 @@ public class MockRandomCodec extends Cod
   }
 
   @Override
-  public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set<String> files) throws IOException {
+  public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
     final String seedFileName = IndexFileNames.segmentFileName(segmentInfo.name, codecId, SEED_EXT);    
     files.add(seedFileName);
     SepPostingsReaderImpl.files(segmentInfo, codecId, files);
@@ -349,7 +354,7 @@ public class MockRandomCodec extends Cod
     BlockTermsReader.files(dir, segmentInfo, codecId, files);
     FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
     VariableGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
-    
+    DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
     // hackish!
     Iterator<String> it = files.iterator();
     while(it.hasNext()) {
@@ -367,7 +372,19 @@ public class MockRandomCodec extends Cod
     BlockTermsReader.getExtensions(extensions);
     FixedGapTermsIndexReader.getIndexExtensions(extensions);
     VariableGapTermsIndexReader.getIndexExtensions(extensions);
+    DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
     extensions.add(SEED_EXT);
     //System.out.println("MockRandom.getExtensions return " + extensions);
   }
+  
+  // can we make this more evil?
+  @Override
+  public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+    return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+  }
+
+  @Override
+  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
+  }
 }

Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java Fri Jun 10 13:18:54 2011
@@ -20,14 +20,19 @@ package org.apache.lucene.index.codecs.m
 import java.io.IOException;
 import java.util.Set;
 
+import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.PostingsReaderBase;
 import org.apache.lucene.index.codecs.PostingsWriterBase;
 import org.apache.lucene.index.codecs.BlockTermsReader;
@@ -130,15 +135,17 @@ public class MockSepCodec extends Codec 
   }
 
   @Override
-  public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set<String> files) throws IOException {
+  public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
     SepPostingsReaderImpl.files(segmentInfo, codecId, files);
     BlockTermsReader.files(dir, segmentInfo, codecId, files);
     FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
+    DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     getSepExtensions(extensions);
+    DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
   }
 
   public static void getSepExtensions(Set<String> extensions) {
@@ -146,4 +153,14 @@ public class MockSepCodec extends Codec 
     BlockTermsReader.getExtensions(extensions);
     FixedGapTermsIndexReader.getIndexExtensions(extensions);
   }
+  
+  @Override
+  public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+    return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+  }
+
+  @Override
+  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
+  }
 }

Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java Fri Jun 10 13:18:54 2011
@@ -291,6 +291,48 @@ public class _TestUtil {
       sb.appendCodePoint(nextInt(r, blockStarts[block], blockEnds[block]));
     return sb.toString();
   }
+  
+  /** Returns random string, with a given UTF-8 byte length*/
+  public static String randomFixedByteLengthUnicodeString(Random r, int length) {
+    
+    final char[] buffer = new char[length*3];
+    int bytes = length;
+    int i = 0;
+    for (; i < buffer.length && bytes != 0; i++) {
+      int t;
+      if (bytes >= 4) {
+        t = r.nextInt(5);
+      } else if (bytes >= 3) {
+        t = r.nextInt(4);
+      } else if (bytes >= 2) {
+        t = r.nextInt(2);
+      } else {
+        t = 0;
+      }
+      if (t == 0) {
+        buffer[i] = (char) r.nextInt(0x80);
+        bytes--;
+      } else if (1 == t) {
+        buffer[i] = (char) nextInt(r, 0x80, 0x7ff);
+        bytes -= 2;
+      } else if (2 == t) {
+        buffer[i] = (char) nextInt(r, 0x800, 0xd7ff);
+        bytes -= 3;
+      } else if (3 == t) {
+        buffer[i] = (char) nextInt(r, 0xe000, 0xffff);
+        bytes -= 3;
+      } else if (4 == t) {
+        // Make a surrogate pair
+        // High surrogate
+        buffer[i++] = (char) nextInt(r, 0xd800, 0xdbff);
+        // Low surrogate
+        buffer[i] = (char) nextInt(r, 0xdc00, 0xdfff);
+        bytes -= 4;
+      }
+
+    }
+    return new String(buffer, 0, i);
+  }
 
   public static CodecProvider alwaysCodec(final Codec c) {
     CodecProvider p = new CodecProvider() {
@@ -375,7 +417,7 @@ public class _TestUtil {
     List<Fieldable> fields = doc.getFields();
     for (Fieldable field : fields) {
       fieldInfos.addOrUpdate(field.name(), field.isIndexed(), field.isTermVectorStored(), field.isStorePositionWithTermVector(),
-              field.isStoreOffsetWithTermVector(), field.getOmitNorms(), false, field.getOmitTermFreqAndPositions());
+              field.isStoreOffsetWithTermVector(), field.getOmitNorms(), false, field.getOmitTermFreqAndPositions(), field.docValuesType());
     }
   }
   

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/TestExternalCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/TestExternalCodecs.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/TestExternalCodecs.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/TestExternalCodecs.java Fri Jun 10 13:18:54 2011
@@ -490,11 +490,21 @@ public class TestExternalCodecs extends 
     }
 
     @Override
+    public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+      return null;
+    }
+
+    @Override
+    public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+      return null;
+    }
+
+    @Override
     public void getExtensions(Set<String> extensions) {
     }
 
     @Override
-    public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set<String> files) {
+    public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) {
     }
   }
 

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java Fri Jun 10 13:18:54 2011
@@ -33,10 +33,14 @@ import org.apache.lucene.index.codecs.Bl
 import org.apache.lucene.index.codecs.BlockTermsWriter;
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.CoreCodecProvider;
+import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.PostingsReaderBase;
 import org.apache.lucene.index.codecs.PostingsWriterBase;
 import org.apache.lucene.index.codecs.TermsIndexReaderBase;
@@ -192,15 +196,17 @@ public class TestDocTermOrds extends Luc
     static final String PROX_EXTENSION = "prx";
 
     @Override
-    public void files(Directory dir, SegmentInfo segmentInfo, String id, Set<String> files) throws IOException {
+    public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
       StandardPostingsReader.files(dir, segmentInfo, id, files);
       BlockTermsReader.files(dir, segmentInfo, id, files);
       FixedGapTermsIndexReader.files(dir, segmentInfo, id, files);
+      DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
     }
 
     @Override
     public void getExtensions(Set<String> extensions) {
       getStandardExtensions(extensions);
+      DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
     }
 
     public static void getStandardExtensions(Set<String> extensions) {
@@ -209,6 +215,16 @@ public class TestDocTermOrds extends Luc
       BlockTermsReader.getExtensions(extensions);
       FixedGapTermsIndexReader.getIndexExtensions(extensions);
     }
+    
+    @Override
+    public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+      return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+    }
+
+    @Override
+    public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+      return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
+    }
   }
 
   public void testRandom() throws Exception {

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java?rev=1134311&r1=1134310&r2=1134311&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java Fri Jun 10 13:18:54 2011
@@ -137,7 +137,7 @@ public class TestFieldInfos extends Luce
     try {
       readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(),
           random.nextBoolean(), random.nextBoolean(), random.nextBoolean(),
-          random.nextBoolean(), random.nextBoolean());
+          random.nextBoolean(), random.nextBoolean(), null);
       fail("instance should be read only");
     } catch (IllegalStateException e) {
       // expected