You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2010/10/29 22:33:37 UTC

svn commit: r1028899 - in /lucene/dev/branches/docvalues/lucene/src: java/org/apache/lucene/index/ java/org/apache/lucene/index/codecs/ java/org/apache/lucene/index/codecs/docvalues/ java/org/apache/lucene/index/values/ java/org/apache/lucene/search/ j...

Author: simonw
Date: Fri Oct 29 20:33:36 2010
New Revision: 1028899

URL: http://svn.apache.org/viewvc?rev=1028899&view=rev
Log:
LUCENE-2700: Added support for MultiFields, MultiFieldsEnum and moved actual merge code to DocValuesCodec

Removed:
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Cache.java
Modified:
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfos.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFields.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentReader.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesConsumer.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesProducerBase.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Values.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/SortField.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ReaderUtil.java
    lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestIndexValues.java

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Fri Oct 29 20:33:36 2010
@@ -49,7 +49,7 @@ final class DocFieldProcessor extends Do
       throws IOException {
     DocValuesConsumer valuesConsumer;
     if ((valuesConsumer = docValues.get(name)) == null) {
-      fieldInfo.setIndexValues(attr.type());
+      fieldInfo.setDocValues(attr.type());
 
       if(fieldsConsumer == null) {
         /* nocommit -- this is a hack and only works since DocValuesCodec supports initializing the FieldsConsumer twice.

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java Fri Oct 29 20:33:36 2010
@@ -24,7 +24,7 @@ public final class FieldInfo {
   public String name;
   public boolean isIndexed;
   public int number;
-  Values indexValues;
+  Values docValues;
 
 
   // true if term vector for this field should be stored
@@ -93,17 +93,21 @@ public final class FieldInfo {
     }
   }
 
-  void setIndexValues(Values v) {
-    if (indexValues != null) {
-      if (indexValues != v) {
-        throw new IllegalArgumentException("indexValues is already set to " + indexValues + "; cannot change to " + v);
+  void setDocValues(Values v) {
+    if (docValues != null) {
+      if (docValues != v) {
+        throw new IllegalArgumentException("indexValues is already set to " + docValues + "; cannot change to " + v);
       }
     } else{
-	   indexValues = v;
+	   docValues = v;
     }
   }
+  
+  public boolean hasDocValues() {
+    return docValues != null;
+  }
 
-  public Values getIndexValues() {
-    return indexValues;
+  public Values getDocValues() {
+    return docValues;
   }
 }

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfos.java Fri Oct 29 20:33:36 2010
@@ -311,10 +311,10 @@ public final class FieldInfos {
 
       final byte b;
 
-      if (fi.indexValues == null) {
+      if (fi.docValues == null) {
         b = 0;
       } else {
-        switch(fi.indexValues) {
+        switch(fi.docValues) {
         case PACKED_INTS:
           b = 1;
           break;
@@ -346,7 +346,7 @@ public final class FieldInfos {
           b = 10;
           break;
         default:
-          throw new IllegalStateException("unhandled indexValues type " + fi.indexValues);
+          throw new IllegalStateException("unhandled indexValues type " + fi.docValues);
         }
       }
       output.writeByte(b);
@@ -377,43 +377,41 @@ public final class FieldInfos {
       boolean omitTermFreqAndPositions = (bits & OMIT_TERM_FREQ_AND_POSITIONS) != 0;
       
       FieldInfo fi = addInternal(name, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions);
-
       if (format <= FORMAT_INDEX_VALUES) {
         final byte b = input.readByte();
-
         switch(b) {
         case 0:
-          fi.indexValues = null;
+          fi.docValues = null;
           break;
         case 1:
-          fi.indexValues = Values.PACKED_INTS;
+          fi.docValues = Values.PACKED_INTS;
           break;
         case 2:
-          fi.indexValues = Values.SIMPLE_FLOAT_4BYTE;
+          fi.docValues = Values.SIMPLE_FLOAT_4BYTE;
           break;
         case 3:
-          fi.indexValues = Values.SIMPLE_FLOAT_8BYTE;
+          fi.docValues = Values.SIMPLE_FLOAT_8BYTE;
           break;
         case 4:
-          fi.indexValues = Values.BYTES_FIXED_STRAIGHT;
+          fi.docValues = Values.BYTES_FIXED_STRAIGHT;
           break;
         case 5:
-          fi.indexValues = Values.BYTES_FIXED_DEREF;
+          fi.docValues = Values.BYTES_FIXED_DEREF;
           break;
         case 6:
-          fi.indexValues = Values.BYTES_FIXED_SORTED;
+          fi.docValues = Values.BYTES_FIXED_SORTED;
           break;
         case 7:
-          fi.indexValues = Values.BYTES_VAR_STRAIGHT;
+          fi.docValues = Values.BYTES_VAR_STRAIGHT;
           break;
         case 8:
-          fi.indexValues = Values.BYTES_VAR_DEREF;
+          fi.docValues = Values.BYTES_VAR_DEREF;
           break;
         case 9:
-          fi.indexValues = Values.BYTES_VAR_SORTED;
+          fi.docValues = Values.BYTES_VAR_SORTED;
           break;
         case 10:
-          fi.indexValues = Values.PACKED_INTS_FIXED;
+          fi.docValues = Values.PACKED_INTS_FIXED;
           break;
         default:
           throw new IllegalStateException("unhandled indexValues type " + b);

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java Fri Oct 29 20:33:36 2010
@@ -21,7 +21,6 @@ import org.apache.lucene.document.Docume
 import org.apache.lucene.document.FieldSelector;
 import org.apache.lucene.search.Similarity;
 import org.apache.lucene.index.codecs.CodecProvider;
-import org.apache.lucene.index.values.Cache;
 import org.apache.lucene.index.values.DocValues;
 import org.apache.lucene.store.*;
 import org.apache.lucene.util.Bits;
@@ -1090,7 +1089,7 @@ public abstract class IndexReader implem
     if (docs == null) return 0;
     int n = 0;
     int doc;
-    while ((doc = docs.nextDoc()) != docs.NO_MORE_DOCS) {
+    while ((doc = docs.nextDoc()) != DocsEnum.NO_MORE_DOCS) {
       deleteDocument(doc);
       n++;
     }
@@ -1386,13 +1385,6 @@ public abstract class IndexReader implem
     return fields.docValues(field);
   }
 
-  private final Cache indexValuesCache = new Cache(this);
-
-  // nocommit -- don't expose readers if we have this?
-  public Cache getIndexValuesCache() {
-    return indexValuesCache;
-  }
-
   private Fields fields;
 
   /** @lucene.internal */

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFields.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFields.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFields.java Fri Oct 29 20:33:36 2010
@@ -25,6 +25,7 @@ import java.util.ArrayList;
 
 import org.apache.lucene.index.values.DocValues;
 import org.apache.lucene.index.values.MultiDocValues;
+import org.apache.lucene.index.values.Values;
 import org.apache.lucene.index.values.MultiDocValues.DocValuesIndex;
 import org.apache.lucene.util.ReaderUtil;
 import org.apache.lucene.util.ReaderUtil.Gather;  // for javadocs
@@ -290,19 +291,30 @@ public final class MultiFields extends F
 
       // Lazy init: first time this field is requested, we
       // create & add to docValues:
-      final List<MultiDocValues.DocValuesIndex> subs2 = new ArrayList<MultiDocValues.DocValuesIndex>();
-      final List<ReaderUtil.Slice> slices2 = new ArrayList<ReaderUtil.Slice>();
-
+      final List<MultiDocValues.DocValuesIndex> docValuesIndex = new ArrayList<MultiDocValues.DocValuesIndex>();
+      int docsUpto = 0;
+      Values type = null;
       // Gather all sub-readers that share this field
       for(int i=0;i<subs.length;i++) {
-        final DocValues values = subs[i].docValues(field);
+         DocValues values = subs[i].docValues(field);
+         final int start = subSlices[i].start;
+         final int length = subSlices[i].length;
         if (values != null) {
-          subs2.add(new MultiDocValues.DocValuesIndex(values, i));
-          slices2.add(subSlices[i]);
+          if (docsUpto != start) {
+            type = values.type();
+            docValuesIndex.add(new MultiDocValues.DocValuesIndex(new MultiDocValues.DummyDocValues(start, type), docsUpto, start - docsUpto));
+          }
+          docValuesIndex.add(new MultiDocValues.DocValuesIndex(values, start, length));
+          docsUpto = start+length;
+
+        } else if (i+1 == subs.length && !docValuesIndex.isEmpty()) {
+          docValuesIndex.add(new MultiDocValues.DocValuesIndex(
+              new MultiDocValues.DummyDocValues(start, type), docsUpto, start
+                  - docsUpto));
         }
+       
       }
-      result = subs2.isEmpty()?null: new MultiDocValues(subs2.toArray(DocValuesIndex.EMPTY_ARRAY),
-                                slices2.toArray(ReaderUtil.Slice.EMPTY_ARRAY));
+      result = docValuesIndex.isEmpty()?null: new MultiDocValues(docValuesIndex.toArray(DocValuesIndex.EMPTY_ARRAY));
       docValues.put(field, result);
     } else {
       result = docValues.get(field);

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java Fri Oct 29 20:33:36 2010
@@ -19,8 +19,10 @@ package org.apache.lucene.index;
 
 import org.apache.lucene.index.values.DocValues;
 import org.apache.lucene.index.values.MultiDocValues;
+import org.apache.lucene.index.values.Values;
 import org.apache.lucene.util.PriorityQueue;
 import org.apache.lucene.util.ReaderUtil;
+import org.apache.lucene.util.ReaderUtil.Slice;
 
 import java.io.IOException;
 import java.util.List;
@@ -40,6 +42,8 @@ public final  class MultiFieldsEnum exte
   // Holds sub-readers containing field we are currently
   // on, popped from queue.
   private final FieldsEnumWithSlice[] top;
+  private final FieldsEnumWithSlice[] enumWithSlices;
+
   private int numTop;
 
   // Re-used TermsEnum
@@ -54,8 +58,9 @@ public final  class MultiFieldsEnum exte
   public MultiFieldsEnum(FieldsEnum[] subs, ReaderUtil.Slice[] subSlices) throws IOException {
     terms = new MultiTermsEnum(subSlices);
     queue = new FieldMergeQueue(subs.length);
-    docValues = new MultiDocValues(subSlices);
+    docValues = new MultiDocValues();
     top = new FieldsEnumWithSlice[subs.length];
+    List<FieldsEnumWithSlice> enumWithSlices = new ArrayList<FieldsEnumWithSlice>();
 
     // Init q
     for(int i=0;i<subs.length;i++) {
@@ -64,10 +69,13 @@ public final  class MultiFieldsEnum exte
       if (field != null) {
         // this FieldsEnum has at least one field
         final FieldsEnumWithSlice sub = new FieldsEnumWithSlice(subs[i], subSlices[i], i);
+        enumWithSlices.add(sub);
         sub.current = field;
         queue.add(sub);
       }
     }
+    this.enumWithSlices = enumWithSlices.toArray(FieldsEnumWithSlice.EMPTY_ARRAY);
+
   }
 
   @Override
@@ -119,6 +127,7 @@ public final  class MultiFieldsEnum exte
   }
 
   public final static class FieldsEnumWithSlice {
+    public static final FieldsEnumWithSlice[] EMPTY_ARRAY = new FieldsEnumWithSlice[0];
     final FieldsEnum fields;
     final ReaderUtil.Slice slice;
     final int index;
@@ -146,16 +155,37 @@ public final  class MultiFieldsEnum exte
 
   @Override
   public DocValues docValues() throws IOException {
-    final List<MultiDocValues.DocValuesIndex> values = new ArrayList<MultiDocValues.DocValuesIndex>();
-    for (int i = 0; i < numTop; i++) {
-      final DocValues docValues = top[i].fields.docValues();
-      if (docValues != null) {
-        values.add(new MultiDocValues.DocValuesIndex(docValues,
-            top[i].index));
+    final List<MultiDocValues.DocValuesIndex> docValuesIndex = new ArrayList<MultiDocValues.DocValuesIndex>();
+    int docsUpto = 0;
+    Values type = null;
+    final int numEnums = enumWithSlices.length;
+    for (int i = 0; i < numEnums; i++) {
+      FieldsEnumWithSlice withSlice = enumWithSlices[i];
+      Slice slice = withSlice.slice;
+      final DocValues values = withSlice.fields.docValues();
+
+      final int start = slice.start;
+      final int length = slice.length;
+      if (values != null) {
+        if (docsUpto != start) {
+          type = values.type();
+          docValuesIndex.add(new MultiDocValues.DocValuesIndex(
+              new MultiDocValues.DummyDocValues(start, type), docsUpto, start
+                  - docsUpto));
+        }
+        docValuesIndex.add(new MultiDocValues.DocValuesIndex(values, start,
+            length));
+        docsUpto = start + length;
+       
+
+      } else if (i+1 == numEnums && !docValuesIndex.isEmpty()) {
+        docValuesIndex.add(new MultiDocValues.DocValuesIndex(
+            new MultiDocValues.DummyDocValues(start, type), docsUpto, start
+                - docsUpto));
       }
     }
-    // TODO return an empty docvalues instance if values are empty
-    return docValues.reset(values.toArray(MultiDocValues.DocValuesIndex.EMPTY_ARRAY));
+    return docValuesIndex.isEmpty() ? null : docValues.reset(docValuesIndex
+        .toArray(MultiDocValues.DocValuesIndex.EMPTY_ARRAY));
   }
 }
 

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Fri Oct 29 20:33:36 2010
@@ -31,12 +31,7 @@ import org.apache.lucene.index.codecs.Co
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.MergeState;
 import org.apache.lucene.index.codecs.FieldsConsumer;
-import org.apache.lucene.index.values.Bytes;
-import org.apache.lucene.index.values.Ints;
-import org.apache.lucene.index.values.DocValues;
-import org.apache.lucene.index.values.Floats;
 import org.apache.lucene.index.values.Values;
-import org.apache.lucene.index.values.Writer;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
@@ -296,13 +291,13 @@ final class SegmentMerger {
                                             fi.storePositionWithTermVector, fi.storeOffsetWithTermVector,
                                             !reader.hasNorms(fi.name), fi.storePayloads,
                                             fi.omitTermFreqAndPositions);
-          final Values fiIndexValues = fi.indexValues;
-          final Values mergedIndexValues = merged.indexValues;
-          if (mergedIndexValues == null) {
-            merged.setIndexValues(fiIndexValues);
-          } else if (mergedIndexValues != fiIndexValues) {
+          final Values fiIndexValues = fi.docValues;
+          final Values mergedDocValues = merged.docValues;
+          if (mergedDocValues == null) {
+            merged.setDocValues(fiIndexValues);
+          } else if (mergedDocValues != fiIndexValues) {
             // TODO -- can we recover from this?
-            throw new IllegalStateException("cannot merge field " + fi.name + " indexValues changed from " + mergedIndexValues + " to " + fiIndexValues);
+            throw new IllegalStateException("cannot merge field " + fi.name + " indexValues changed from " + mergedDocValues + " to " + fiIndexValues);
           }
         }
       } else {

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentReader.java Fri Oct 29 20:33:36 2010
@@ -968,7 +968,7 @@ public class SegmentReader extends Index
                 fieldOption == IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET) {
         fieldSet.add(fi.name);
       }
-      else if (fi.indexValues != null && fieldOption == IndexReader.FieldOption.DOC_VALUES) {
+      else if (fi.docValues != null && fieldOption == IndexReader.FieldOption.DOC_VALUES) {
         fieldSet.add(fi.name);
       }
     }

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java Fri Oct 29 20:33:36 2010
@@ -65,32 +65,13 @@ public abstract class FieldsConsumer imp
         final TermsConsumer termsConsumer = addField(mergeState.fieldInfo);
         termsConsumer.merge(mergeState, terms);
       }
-      
-      DocValues docValues = fieldsEnum.docValues();   // fix this - does not work due to multi fields
-      if(docValues != null) {
-      // TODO we need some kind of compatibility notation for values such
-      // that two slighly different segments can be merged eg. fixed vs.
-      // variable byte len or float32 vs. float64
-        int docBase = 0;
-        final List<Writer.MergeState> mergeStates = new ArrayList<Writer.MergeState>();
-        for (IndexReader reader : mergeState.readers) {
-          DocValues r = reader.docValues(mergeState.fieldInfo.name);
-          if (r != null) {
-            mergeStates.add(new Writer.MergeState(r, docBase, reader
-                .maxDoc(), reader.getDeletedDocs()));
-          }
-          docBase += reader.numDocs();
-        }
-        if (mergeStates.isEmpty()) {
-          continue;
-        }
+      if (mergeState.fieldInfo.hasDocValues()) {
+        final DocValues docValues = fieldsEnum.docValues();
+        assert docValues != null : "DocValues are null for " + mergeState.fieldInfo.getDocValues();
         final DocValuesConsumer docValuesConsumer = addValuesField(mergeState.fieldInfo);
-        docValuesConsumer.merge(mergeStates);
-        docValuesConsumer.finish(mergeState.mergedDocCount);
+        assert docValuesConsumer != null;
+        docValuesConsumer.merge(mergeState, docValues);
       }
-      
-      // merge doc values
-//   
     }
   }
  

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesConsumer.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesConsumer.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesConsumer.java Fri Oct 29 20:33:36 2010
@@ -1,4 +1,5 @@
 package org.apache.lucene.index.codecs.docvalues;
+
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -18,9 +19,9 @@ package org.apache.lucene.index.codecs.d
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Comparator;
-import java.util.List;
 
 import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.values.DocValues;
 import org.apache.lucene.index.values.ValuesAttribute;
 import org.apache.lucene.index.values.Writer;
@@ -28,22 +29,50 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 
+/**
+ * @lucene.experimental
+ */
+// TODO this might need to go in the codec package since is a direct relative to
+// TermsConsumer
 public abstract class DocValuesConsumer {
+
   public abstract void add(int docID, ValuesAttribute attr) throws IOException;
 
   public abstract void finish(int docCount) throws IOException;
 
   public abstract void files(Collection<String> files) throws IOException;
-  
-  public void merge(List<MergeState> states) throws IOException {
-    for (MergeState state : states) {
-      merge(state);
+
+  public void merge(org.apache.lucene.index.codecs.MergeState mergeState,
+      DocValues values) throws IOException {
+    // TODO we need some kind of compatibility notation for values such
+    // that two slightly different segments can be merged eg. fixed vs.
+    // variable byte len or float32 vs. float64
+    int docBase = 0;
+    boolean merged = false;
+    /*
+     * We ignore the given DocValues here and merge from the subReaders directly
+     * to support bulk copies on the DocValues Writer level. if this gets merged
+     * with MultiDocValues the writer can not optimize for bulk-copyable data
+     */
+    for (final IndexReader reader : mergeState.readers) {
+      final DocValues r = reader.docValues(mergeState.fieldInfo.name);
+      if (r != null) {
+        merged = true;
+        merge(new Writer.MergeState(r, docBase, reader.maxDoc(), reader
+            .getDeletedDocs()));
+      }
+      docBase += reader.numDocs();
     }
+    if (merged)
+      finish(mergeState.mergedDocCount);
   }
-  
+
   protected abstract void merge(MergeState mergeState) throws IOException;
-  
-  
+
+  /*
+   * specialized auxiliary MergeState is necessary since we don't want to
+   * exploit internals up to the codec ones
+   */
   public static class MergeState {
     public final DocValues reader;
     public final int docBase;
@@ -59,9 +88,10 @@ public abstract class DocValuesConsumer 
     }
   }
 
-  public static DocValuesConsumer create(String segmentName, Directory directory,
-      FieldInfo field, Comparator<BytesRef> comp) throws IOException {
+  public static DocValuesConsumer create(String segmentName,
+      Directory directory, FieldInfo field, Comparator<BytesRef> comp)
+      throws IOException {
     final String id = segmentName + "_" + field.number;
-    return Writer.create(field.getIndexValues(), id, directory, comp);
+    return Writer.create(field.getDocValues(), id, directory, comp);
   }
 }

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesProducerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesProducerBase.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesProducerBase.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesProducerBase.java Fri Oct 29 20:33:36 2010
@@ -51,13 +51,13 @@ public abstract class DocValuesProducerB
     final int numFields = fieldInfos.size();
     for (int i = 0; i < numFields; i++) {
       final FieldInfo fieldInfo = fieldInfos.fieldInfo(i);
-      final Values v = fieldInfo.getIndexValues();
+      final Values v = fieldInfo.getDocValues();
       final String field = fieldInfo.name;
       final String id = IndexFileNames.segmentFileName(segment, Integer
-          .toString(fieldInfo.number), "");
-      if (v != null && dir.fileExists(id + "." + Writer.DATA_EXTENSION)) {
+          .toString(fieldInfo.number),"");
+      if (v != null && dir.fileExists(id + "." +  Writer.DATA_EXTENSION)) {
         docValues.put(field, loadDocValues(docCount, dir, id, v));
-      }
+      } 
     }
   }
 

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java Fri Oct 29 20:33:36 2010
@@ -287,7 +287,9 @@ public final class Bytes {
       return idxIn == null ? null : (IndexInput) idxIn.clone();
     }
 
+    @Override
     public void close() throws IOException {
+      super.close();
       if (datIn != null) {
         datIn.close();
       }

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java Fri Oct 29 20:33:36 2010
@@ -24,24 +24,48 @@ import org.apache.lucene.util.AttributeS
 import org.apache.lucene.util.BytesRef;
 
 public abstract class DocValues implements Closeable {
-  
-  
+
+  private final Object lock = new Object();
+
+  private Source cachedReference;
+
   public static final DocValues[] EMPTY_ARRAY = new DocValues[0];
 
-  public ValuesEnum getEnum() throws IOException{
+  public ValuesEnum getEnum() throws IOException {
     return getEnum(null);
   }
 
-  public abstract ValuesEnum getEnum(AttributeSource attrSource) throws IOException;
+  public abstract ValuesEnum getEnum(AttributeSource attrSource)
+      throws IOException;
 
   public abstract Source load() throws IOException;
 
-  public SortedSource loadSorted(Comparator<BytesRef> comparator) throws IOException {
+  public Source getCached(boolean load) throws IOException {
+    synchronized (lock) { // TODO make sorted source cachable too 
+      if (load && cachedReference == null)
+        cachedReference = load();
+      return cachedReference;
+    }
+  }
+
+  public Source releaseCached() {
+    synchronized (lock) {
+      final Source retVal = cachedReference;
+      cachedReference = null;
+      return retVal;
+    }
+  }
+
+  public SortedSource loadSorted(Comparator<BytesRef> comparator)
+      throws IOException {
     throw new UnsupportedOperationException();
   }
-  
+
   public abstract Values type();
   
+  public void close() throws IOException {
+    releaseCached();
+  }
 
   /**
    * Source of integer (returned as java long), per document. The underlying
@@ -50,30 +74,34 @@ public abstract class DocValues implemen
    */
   public static abstract class Source {
 
-    public long ints(int docID) {
+    public long getInt(int docID) {
       throw new UnsupportedOperationException("ints are not supported");
     }
 
-    public double floats(int docID) {
+    public double getFloat(int docID) {
       throw new UnsupportedOperationException("floats are not supported");
     }
 
-    public BytesRef bytes(int docID) {
+    public BytesRef getBytes(int docID) {
       throw new UnsupportedOperationException("bytes are not supported");
     }
-    
-    /** Returns number of unique values.  Some impls may
-     * throw UnsupportedOperationException. */
+
+    /**
+     * Returns number of unique values. Some impls may throw
+     * UnsupportedOperationException.
+     */
     public int getValueCount() {
       throw new UnsupportedOperationException();
     }
-    
-    public ValuesEnum getEnum() throws IOException{
+
+    public ValuesEnum getEnum() throws IOException {
       return getEnum(null);
     }
-    
-    // nocommit - enable obtaining enum from source since this is already in memory
-    public /*abstract*/ ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
+
+    // nocommit - enable obtaining enum from source since this is already in
+    // memory
+    public/* abstract */ValuesEnum getEnum(AttributeSource attrSource)
+        throws IOException {
       throw new UnsupportedOperationException();
     }
 
@@ -83,7 +111,7 @@ public abstract class DocValues implemen
   public static abstract class SortedSource extends Source {
 
     @Override
-    public BytesRef bytes(int docID) {
+    public BytesRef getBytes(int docID) {
       return getByOrd(ord(docID));
     }
 
@@ -109,5 +137,5 @@ public abstract class DocValues implemen
      */
     public abstract LookupResult getByValue(BytesRef value);
   }
-  
+
 }

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java Fri Oct 29 20:33:36 2010
@@ -157,7 +157,7 @@ class FixedDerefBytesImpl {
       }
 
       @Override
-      public BytesRef bytes(int docID) {
+      public BytesRef getBytes(int docID) {
         final int id = (int) index.get(docID);
         if (id == 0) {
           return defaultValue;

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java Fri Oct 29 20:33:36 2010
@@ -150,7 +150,7 @@ class FixedStraightBytesImpl {
       }
 
       @Override
-      public BytesRef bytes(int docID) {
+      public BytesRef getBytes(int docID) {
         bytesRef.offset = docID * size;
         return bytesRef;
       }

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java Fri Oct 29 20:33:36 2010
@@ -270,7 +270,7 @@ public class Floats {
       }
 
       @Override
-      public double floats(int docID) {
+      public double getFloat(int docID) {
         final float f = values.get(docID);
         // nocommit should we return NaN as default instead of 0.0?
         return Float.isNaN(f) ? 0.0f : f;
@@ -290,9 +290,9 @@ public class Floats {
       }
 
       @Override
-      public double floats(int docID) {
+      public double getFloat(int docID) {
         final double d = values.get(docID);
-        // nocommit should we return NaN as default instead of 0.0?
+        // TODO should we return NaN as default instead of 0.0?
         return Double.isNaN(d) ? 0.0d : d;
       }
 
@@ -302,7 +302,9 @@ public class Floats {
       }
     }
 
+    @Override
     public void close() throws IOException {
+      super.close();
       datIn.close();
     }
 

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java Fri Oct 29 20:33:36 2010
@@ -1,4 +1,5 @@
 package org.apache.lucene.index.values;
+
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -16,196 +17,214 @@ package org.apache.lucene.index.values;
  * limitations under the License.
  */
 import java.io.IOException;
-import java.util.List;
+import java.util.Arrays;
 
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.MultiTermsEnum.TermsEnumIndex;
+import org.apache.lucene.index.values.DocValues.Source;
 import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.FloatsRef;
 import org.apache.lucene.util.LongsRef;
-import org.apache.lucene.util.ReaderUtil.Slice;
+import org.apache.lucene.util.ReaderUtil;
 
 public class MultiDocValues extends DocValues {
 
-  public static class DocValuesIndex {
+  public static class DocValuesIndex { // nocommit is this necessary?
     public final static DocValuesIndex[] EMPTY_ARRAY = new DocValuesIndex[0];
-    final int subIndex;
+    final int start;
+    final int length;
     final DocValues docValues;
 
-    public DocValuesIndex(DocValues docValues, int subIndex) {
+    public DocValuesIndex(DocValues docValues, int start, int length) {
       this.docValues = docValues;
-      this.subIndex = subIndex;
+      this.start = start;
+      this.length = length;
     }
   }
 
   private DocValuesIndex[] docValuesIdx;
-  private Slice[] subSlices;
+  private int[] starts;
 
-  public MultiDocValues(Slice[] subSlices) {
-    this.subSlices = subSlices;
+  public MultiDocValues() {
+    starts = new int[0];
+    docValuesIdx = new DocValuesIndex[0];
   }
 
-  public MultiDocValues(DocValuesIndex[] docValuesIdx, Slice[] subSlices) {
-    this(subSlices);
+  public MultiDocValues(DocValuesIndex[] docValuesIdx) {
     reset(docValuesIdx);
   }
 
   @Override
   public ValuesEnum getEnum(AttributeSource source) throws IOException {
-    return new MultiValuesEnum(subSlices, docValuesIdx, docValuesIdx[0].docValues.type());
+    return new MultiValuesEnum(docValuesIdx, starts);
   }
 
   @Override
   public Source load() throws IOException {
-    return new MultiSource(subSlices, docValuesIdx);
+    return new MultiSource(docValuesIdx, starts);
   }
 
   public void close() throws IOException {
-    //      
+    super.close();
   }
 
   public DocValues reset(DocValuesIndex[] docValuesIdx) {
+    int[] start = new int[docValuesIdx.length];
+    for (int i = 0; i < docValuesIdx.length; i++) {
+      start[i] = docValuesIdx[i].start;
+    }
+    this.starts = start;
     this.docValuesIdx = docValuesIdx;
     return this;
   }
 
+  public static class DummyDocValues extends DocValues {
+    final int maxDoc;
+    final Values type;
+    static final Source DUMMY = new DummySource();
+
+    public DummyDocValues(int maxDoc, Values type) {
+      this.type = type;
+      this.maxDoc = maxDoc;
+    }
+
+    @Override
+    public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
+      return new DummyEnum(attrSource, maxDoc, type);
+    }
+
+    @Override
+    public Source load() throws IOException {
+      return DUMMY;
+    }
+
+    @Override
+    public Source getCached(boolean load) throws IOException {
+      return DUMMY;
+    }
+
+    @Override
+    public Source releaseCached() {
+      return DUMMY;
+    }
+
+    @Override
+    public Values type() {
+      return type;
+    }
+
+    public void close() throws IOException {
+      super.close();
+    }
+
+  }
+
   private static class MultiValuesEnum extends ValuesEnum {
-    private int numDocs_ = 0;
-    private int pos = -1;
-    private int start = 0;
-    private ValuesEnum current;
-    private Slice[] subSlices;
     private DocValuesIndex[] docValuesIdx;
     private final int maxDoc;
-
-    public MultiValuesEnum(Slice[] subSlices, DocValuesIndex[] docValuesIdx, Values type) {
-      super(type);
-      this.subSlices = subSlices;
+    private int currentStart;
+    private int currentMax;
+    private int currentDoc = -1;
+    private ValuesEnum currentEnum;
+    private final int[] starts;
+
+    public MultiValuesEnum(DocValuesIndex[] docValuesIdx, int[] starts)
+        throws IOException {
+      super(docValuesIdx[0].docValues.type());
       this.docValuesIdx = docValuesIdx;
-      Slice slice = subSlices[subSlices.length-1];
-      maxDoc = slice.start + slice.length;
+      final DocValuesIndex last = docValuesIdx[docValuesIdx.length - 1];
+      maxDoc = last.start + last.length;
+      final DocValuesIndex idx = docValuesIdx[0];
+      currentEnum = idx.docValues.getEnum(this.attributes());
+      currentMax = idx.length;
+      currentStart = 0;
+      this.starts = starts;
     }
 
     @Override
     public void close() throws IOException {
-      
+      currentEnum.close();
     }
 
     @Override
     public int advance(int target) throws IOException {
-//      int n = target - start;
-//      do {
-//        if (target >= maxDoc)
-//          return pos = NO_MORE_DOCS;
-//        if (n >= numDocs_) {
-//          int idx = readerIndex(target);
-//          if (enumCache[idx] == null) {
-//            try {
-//              DocValues indexValues = subReaders[idx].docValues(id);
-//              if (indexValues != null) // nocommit does that work with default
-//                // values?
-//                enumCache[idx] = indexValues.getEnum(this.attributes());
-//              else
-//                enumCache[idx] = new DummyEnum(this.attributes(),
-//                    subSlices[idx].length, attr.type());
-//            } catch (IOException ex) {
-//              // nocommit what to do here?
-//              throw new RuntimeException(ex);
-//            }
-//          }
-//          current = enumCache[idx];
-//          start = subSlices[idx].start;
-//          numDocs_ = subSlices[idx].length;
-//          n = target - start;
-//        }
-//        target = start + numDocs_;
-//      } while ((n = current.advance(n)) == NO_MORE_DOCS);
-      return pos = start + current.docID();
+      assert target > currentDoc : "target " + target
+          + " must be > than the current doc " + currentDoc;
+      int relativeDoc = target - currentStart;
+      do {
+        if (target >= maxDoc) // we are beyond max doc
+          return currentDoc = NO_MORE_DOCS;
+        if (target >= currentMax) {
+          final int idx = ReaderUtil.subIndex(target, starts);
+          currentEnum.close();
+          currentEnum = docValuesIdx[idx].docValues.getEnum(this.attributes());
+          currentStart = docValuesIdx[idx].start;
+          currentMax = currentStart + docValuesIdx[idx].length;
+          relativeDoc = target - currentStart;
+        } else {
+          return currentDoc = currentStart + currentEnum.advance(relativeDoc);
+        }
+      } while ((relativeDoc = currentEnum.advance(relativeDoc)) == NO_MORE_DOCS);
+      return currentDoc = currentStart + relativeDoc;
     }
 
     @Override
     public int docID() {
-      return pos;
+      return currentDoc;
     }
 
     @Override
     public int nextDoc() throws IOException {
-      return advance(pos + 1);
+      return advance(currentDoc + 1);
     }
   }
 
-  private class MultiSource extends Source {
-    private int numDocs_ = 0;
+  private static class MultiSource extends Source {
+    private int numDocs = 0;
     private int start = 0;
     private Source current;
-    private Slice[] subSlices;
-    private DocValuesIndex[] docVAluesIdx;
+    private final int[] starts;
+    private final DocValuesIndex[] docValuesIdx;
 
-    public MultiSource(Slice[] subSlices, DocValuesIndex[] docValuesIdx) {
-      this.subSlices = subSlices;
-      this.docVAluesIdx = docValuesIdx;
-    }
-
-    public long ints(int docID) {
-//      int n = docID - start;
-//      if (n >= numDocs_) {
-//        int idx = readerIndex(docID);
-//        try {
-//          current = subReaders[idx].getIndexValuesCache().getInts(id);
-//          if (current == null) // nocommit does that work with default values?
-//            current = new DummySource();
-//        } catch (IOException ex) {
-//          // nocommit what to do here?
-//          throw new RuntimeException(ex);
-//        }
-//        start = starts[idx];
-//        numDocs_ = subReaders[idx].maxDoc();
-//        n = docID - start;
-//      }
-//      return current.ints(n);
-      return 0l;
-    }
-
-    public double floats(int docID) {
-//      int n = docID - start;
-//      if (n >= numDocs_) {
-//        int idx = readerIndex(docID);
-//        try {
-//          current = subReaders[idx].getIndexValuesCache().getFloats(id);
-//          if (current == null) // nocommit does that work with default values?
-//            current = new DummySource();
-//        } catch (IOException ex) {
-//          // nocommit what to do here?
-//          throw new RuntimeException(ex);
-//        }
-//        numDocs_ = subReaders[idx].maxDoc();
-//
-//        start = starts[idx];
-//        n = docID - start;
-//      }
-//      return current.floats(n);
-      return 0d;
-    }
-
-    public BytesRef bytes(int docID) {
-//      int n = docID - start;
-//      if (n >= numDocs_) {
-//        int idx = readerIndex(docID);
-//        try {
-//          current = subReaders[idx].getIndexValuesCache().getBytes(id);
-//          if (current == null) // nocommit does that work with default values?
-//            current = new DummySource();
-//        } catch (IOException ex) {
-//          // nocommit what to do here?
-//          throw new RuntimeException(ex);
-//        }
-//        numDocs_ = subReaders[idx].maxDoc();
-//        start = starts[idx];
-//        n = docID - start;
-//      }
-//      return current.bytes(n);
-      return null;
+    public MultiSource(DocValuesIndex[] docValuesIdx, int[] starts) {
+      this.docValuesIdx = docValuesIdx;
+      this.starts = starts;
+
+    }
+
+    public long getInt(int docID) {
+      final int doc = ensureSource(docID);
+      return current.getInt(doc);
+    }
+
+    private final int ensureSource(int docID) {
+      int n = docID - start;
+      if (n >= numDocs) {
+        final int idx = ReaderUtil.subIndex(docID, starts);
+        assert idx >= 0 && idx < docValuesIdx.length : "idx was " + idx
+            + " for doc id: " + docID + " slices : " + Arrays.toString(starts);
+        assert docValuesIdx[idx] != null;
+        try {
+          current = docValuesIdx[idx].docValues.load();
+        } catch (IOException e) {
+          throw new RuntimeException("load failed", e); // TODO how should we
+          // handle this
+        }
+
+        start = docValuesIdx[idx].start;
+        numDocs = docValuesIdx[idx].length;
+        n = docID - start;
+      }
+      return n;
+    }
+
+    public double getFloat(int docID) {
+      final int doc = ensureSource(docID);
+      return current.getFloat(doc);
+    }
+
+    public BytesRef getBytes(int docID) {
+      final int doc = ensureSource(docID);
+      return current.getBytes(doc);
     }
 
     public long ramBytesUsed() {
@@ -218,17 +237,17 @@ public class MultiDocValues extends DocV
     private final BytesRef ref = new BytesRef();
 
     @Override
-    public BytesRef bytes(int docID) {
+    public BytesRef getBytes(int docID) {
       return ref;
     }
 
     @Override
-    public double floats(int docID) {
+    public double getFloat(int docID) {
       return 0.0d;
     }
 
     @Override
-    public long ints(int docID) {
+    public long getInt(int docID) {
       return 0;
     }
 
@@ -296,5 +315,4 @@ public class MultiDocValues extends DocV
   public Values type() {
     return this.docValuesIdx[0].docValues.type();
   }
-
 }

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java Fri Oct 29 20:33:36 2010
@@ -179,8 +179,8 @@ class PackedIntsImpl {
       }
 
       @Override
-      public long ints(int docID) {
-        // nocommit -- can we somehow avoid 2X method calls
+      public long getInt(int docID) {
+        // TODO -- can we somehow avoid 2X method calls
         // on each get? must push minValue down, and make
         // PackedInts implement Ints.Source
         final long val = values.get(docID);
@@ -195,7 +195,9 @@ class PackedIntsImpl {
       }
     }
 
+    @Override
     public void close() throws IOException {
+      super.close();
       datIn.close();
     }
 

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Values.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Values.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Values.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Values.java Fri Oct 29 20:33:36 2010
@@ -34,7 +34,7 @@ public enum Values {
   SIMPLE_FLOAT_4BYTE,
   SIMPLE_FLOAT_8BYTE,
 
-  // nocommit -- shouldn't lucene decide/detect straight vs
+  // TODO(simonw): -- shouldn't lucene decide/detect straight vs
   // deref, as well fixed vs var?
   BYTES_FIXED_STRAIGHT,
   BYTES_FIXED_DEREF,
@@ -44,5 +44,5 @@ public enum Values {
   BYTES_VAR_DEREF,
   BYTES_VAR_SORTED
 
-  // nocommit -- need STRING variants as well
+  // TODO(simonw): -- need STRING variants as well
 }

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java Fri Oct 29 20:33:36 2010
@@ -135,7 +135,7 @@ class VarDerefBytesImpl {
       idxOut.writeInt(address-1);
 
       // write index
-      // nocommit -- allow forcing fixed array (not -1)
+      // TODO(simonw): -- allow forcing fixed array (not -1)
       // TODO(simonw): check the address calculation / make it more intuitive
       final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount, PackedInts.bitsRequired(address-1));
       final int limit;
@@ -188,7 +188,7 @@ class VarDerefBytesImpl {
       }
 
       @Override
-      public BytesRef bytes(int docID) {
+      public BytesRef getBytes(int docID) {
         int address = (int) index.get(docID);
         if (address == 0) {
           assert defaultValue.length == 0: " default value manipulated";

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java Fri Oct 29 20:33:36 2010
@@ -120,7 +120,7 @@ class VarSortedBytesImpl {
       idxOut.writeLong(offset);
 
       // write index -- first doc -> 1+ord
-      // nocommit -- allow not -1:
+      // TODO(simonw): allow not -1:
       final PackedInts.Writer indexWriter = PackedInts.getWriter(idxOut,
           docCount, PackedInts.bitsRequired(count));
       final int limit = docCount > docToEntry.length ? docToEntry.length
@@ -135,7 +135,7 @@ class VarSortedBytesImpl {
       indexWriter.finish();
 
       // next ord (0-based) -> offset
-      // nocommit -- allow not -1:
+      // TODO(simonw): -- allow not -1:
       PackedInts.Writer offsetWriter = PackedInts.getWriter(idxOut, count,
           PackedInts.bitsRequired(lastOffset));
       for (int i = 0; i < count; i++) {

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java Fri Oct 29 20:33:36 2010
@@ -82,14 +82,15 @@ class VarStraightBytesImpl {
 
     @Override
     synchronized public void finish(int docCount) throws IOException {
-      if (datOut == null)
+      if (datOut == null) {
         return;
+      }
       initIndexOut();
       // write all lengths to index
       // write index
       fill(docCount);
       idxOut.writeVInt(address);
-      // nocommit -- allow not -1
+      // TODO(simonw): allow not -1
       final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount,
           PackedInts.bitsRequired(address));
       for (int i = 0; i < docCount; i++) {
@@ -136,7 +137,7 @@ class VarStraightBytesImpl {
       }
 
       @Override
-      public BytesRef bytes(int docID) {
+      public BytesRef getBytes(int docID) {
         final int address = (int) addresses.get(docID);
         bytesRef.offset = address;
         if (docID == maxDoc - 1) {

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java Fri Oct 29 20:33:36 2010
@@ -336,7 +336,7 @@ public abstract class FieldComparator {
 
     @Override
     public int compareBottom(int doc) {
-      final double v2 = currentReaderValues.floats(doc);
+      final double v2 = currentReaderValues.getFloat(doc);
       if (bottom > v2) {
         return 1;
       } else if (bottom < v2) {
@@ -348,12 +348,12 @@ public abstract class FieldComparator {
 
     @Override
     public void copy(int slot, int doc) {
-      values[slot] = currentReaderValues.floats(doc);
+      values[slot] = currentReaderValues.getFloat(doc);
     }
 
     @Override
     public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException {
-      currentReaderValues = reader.getIndexValuesCache().getFloats(field);
+      currentReaderValues = reader.docValues(field).getCached(true);
       return this;
     }
     
@@ -538,7 +538,7 @@ public abstract class FieldComparator {
     public int compareBottom(int doc) {
       // TODO: there are sneaky non-branch ways to compute
       // -1/+1/0 sign
-      final long v2 = currentReaderValues.ints(doc);
+      final long v2 = currentReaderValues.getInt(doc);
       if (bottom > v2) {
         return 1;
       } else if (bottom < v2) {
@@ -550,12 +550,12 @@ public abstract class FieldComparator {
 
     @Override
     public void copy(int slot, int doc) {
-      values[slot] = currentReaderValues.ints(doc);
+      values[slot] = currentReaderValues.getInt(doc);
     }
 
     @Override
     public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException {
-      currentReaderValues = reader.getIndexValuesCache().getInts(field);
+      currentReaderValues = reader.docValues(field).getCached(true);
       return this;
     }
     

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/SortField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/SortField.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/SortField.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/SortField.java Fri Oct 29 20:33:36 2010
@@ -32,7 +32,7 @@ import org.apache.lucene.search.cache.Sh
 import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.BytesRef;
 
-// nocommit -- for cleaner transition, maybe we should make
+// TODO(simonw) -- for cleaner transition, maybe we should make
 // a new SortField that subclasses this one and always uses
 // index values?
 

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ReaderUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ReaderUtil.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ReaderUtil.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ReaderUtil.java Fri Oct 29 20:33:36 2010
@@ -173,4 +173,26 @@ public class ReaderUtil {
     }
     return hi;
   }
+  
+  public static int subIndex(int n, Slice[] slices) {
+    // searcher/reader for doc n:
+    int size = slices.length;
+    int lo = 0; // search starts array
+    int hi = size - 1; // for first element less than n, return its index
+    while (hi >= lo) {
+      int mid = (lo + hi) >>> 1;
+      int midValue = slices[mid].start;
+      if (n < midValue)
+        hi = mid - 1;
+      else if (n > midValue)
+        lo = mid + 1;
+      else { // found a match
+        while (mid + 1 < size && slices[mid + 1].start == midValue) {
+          mid++; // scan to last match
+        }
+        return mid;
+      }
+    }
+    return hi;
+  }
 }

Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestIndexValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestIndexValues.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestIndexValues.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestIndexValues.java Fri Oct 29 20:33:36 2010
@@ -70,19 +70,19 @@ public class TestIndexValues extends Luc
   public static void beforeClassLuceneTestCaseJ4() {
     LuceneTestCase.beforeClassLuceneTestCaseJ4();
     final CodecProvider cp = CodecProvider.getDefault();
-    docValuesCodec = new DocValuesCodec(cp.lookup(CodecProvider.getDefaultCodec()));
+    docValuesCodec = new DocValuesCodec(cp.lookup(CodecProvider
+        .getDefaultCodec()));
     cp.register(docValuesCodec);
     CodecProvider.setDefaultCodec(docValuesCodec.name);
   }
-  
+
   @AfterClass
   public static void afterClassLuceneTestCaseJ4() {
     final CodecProvider cp = CodecProvider.getDefault();
     cp.unregister(docValuesCodec);
-    LuceneTestCase.afterClassLuceneTestCaseJ4();    
+    LuceneTestCase.afterClassLuceneTestCaseJ4();
   }
-  
-  
+
   public void testBytesStraight() throws IOException {
     runTestBytes(Bytes.Mode.STRAIGHT, true);
     runTestBytes(Bytes.Mode.STRAIGHT, false);
@@ -164,14 +164,14 @@ public class TestIndexValues extends Luc
       if (mode == Bytes.Mode.SORTED) {
         s = ss = r.loadSorted(comp);
       } else {
-        s = r.load();
+        s = getSource(r);
         ss = null;
       }
 
       for (int i = 0; i < 100; i++) {
         final int idx = 2 * i;
-        assertNotNull("doc " + idx + "; value=" + values[idx], s.bytes(idx));
-        assertEquals("doc " + idx, values[idx], s.bytes(idx).utf8ToString());
+        assertNotNull("doc " + idx + "; value=" + values[idx], s.getBytes(idx));
+        assertEquals("doc " + idx, values[idx], s.getBytes(idx).utf8ToString());
         if (ss != null) {
           assertEquals("doc " + idx, values[idx], ss.getByOrd(ss.ord(idx))
               .utf8ToString());
@@ -247,9 +247,9 @@ public class TestIndexValues extends Luc
 
         DocValues r = Ints.getValues(dir, "test", useFixedArrays);
         for (int iter = 0; iter < 2; iter++) {
-          Source s = r.load();
+          Source s = getSource(r);
           for (int i = 0; i < NUM_VALUES; i++) {
-            final long v = s.ints(i);
+            final long v = s.getInt(i);
             assertEquals("index " + i + " b: " + b, values[i], v);
           }
         }
@@ -311,9 +311,9 @@ public class TestIndexValues extends Luc
 
     DocValues r = Floats.getValues(dir, "test", NUM_VALUES + additionalValues);
     for (int iter = 0; iter < 2; iter++) {
-      Source s = r.load();
+      Source s = getSource(r);
       for (int i = 0; i < NUM_VALUES; i++) {
-        assertEquals(values[i], s.floats(i), 0.0f);
+        assertEquals(values[i], s.getFloat(i), 0.0f);
       }
     }
 
@@ -437,12 +437,12 @@ public class TestIndexValues extends Luc
       case PACKED_INTS:
       case PACKED_INTS_FIXED: {
         DocValues intsReader = getDocValues(r, val.name());
-        Source ints = intsReader.load();
+        Source ints = getSource(intsReader);
         ValuesEnum intsEnum = intsReader.getEnum();
         assertNotNull(intsEnum);
         LongsRef enumRef = intsEnum.addAttribute(ValuesAttribute.class).ints();
         for (int i = 0; i < base; i++) {
-          assertEquals(0, ints.ints(i));
+          assertEquals(0, ints.getInt(i));
           assertEquals(val.name() + " base: " + base + " index: " + i, i,
               random.nextBoolean() ? intsEnum.advance(i) : intsEnum.nextDoc());
           assertEquals(0, enumRef.get());
@@ -454,8 +454,8 @@ public class TestIndexValues extends Luc
           }
           assertEquals("advance failed at index: " + i + " of " + r.numDocs()
               + " docs", i, intsEnum.advance(i));
-          assertEquals(expected, ints.ints(i));
           assertEquals(expected, enumRef.get());
+          assertEquals(expected, ints.getInt(i));
 
         }
       }
@@ -463,14 +463,16 @@ public class TestIndexValues extends Luc
       case SIMPLE_FLOAT_4BYTE:
       case SIMPLE_FLOAT_8BYTE: {
         DocValues floatReader = getDocValues(r, val.name());
-        Source floats = floatReader.load();
+        assertNotNull(floatReader);
+        Source floats = getSource(floatReader);
         ValuesEnum floatEnum = floatReader.getEnum();
         assertNotNull(floatEnum);
         FloatsRef enumRef = floatEnum.addAttribute(ValuesAttribute.class)
             .floats();
 
         for (int i = 0; i < base; i++) {
-          assertEquals(0.0d, floats.floats(i), 0.0d);
+          assertEquals(" floats failed for doc: " + i + " base: " + base, 0.0d,
+              floats.getFloat(i), 0.0d);
           assertEquals(i, random.nextBoolean() ? floatEnum.advance(i)
               : floatEnum.nextDoc());
           assertEquals("index " + i, 0.0, enumRef.get(), 0.0);
@@ -483,7 +485,8 @@ public class TestIndexValues extends Luc
           assertEquals("advance failed at index: " + i + " of " + r.numDocs()
               + " docs base:" + base, i, floatEnum.advance(i));
           assertEquals("index " + i, 2.0 * expected, enumRef.get(), 0.00001);
-          assertEquals("index " + i, 2.0 * expected, floats.floats(i), 0.00001);
+          assertEquals("index " + i, 2.0 * expected, floats.getFloat(i),
+              0.00001);
         }
       }
         break;
@@ -505,15 +508,13 @@ public class TestIndexValues extends Luc
       Values.BYTES_VAR_DEREF, Values.BYTES_VAR_SORTED,
       Values.BYTES_VAR_STRAIGHT);
 
-  private static EnumSet<Values> STRAIGHT_BYTES = EnumSet.of(
-      Values.BYTES_FIXED_STRAIGHT, Values.BYTES_VAR_STRAIGHT);
-
   private static EnumSet<Values> NUMERICS = EnumSet.of(Values.PACKED_INTS,
       Values.PACKED_INTS_FIXED, Values.SIMPLE_FLOAT_4BYTE,
       Values.SIMPLE_FLOAT_8BYTE);
 
   private static Index[] IDX_VALUES = new Index[] { Index.ANALYZED,
-      Index.ANALYZED_NO_NORMS, Index.NOT_ANALYZED, Index.NOT_ANALYZED_NO_NORMS };
+      Index.ANALYZED_NO_NORMS, Index.NOT_ANALYZED, Index.NOT_ANALYZED_NO_NORMS,
+      Index.NO };
 
   private OpenBitSet indexValues(IndexWriter w, int numValues, Values value,
       List<Values> valueVarList, boolean withDeletions, int multOfSeven)
@@ -521,9 +522,10 @@ public class TestIndexValues extends Luc
     final boolean isNumeric = NUMERICS.contains(value);
     OpenBitSet deleted = new OpenBitSet(numValues);
     Document doc = new Document();
+    Index idx = IDX_VALUES[random.nextInt(IDX_VALUES.length)];
     Fieldable field = random.nextBoolean() ? new ValuesField(value.name())
         : newField(value.name(), _TestUtil.randomRealisticUnicodeString(random,
-            10), IDX_VALUES[random.nextInt(IDX_VALUES.length)]);
+            10), idx == Index.NO ? Store.YES : Store.NO, idx);
     doc.add(field);
 
     ValuesAttribute valuesAttribute = ValuesField.values(field);
@@ -582,9 +584,10 @@ public class TestIndexValues extends Luc
     }
     w.commit();
 
-    // nocommit test unoptimized with deletions
-    if (true || withDeletions || random.nextBoolean())
-      w.optimize();
+    // TODO test unoptimized with deletions
+    if (withDeletions || random.nextBoolean())
+      ;
+    w.optimize();
     return deleted;
   }
 
@@ -593,10 +596,9 @@ public class TestIndexValues extends Luc
     Directory d = newDirectory();
     IndexWriter w = new IndexWriter(d, cfg);
     final List<Values> byteVariantList = new ArrayList<Values>(BYTES);
-
     // run in random order to test if fill works correctly during merges
     Collections.shuffle(byteVariantList, random);
-    final int numValues = 350;
+    final int numValues = 333 + random.nextInt(150);
     for (Values byteIndexValue : byteVariantList) {
       List<Closeable> closeables = new ArrayList<Closeable>();
 
@@ -607,11 +609,10 @@ public class TestIndexValues extends Luc
       assertEquals(0, r.numDeletedDocs());
       final int numRemainingValues = (int) (numValues - deleted.cardinality());
       final int base = r.numDocs() - numRemainingValues;
-
       DocValues bytesReader = getDocValues(r, byteIndexValue.name());
       assertNotNull("field " + byteIndexValue.name()
           + " returned null reader - maybe merged failed", bytesReader);
-      Source bytes = bytesReader.load();
+      Source bytes = getSource(bytesReader);
       ValuesEnum bytesEnum = bytesReader.getEnum();
       assertNotNull(bytesEnum);
       final ValuesAttribute attr = bytesEnum
@@ -619,7 +620,7 @@ public class TestIndexValues extends Luc
       byte upto = 0;
       // test the filled up slots for correctness
       for (int i = 0; i < base; i++) {
-        final BytesRef br = bytes.bytes(i);
+        final BytesRef br = bytes.getBytes(i);
         String msg = " field: " + byteIndexValue.name() + " at index: " + i
             + " base: " + base + " numDocs:" + r.numDocs();
         switch (byteIndexValue) {
@@ -645,7 +646,7 @@ public class TestIndexValues extends Luc
         default:
           assertNotNull("expected none null - " + msg, br);
           if (br.length != 0) {
-            bytes.bytes(i);
+            bytes.getBytes(i);
           }
           assertEquals("expected empty bytes - " + br.utf8ToString() + msg, 0,
               br.length);
@@ -665,7 +666,7 @@ public class TestIndexValues extends Luc
           upto += bytesSize;
         }
 
-        BytesRef br = bytes.bytes(i);
+        BytesRef br = bytes.getBytes(i);
         if (bytesEnum.docID() != i)
           assertEquals("seek failed for index " + i + " " + msg, i, bytesEnum
               .advance(i));
@@ -692,10 +693,9 @@ public class TestIndexValues extends Luc
   private DocValues getDocValues(IndexReader reader, String field)
       throws IOException {
     boolean optimized = reader.isOptimized();
-    Fields fields = optimized ? reader.getSequentialSubReaders()[0].fields() : MultiFields
-        .getFields(reader);
-//    return fields.docValues(field);
-    switch (random.nextInt(optimized ? 3 : 2)) {
+    Fields fields = optimized ? reader.getSequentialSubReaders()[0].fields()
+        : MultiFields.getFields(reader);
+    switch (random.nextInt(optimized ? 3 : 2)) { // case 2 only if optimized
     case 0:
       return fields.docValues(field);
     case 1:
@@ -706,10 +706,14 @@ public class TestIndexValues extends Luc
           return iterator.docValues();
       }
       throw new RuntimeException("no such field " + field);
-    case 2:
+    case 2:// this only works if we are on an optimized index!
       return reader.getSequentialSubReaders()[0].docValues(field);
     }
-throw new RuntimeException();
-}
+    throw new RuntimeException();
+  }
+
+  private Source getSource(DocValues values) throws IOException {
+    return random.nextBoolean() ? values.load() : values.getCached(true);
+  }
 
 }