svn commit: r1443717 [13/14] - in /lucene/dev/trunk: ./ dev-tools/ lucene/ lucene/analysis/ lucene/analysis/common/ lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/analysis/icu/src/test/org/apache/lucene/collation/ lucene/backwards/ lu...

[rm...@apache.org]

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java Thu Feb  7 20:48:21 2013
@@ -77,11 +77,11 @@ public final class NestedPulsingPostings
     PostingsReaderBase pulsingReader = null;
     boolean success = false;
     try {
-      docsReader = new Lucene41PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
+      docsReader = new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
       pulsingReaderInner = new PulsingPostingsReader(docsReader);
       pulsingReader = new PulsingPostingsReader(pulsingReaderInner);
       FieldsProducer ret = new BlockTreeTermsReader(
-                                                    state.dir, state.fieldInfos, state.segmentInfo,
+                                                    state.directory, state.fieldInfos, state.segmentInfo,
                                                     pulsingReader,
                                                     state.context,
                                                     state.segmentSuffix,

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java Thu Feb  7 20:48:21 2013
@@ -555,7 +555,7 @@ public final class RAMOnlyPostingsFormat
 
     // Load our ID:
     final String idFileName = IndexFileNames.segmentFileName(readState.segmentInfo.name, readState.segmentSuffix, ID_EXTENSION);
-    IndexInput in = readState.dir.openInput(idFileName, readState.context);
+    IndexInput in = readState.directory.openInput(idFileName, readState.context);
     boolean success = false;
     final int id;
     try {

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java Thu Feb  7 20:48:21 2013
@@ -86,6 +86,7 @@ public class AssertingAtomicReader exten
     public TermsEnum intersect(CompiledAutomaton automaton, BytesRef bytes) throws IOException {
       TermsEnum termsEnum = super.intersect(automaton, bytes);
       assert termsEnum != null;
+      assert bytes == null || bytes.isValid();
       return new AssertingTermsEnum(termsEnum);
     }
 
@@ -145,6 +146,7 @@ public class AssertingAtomicReader exten
       if (result == null) {
         state = State.UNPOSITIONED;
       } else {
+        assert result.isValid();
         state = State.POSITIONED;
       }
       return result;
@@ -171,7 +173,9 @@ public class AssertingAtomicReader exten
     @Override
     public BytesRef term() throws IOException {
       assert state == State.POSITIONED : "term() called on unpositioned TermsEnum";
-      return super.term();
+      BytesRef ret = super.term();
+      assert ret == null || ret.isValid();
+      return ret;
     }
 
     @Override
@@ -182,6 +186,7 @@ public class AssertingAtomicReader exten
 
     @Override
     public SeekStatus seekCeil(BytesRef term, boolean useCache) throws IOException {
+      assert term.isValid();
       SeekStatus result = super.seekCeil(term, useCache);
       if (result == SeekStatus.END) {
         state = State.UNPOSITIONED;
@@ -193,6 +198,7 @@ public class AssertingAtomicReader exten
 
     @Override
     public boolean seekExact(BytesRef text, boolean useCache) throws IOException {
+      assert text.isValid();
       if (super.seekExact(text, useCache)) {
         state = State.POSITIONED;
         return true;
@@ -210,6 +216,7 @@ public class AssertingAtomicReader exten
 
     @Override
     public void seekExact(BytesRef term, TermState state) throws IOException {
+      assert term.isValid();
       super.seekExact(term, state);
       this.state = State.POSITIONED;
     }
@@ -352,10 +359,156 @@ public class AssertingAtomicReader exten
       assert state != DocsEnumState.FINISHED : "getPayload() called after NO_MORE_DOCS";
       assert positionCount > 0 : "getPayload() called before nextPosition()!";
       BytesRef payload = super.getPayload();
-      assert payload == null || payload.length > 0 : "getPayload() returned payload with invalid length!";
+      assert payload == null || payload.isValid() && payload.length > 0 : "getPayload() returned payload with invalid length!";
       return payload;
     }
   }
+  
+  /** Wraps a NumericDocValues but with additional asserts */
+  public static class AssertingNumericDocValues extends NumericDocValues {
+    private final NumericDocValues in;
+    private final int maxDoc;
+    
+    public AssertingNumericDocValues(NumericDocValues in, int maxDoc) {
+      this.in = in;
+      this.maxDoc = maxDoc;
+    }
+
+    @Override
+    public long get(int docID) {
+      assert docID >= 0 && docID < maxDoc;
+      return in.get(docID);
+    }    
+  }
+  
+  /** Wraps a BinaryDocValues but with additional asserts */
+  public static class AssertingBinaryDocValues extends BinaryDocValues {
+    private final BinaryDocValues in;
+    private final int maxDoc;
+    
+    public AssertingBinaryDocValues(BinaryDocValues in, int maxDoc) {
+      this.in = in;
+      this.maxDoc = maxDoc;
+    }
+
+    @Override
+    public void get(int docID, BytesRef result) {
+      assert docID >= 0 && docID < maxDoc;
+      assert result.isValid();
+      in.get(docID, result);
+      assert result.isValid();
+    }
+  }
+  
+  /** Wraps a SortedDocValues but with additional asserts */
+  public static class AssertingSortedDocValues extends SortedDocValues {
+    private final SortedDocValues in;
+    private final int maxDoc;
+    private final int valueCount;
+    
+    public AssertingSortedDocValues(SortedDocValues in, int maxDoc) {
+      this.in = in;
+      this.maxDoc = maxDoc;
+      this.valueCount = in.getValueCount();
+      assert valueCount >= 1 && valueCount <= maxDoc;
+    }
+
+    @Override
+    public int getOrd(int docID) {
+      assert docID >= 0 && docID < maxDoc;
+      int ord = in.getOrd(docID);
+      assert ord >= 0 && ord < valueCount;
+      return ord;
+    }
+
+    @Override
+    public void lookupOrd(int ord, BytesRef result) {
+      assert ord >= 0 && ord < valueCount;
+      assert result.isValid();
+      in.lookupOrd(ord, result);
+      assert result.isValid();
+    }
+
+    @Override
+    public int getValueCount() {
+      int valueCount = in.getValueCount();
+      assert valueCount == this.valueCount; // should not change
+      return valueCount;
+    }
+
+    @Override
+    public void get(int docID, BytesRef result) {
+      assert docID >= 0 && docID < maxDoc;
+      assert result.isValid();
+      in.get(docID, result);
+      assert result.isValid();
+    }
+
+    @Override
+    public int lookupTerm(BytesRef key) {
+      assert key.isValid();
+      int result = in.lookupTerm(key);
+      assert result < valueCount;
+      assert key.isValid();
+      return result;
+    }
+  }
+
+  @Override
+  public NumericDocValues getNumericDocValues(String field) throws IOException {
+    NumericDocValues dv = super.getNumericDocValues(field);
+    FieldInfo fi = getFieldInfos().fieldInfo(field);
+    if (dv != null) {
+      assert fi != null;
+      assert fi.getDocValuesType() == FieldInfo.DocValuesType.NUMERIC;
+      return new AssertingNumericDocValues(dv, maxDoc());
+    } else {
+      assert fi == null || fi.getDocValuesType() != FieldInfo.DocValuesType.NUMERIC;
+      return null;
+    }
+  }
+
+  @Override
+  public BinaryDocValues getBinaryDocValues(String field) throws IOException {
+    BinaryDocValues dv = super.getBinaryDocValues(field);
+    FieldInfo fi = getFieldInfos().fieldInfo(field);
+    if (dv != null) {
+      assert fi != null;
+      assert fi.getDocValuesType() == FieldInfo.DocValuesType.BINARY;
+      return new AssertingBinaryDocValues(dv, maxDoc());
+    } else {
+      assert fi == null || fi.getDocValuesType() != FieldInfo.DocValuesType.BINARY;
+      return null;
+    }
+  }
+
+  @Override
+  public SortedDocValues getSortedDocValues(String field) throws IOException {
+    SortedDocValues dv = super.getSortedDocValues(field);
+    FieldInfo fi = getFieldInfos().fieldInfo(field);
+    if (dv != null) {
+      assert fi != null;
+      assert fi.getDocValuesType() == FieldInfo.DocValuesType.SORTED;
+      return new AssertingSortedDocValues(dv, maxDoc());
+    } else {
+      assert fi == null || fi.getDocValuesType() != FieldInfo.DocValuesType.SORTED;
+      return null;
+    }
+  }
+
+  @Override
+  public NumericDocValues getNormValues(String field) throws IOException {
+    NumericDocValues dv = super.getNormValues(field);
+    FieldInfo fi = getFieldInfos().fieldInfo(field);
+    if (dv != null) {
+      assert fi != null;
+      assert fi.hasNorms();
+      return new AssertingNumericDocValues(dv, maxDoc());
+    } else {
+      assert fi == null || fi.hasNorms() == false;
+      return null;
+    }
+  }
 
   // this is the same hack as FCInvisible
   @Override

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java Thu Feb  7 20:48:21 2013
@@ -38,6 +38,7 @@ import org.apache.lucene.codecs.FieldsPr
 import org.apache.lucene.codecs.PostingsConsumer;
 import org.apache.lucene.codecs.TermStats;
 import org.apache.lucene.codecs.TermsConsumer;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FlushInfo;
@@ -345,7 +346,7 @@ public abstract class BasePostingsFormat
 
       fieldInfoArray[fieldUpto] = new FieldInfo(field, true, fieldUpto, false, false, true,
                                                 IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
-                                                null, DocValues.Type.FIXED_INTS_8, null);
+                                                null, DocValuesType.NUMERIC, null);
       fieldUpto++;
 
       Map<BytesRef,Long> postings = new TreeMap<BytesRef,Long>();
@@ -470,7 +471,7 @@ public abstract class BasePostingsFormat
                                                    doPayloads,
                                                    indexOptions,
                                                    null,
-                                                   DocValues.Type.FIXED_INTS_8,
+                                                   DocValuesType.NUMERIC,
                                                    null);
     }
 

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java Thu Feb  7 20:48:21 2013
@@ -113,15 +113,27 @@ public final class FieldFilterAtomicRead
     final Fields f = super.fields();
     return (f == null) ? null : new FieldFilterFields(f);
   }
+  
+  
+
+  @Override
+  public NumericDocValues getNumericDocValues(String field) throws IOException {
+    return hasField(field) ? super.getNumericDocValues(field) : null;
+  }
+
+  @Override
+  public BinaryDocValues getBinaryDocValues(String field) throws IOException {
+    return hasField(field) ? super.getBinaryDocValues(field) : null;
+  }
 
   @Override
-  public DocValues docValues(String field) throws IOException {
-    return hasField(field) ? super.docValues(field) : null;
+  public SortedDocValues getSortedDocValues(String field) throws IOException {
+    return hasField(field) ? super.getSortedDocValues(field) : null;
   }
 
   @Override
-  public DocValues normValues(String field) throws IOException {
-    return hasField(field) ? super.normValues(field) : null;
+  public NumericDocValues getNormValues(String field) throws IOException {
+    return hasField(field) ? super.getNormValues(field) : null;
   }
 
   @Override

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java Thu Feb  7 20:48:21 2013
@@ -28,11 +28,16 @@ import java.util.Random;
 import java.util.Set;
 
 import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.asserting.AssertingDocValuesFormat;
 import org.apache.lucene.codecs.asserting.AssertingPostingsFormat;
 import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
 import org.apache.lucene.codecs.lucene41ords.Lucene41WithOrds;
 import org.apache.lucene.codecs.lucene42.Lucene42Codec;
+import org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat;
 import org.apache.lucene.codecs.bloom.TestBloomFilteredLucene41Postings;
+import org.apache.lucene.codecs.cheapbastard.CheapBastardDocValuesFormat;
+import org.apache.lucene.codecs.diskdv.DiskDocValuesFormat;
 import org.apache.lucene.codecs.memory.DirectPostingsFormat;
 import org.apache.lucene.codecs.memory.MemoryPostingsFormat;
 import org.apache.lucene.codecs.mockintblock.MockFixedIntBlockPostingsFormat;
@@ -42,6 +47,7 @@ import org.apache.lucene.codecs.mocksep.
 import org.apache.lucene.codecs.nestedpulsing.NestedPulsingPostingsFormat;
 import org.apache.lucene.codecs.pulsing.Pulsing41PostingsFormat;
 import org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat;
+import org.apache.lucene.codecs.simpletext.SimpleTextDocValuesFormat;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
 
@@ -58,14 +64,21 @@ public class RandomCodec extends Lucene4
   /** Shuffled list of postings formats to use for new mappings */
   private List<PostingsFormat> formats = new ArrayList<PostingsFormat>();
   
+  /** Shuffled list of docvalues formats to use for new mappings */
+  private List<DocValuesFormat> dvFormats = new ArrayList<DocValuesFormat>();
+  
   /** unique set of format names this codec knows about */
   public Set<String> formatNames = new HashSet<String>();
+  
+  /** unique set of docvalues format names this codec knows about */
+  public Set<String> dvFormatNames = new HashSet<String>();
 
   /** memorized field->postingsformat mappings */
   // note: we have to sync this map even though its just for debugging/toString, 
   // otherwise DWPT's .toString() calls that iterate over the map can 
   // cause concurrentmodificationexception if indexwriter's infostream is on
   private Map<String,PostingsFormat> previousMappings = Collections.synchronizedMap(new HashMap<String,PostingsFormat>());
+  private Map<String,DocValuesFormat> previousDVMappings = Collections.synchronizedMap(new HashMap<String,DocValuesFormat>());
   private final int perFieldSeed;
 
   @Override
@@ -84,6 +97,22 @@ public class RandomCodec extends Lucene4
     return codec;
   }
 
+  @Override
+  public DocValuesFormat getDocValuesFormatForField(String name) {
+    DocValuesFormat codec = previousDVMappings.get(name);
+    if (codec == null) {
+      codec = dvFormats.get(Math.abs(perFieldSeed ^ name.hashCode()) % dvFormats.size());
+      if (codec instanceof SimpleTextDocValuesFormat && perFieldSeed % 5 != 0) {
+        // make simpletext rarer, choose again
+        codec = dvFormats.get(Math.abs(perFieldSeed ^ name.toUpperCase(Locale.ROOT).hashCode()) % dvFormats.size());
+      }
+      previousDVMappings.put(name, codec);
+      // Safety:
+      assert previousDVMappings.size() < 10000: "test went insane";
+    }
+    return codec;
+  }
+
   public RandomCodec(Random random, Set<String> avoidCodecs) {
     this.perFieldSeed = random.nextInt();
     // TODO: make it possible to specify min/max iterms per
@@ -113,11 +142,20 @@ public class RandomCodec extends Lucene4
         new AssertingPostingsFormat(),
         new MemoryPostingsFormat(true, random.nextFloat()),
         new MemoryPostingsFormat(false, random.nextFloat()));
+    
+    addDocValues(avoidCodecs,
+        new Lucene42DocValuesFormat(),
+        new DiskDocValuesFormat(),
+        new SimpleTextDocValuesFormat(),
+        new AssertingDocValuesFormat(),
+        new CheapBastardDocValuesFormat());
 
     Collections.shuffle(formats, random);
+    Collections.shuffle(dvFormats, random);
 
     // Avoid too many open files:
     formats = formats.subList(0, 4);
+    dvFormats = dvFormats.subList(0, 4);
   }
 
   public RandomCodec(Random random) {
@@ -132,9 +170,18 @@ public class RandomCodec extends Lucene4
       }
     }
   }
+  
+  private final void addDocValues(Set<String> avoidCodecs, DocValuesFormat... docvalues) {
+    for (DocValuesFormat d : docvalues) {
+      if (!avoidCodecs.contains(d.getName())) {
+        dvFormats.add(d);
+        dvFormatNames.add(d.getName());
+      }
+    }
+  }
 
   @Override
   public String toString() {
-    return super.toString() + ": " + previousMappings.toString();
+    return super.toString() + ": " + previousMappings.toString() + ", docValues:" + previousDVMappings.toString();
   }
 }

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java Thu Feb  7 20:48:21 2013
@@ -25,18 +25,12 @@ import java.util.Random;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.document.ByteDocValuesField; 
-import org.apache.lucene.document.DerefBytesDocValuesField; 
+import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.DoubleDocValuesField; 
 import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FloatDocValuesField; 
-import org.apache.lucene.document.IntDocValuesField; 
-import org.apache.lucene.document.LongDocValuesField; 
-import org.apache.lucene.document.PackedLongDocValuesField; 
-import org.apache.lucene.document.ShortDocValuesField; 
-import org.apache.lucene.document.SortedBytesDocValuesField; 
-import org.apache.lucene.document.StraightBytesDocValuesField; 
+import org.apache.lucene.document.NumericDocValuesField; 
+import org.apache.lucene.document.SortedDocValuesField; 
+import org.apache.lucene.index.FieldInfo.DocValuesType;
 import org.apache.lucene.index.IndexWriter; // javadoc
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
@@ -59,9 +53,6 @@ public class RandomIndexWriter implement
   int flushAt;
   private double flushAtFactor = 1.0;
   private boolean getReaderCalled;
-  private final int fixedBytesLength;
-  private final long docValuesFieldPrefix;
-  private volatile boolean doDocValues;
   private final Codec codec; // sugar
 
   // Randomly calls Thread.yield so we mixup thread scheduling
@@ -109,49 +100,12 @@ public class RandomIndexWriter implement
       System.out.println("RIW dir=" + dir + " config=" + w.getConfig());
       System.out.println("codec default=" + codec.getName());
     }
-    /* TODO: find some way to make this random...
-     * This must be fixed across all fixed bytes 
-     * fields in one index. so if you open another writer
-     * this might change if I use r.nextInt(x)
-     * maybe we can peek at the existing files here? 
-     */
-    fixedBytesLength = 17; 
-
-    // NOTE: this means up to 13 * 5 unique fields (we have
-    // 13 different DV types):
-    docValuesFieldPrefix = r.nextInt(5);
-    switchDoDocValues();
 
     // Make sure we sometimes test indices that don't get
     // any forced merges:
     doRandomForceMerge = r.nextBoolean();
   } 
   
-  private boolean addDocValuesFields = true;
-  
-  /**
-   * set to false if you don't want RandomIndexWriter
-   * adding docvalues fields.
-   */
-  public void setAddDocValuesFields(boolean v) {
-    addDocValuesFields = v;
-    switchDoDocValues();
-  }
-
-  private void switchDoDocValues() {
-    if (addDocValuesFields == false) {
-      doDocValues = false;
-      return;
-    }
-    // randomly enable / disable docValues 
-    doDocValues = LuceneTestCase.rarely(r);
-    if (LuceneTestCase.VERBOSE) {
-      if (doDocValues) {
-        System.out.println("NOTE: RIW: turning on random DocValues fields");
-      }
-    }
-  }
-  
   /**
    * Adds a Document.
    * @see IndexWriter#addDocument(org.apache.lucene.index.IndexDocument)
@@ -161,9 +115,6 @@ public class RandomIndexWriter implement
   }
 
   public <T extends IndexableField> void addDocument(final IndexDocument doc, Analyzer a) throws IOException {
-    if (doDocValues && doc instanceof Document) {
-      randomPerDocFieldValues((Document) doc);
-    }
     if (r.nextInt(5) == 3) {
       // TODO: maybe, we should simply buffer up added docs
       // (but we need to clone them), and only when
@@ -204,75 +155,6 @@ public class RandomIndexWriter implement
     maybeCommit();
   }
 
-  private BytesRef getFixedRandomBytes() {
-    final String randomUnicodeString = _TestUtil.randomFixedByteLengthUnicodeString(r, fixedBytesLength);
-    BytesRef fixedRef = new BytesRef(randomUnicodeString);
-    if (fixedRef.length > fixedBytesLength) {
-      fixedRef = new BytesRef(fixedRef.bytes, 0, fixedBytesLength);
-    } else {
-      fixedRef.grow(fixedBytesLength);
-      fixedRef.length = fixedBytesLength;
-    }
-    return fixedRef;
-  }
-  
-  private void randomPerDocFieldValues(Document doc) {
-    
-    DocValues.Type[] values = DocValues.Type.values();
-    DocValues.Type type = values[r.nextInt(values.length)];
-    String name = "random_" + type.name() + "" + docValuesFieldPrefix;
-    if (doc.getField(name) != null) {
-      return;
-    }
-    final Field f;
-    switch (type) {
-    case BYTES_FIXED_DEREF:
-      f = new DerefBytesDocValuesField(name, getFixedRandomBytes(), true);
-      break;
-    case BYTES_VAR_DEREF:
-      f = new DerefBytesDocValuesField(name, new BytesRef(_TestUtil.randomUnicodeString(r, 20)), false);
-      break;
-    case BYTES_FIXED_STRAIGHT:
-      f = new StraightBytesDocValuesField(name, getFixedRandomBytes(), true);
-      break;
-    case BYTES_VAR_STRAIGHT:
-      f = new StraightBytesDocValuesField(name, new BytesRef(_TestUtil.randomUnicodeString(r, 20)), false);
-      break;
-    case BYTES_FIXED_SORTED:
-      f = new SortedBytesDocValuesField(name, getFixedRandomBytes(), true);
-      break;
-    case BYTES_VAR_SORTED:
-      f = new SortedBytesDocValuesField(name, new BytesRef(_TestUtil.randomUnicodeString(r, 20)), false);
-      break;
-    case FLOAT_32:
-      f = new FloatDocValuesField(name, r.nextFloat());
-      break;
-    case FLOAT_64:
-      f = new DoubleDocValuesField(name, r.nextDouble());
-      break;
-    case VAR_INTS:
-      f = new PackedLongDocValuesField(name, r.nextLong());
-      break;
-    case FIXED_INTS_16:
-      // TODO: we should test negatives too?
-      f = new ShortDocValuesField(name, (short) r.nextInt(Short.MAX_VALUE));
-      break;
-    case FIXED_INTS_32:
-      f = new IntDocValuesField(name, r.nextInt());
-      break;
-    case FIXED_INTS_64:
-      f = new LongDocValuesField(name, r.nextLong());
-      break;
-    case FIXED_INTS_8:  
-      // TODO: we should test negatives too?
-      f = new ByteDocValuesField(name, (byte) r.nextInt(128));
-      break;
-    default:
-      throw new IllegalArgumentException("no such type: " + type);
-    }
-    doc.add(f);
-  }
-
   private void maybeCommit() throws IOException {
     if (docCount++ == flushAt) {
       if (LuceneTestCase.VERBOSE) {
@@ -284,7 +166,6 @@ public class RandomIndexWriter implement
         // gradually but exponentially increase time b/w flushes
         flushAtFactor *= 1.05;
       }
-      switchDoDocValues();
     }
   }
   
@@ -303,9 +184,6 @@ public class RandomIndexWriter implement
    * @see IndexWriter#updateDocument(Term, org.apache.lucene.index.IndexDocument)
    */
   public <T extends IndexableField> void updateDocument(Term t, final IndexDocument doc) throws IOException {
-    if (doDocValues) {
-      randomPerDocFieldValues((Document) doc);
-    }
     if (r.nextInt(5) == 3) {
       w.updateDocuments(t, new Iterable<IndexDocument>() {
 
@@ -359,7 +237,6 @@ public class RandomIndexWriter implement
   
   public void commit() throws IOException {
     w.commit();
-    switchDoDocValues();
   }
   
   public int numDocs() {
@@ -416,7 +293,6 @@ public class RandomIndexWriter implement
         assert !doRandomForceMergeAssert || w.getSegmentCount() <= limit: "limit=" + limit + " actual=" + w.getSegmentCount();
       }
     }
-    switchDoDocValues();
   }
 
   public DirectoryReader getReader(boolean applyDeletions) throws IOException {
@@ -437,7 +313,6 @@ public class RandomIndexWriter implement
         System.out.println("RIW.getReader: open new reader");
       }
       w.commit();
-      switchDoDocValues();
       if (r.nextBoolean()) {
         return DirectoryReader.open(w.getDirectory(), _TestUtil.nextInt(r, 1, 10));
       } else {

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java Thu Feb  7 20:48:21 2013
@@ -637,7 +637,6 @@ public abstract class ThreadedIndexingAn
     s.search(q, 10);
     int hitCount = s.search(q, null, 10, new Sort(new SortField("title", SortField.Type.STRING))).totalHits;
     final Sort dvSort = new Sort(new SortField("title", SortField.Type.STRING));
-    dvSort.getSort()[0].setUseIndexValues(true);
     int hitCount2 = s.search(q, null, 10, dvSort).totalHits;
     assertEquals(hitCount, hitCount2);
     return hitCount;

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java Thu Feb  7 20:48:21 2013
@@ -36,7 +36,7 @@ import java.util.zip.GZIPInputStream;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.SortedBytesDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
 
@@ -184,7 +184,7 @@ public class LineFileDocs implements Clo
       doc.add(date);
 
       if (useDocValues) {
-        titleDV = new SortedBytesDocValuesField("titleDV", new BytesRef());
+        titleDV = new SortedDocValuesField("titleDV", new BytesRef());
         doc.add(titleDV);
       } else {
         titleDV = null;

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Thu Feb  7 20:48:21 2013
@@ -253,6 +253,9 @@ public abstract class LuceneTestCase ext
 
   /** Gets the postingsFormat to run tests with. */
   public static final String TEST_POSTINGSFORMAT = System.getProperty("tests.postingsformat", "random");
+  
+  /** Gets the docValuesFormat to run tests with */
+  public static final String TEST_DOCVALUESFORMAT = System.getProperty("tests.docvaluesformat", "random");
 
   /** Gets the directory to run tests with */
   public static final String TEST_DIRECTORY = System.getProperty("tests.directory", "random");
@@ -1243,6 +1246,9 @@ public abstract class LuceneTestCase ext
       if (maybeWrap) {
         r = maybeWrapReader(r);
       }
+      if (r instanceof AtomicReader) {
+        _TestUtil.checkReader((AtomicReader)r);
+      }
       IndexSearcher ret = random.nextBoolean() ? new AssertingIndexSearcher(random, r) : new AssertingIndexSearcher(random, r.getContext());
       ret.setSimilarity(classEnvRule.similarity);
       return ret;

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java Thu Feb  7 20:48:21 2013
@@ -150,6 +150,7 @@ public final class RunListenerPrintRepro
     // Codec, postings, directories.
     if (!TEST_CODEC.equals("random")) addVmOpt(b, "tests.codec", TEST_CODEC);
     if (!TEST_POSTINGSFORMAT.equals("random")) addVmOpt(b, "tests.postingsformat", TEST_POSTINGSFORMAT);
+    if (!TEST_DOCVALUESFORMAT.equals("random")) addVmOpt(b, "tests.docvaluesformat", TEST_DOCVALUESFORMAT);
     if (!TEST_DIRECTORY.equals("random")) addVmOpt(b, "tests.directory", TEST_DIRECTORY);
 
     // Environment.

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java Thu Feb  7 20:48:21 2013
@@ -28,10 +28,13 @@ import java.util.Set;
 import java.util.TimeZone;
 
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.asserting.AssertingCodec;
+import org.apache.lucene.codecs.cheapbastard.CheapBastardCodec;
 import org.apache.lucene.codecs.compressing.CompressingCodec;
 import org.apache.lucene.codecs.lucene40.Lucene40Codec;
+import org.apache.lucene.codecs.lucene40.Lucene40RWCodec;
 import org.apache.lucene.codecs.lucene40.Lucene40RWPostingsFormat;
 import org.apache.lucene.codecs.lucene41.Lucene41Codec;
 import org.apache.lucene.codecs.lucene41.Lucene41RWCodec;
@@ -137,23 +140,39 @@ final class TestRuleSetupAndRestoreClass
     int randomVal = random.nextInt(10);
     if ("Lucene40".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) &&
                                           "random".equals(TEST_POSTINGSFORMAT) &&
+                                          "random".equals(TEST_DOCVALUESFORMAT) &&
                                           randomVal == 0 &&
                                           !shouldAvoidCodec("Lucene40"))) {
       codec = Codec.forName("Lucene40");
+      assert codec instanceof Lucene40RWCodec : "fix your classpath to have tests-framework.jar before lucene-core.jar";
       assert (PostingsFormat.forName("Lucene40") instanceof Lucene40RWPostingsFormat) : "fix your classpath to have tests-framework.jar before lucene-core.jar";
     } else if ("Lucene41".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) &&
                                                  "random".equals(TEST_POSTINGSFORMAT) &&
+                                                 "random".equals(TEST_DOCVALUESFORMAT) &&
                                                  randomVal == 1 &&
                                                  !shouldAvoidCodec("Lucene41"))) { 
       codec = Codec.forName("Lucene41");
       assert codec instanceof Lucene41RWCodec : "fix your classpath to have tests-framework.jar before lucene-core.jar";
-    } else if (!"random".equals(TEST_POSTINGSFORMAT)) {
+    } else if (("random".equals(TEST_POSTINGSFORMAT) == false) || ("random".equals(TEST_DOCVALUESFORMAT) == false)) {
+      // the user wired postings or DV: this is messy
+      // refactor into RandomCodec....
+      
       final PostingsFormat format;
-      if ("MockRandom".equals(TEST_POSTINGSFORMAT)) {
-        format = new MockRandomPostingsFormat(random);
+      if ("random".equals(TEST_POSTINGSFORMAT)) {
+        format = PostingsFormat.forName("Lucene41");
       } else {
         format = PostingsFormat.forName(TEST_POSTINGSFORMAT);
       }
+      
+      final DocValuesFormat dvFormat;
+      if ("random".equals(TEST_DOCVALUESFORMAT)) {
+        // pick one from SPI
+        String formats[] = DocValuesFormat.availableDocValuesFormats().toArray(new String[0]);
+        dvFormat = DocValuesFormat.forName(formats[random.nextInt(formats.length)]);
+      } else {
+        dvFormat = DocValuesFormat.forName(TEST_DOCVALUESFORMAT);
+      }
+      
       codec = new Lucene42Codec() {       
         @Override
         public PostingsFormat getPostingsFormatForField(String field) {
@@ -161,12 +180,19 @@ final class TestRuleSetupAndRestoreClass
         }
 
         @Override
+        public DocValuesFormat getDocValuesFormatForField(String field) {
+          return dvFormat;
+        }
+
+        @Override
         public String toString() {
-          return super.toString() + ": " + format.toString();
+          return super.toString() + ": " + format.toString() + ", " + dvFormat.toString();
         }
       };
     } else if ("SimpleText".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 9 && !shouldAvoidCodec("SimpleText"))) {
       codec = new SimpleTextCodec();
+    } else if ("CheapBastard".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 8 && !shouldAvoidCodec("CheapBastard"))) {
+      codec = new CheapBastardCodec();
     } else if ("Asserting".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 7 && !shouldAvoidCodec("Asserting"))) {
       codec = new AssertingCodec();
     } else if ("Compressing".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 6 && !shouldAvoidCodec("Compressing"))) {

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java Thu Feb  7 20:48:21 2013
@@ -43,26 +43,26 @@ import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.lucene42.Lucene42Codec;
 import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
-import org.apache.lucene.document.ByteDocValuesField;
-import org.apache.lucene.document.DerefBytesDocValuesField;
+import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.DoubleDocValuesField;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FloatDocValuesField;
-import org.apache.lucene.document.IntDocValuesField;
-import org.apache.lucene.document.LongDocValuesField;
-import org.apache.lucene.document.PackedLongDocValuesField;
-import org.apache.lucene.document.ShortDocValuesField;
-import org.apache.lucene.document.SortedBytesDocValuesField;
-import org.apache.lucene.document.StraightBytesDocValuesField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.CheckIndex;
+import org.apache.lucene.index.CheckIndex.Status.DocValuesStatus;
+import org.apache.lucene.index.CheckIndex.Status.FieldNormStatus;
+import org.apache.lucene.index.CheckIndex.Status.StoredFieldStatus;
+import org.apache.lucene.index.CheckIndex.Status.TermIndexStatus;
+import org.apache.lucene.index.CheckIndex.Status.TermVectorStatus;
 import org.apache.lucene.index.ConcurrentMergeScheduler;
-import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.IndexReader;
@@ -77,15 +77,14 @@ import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.TieredMergePolicy;
 import org.apache.lucene.search.FieldDoc;
-import org.apache.lucene.search.FilteredQuery;
 import org.apache.lucene.search.FilteredQuery.FilterStrategy;
+import org.apache.lucene.search.FilteredQuery;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.CompoundFileDirectory;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.junit.Assert;
-
 import com.carrotsearch.randomizedtesting.RandomizedContext;
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
@@ -226,6 +225,37 @@ public class _TestUtil {
       return indexStatus;
     }
   }
+  
+  /** This runs the CheckIndex tool on the Reader.  If any
+   *  issues are hit, a RuntimeException is thrown */
+  public static void checkReader(AtomicReader reader) throws IOException {
+    checkReader(reader, true);
+  }
+  
+  public static void checkReader(AtomicReader reader, boolean crossCheckTermVectors) throws IOException {
+    ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
+    PrintStream infoStream = new PrintStream(bos, false, "UTF-8");
+
+    FieldNormStatus fieldNormStatus = CheckIndex.testFieldNorms(reader, infoStream);
+    TermIndexStatus termIndexStatus = CheckIndex.testPostings(reader, infoStream);
+    StoredFieldStatus storedFieldStatus = CheckIndex.testStoredFields(reader, infoStream);
+    TermVectorStatus termVectorStatus = CheckIndex.testTermVectors(reader, infoStream, false, crossCheckTermVectors);
+    DocValuesStatus docValuesStatus = CheckIndex.testDocValues(reader, infoStream);
+    
+    if (fieldNormStatus.error != null || 
+      termIndexStatus.error != null ||
+      storedFieldStatus.error != null ||
+      termVectorStatus.error != null ||
+      docValuesStatus.error != null) {
+      System.out.println("CheckReader failed");
+      System.out.println(bos.toString("UTF-8"));
+      throw new RuntimeException("CheckReader failed");
+    } else {
+      if (LuceneTestCase.INFOSTREAM) {
+        System.out.println(bos.toString("UTF-8"));
+      }
+    }
+  }
 
   // NOTE: only works for TMP and LMP!!
   public static void setUseCompoundFile(MergePolicy mp, boolean v) {
@@ -233,6 +263,8 @@ public class _TestUtil {
       ((TieredMergePolicy) mp).setUseCompoundFile(v);
     } else if (mp instanceof LogMergePolicy) {
       ((LogMergePolicy) mp).setUseCompoundFile(v);
+    } else {
+      throw new IllegalArgumentException("cannot set compound file for MergePolicy " + mp);
     }
   }
 
@@ -676,6 +708,24 @@ public class _TestUtil {
       }
     };
   }
+  
+  /** Return a Codec that can read any of the
+   *  default codecs and formats, but always writes in the specified
+   *  format. */
+  public static Codec alwaysDocValuesFormat(final DocValuesFormat format) {
+    // TODO: we really need for docvalues impls etc to announce themselves
+    // (and maybe their params, too) to infostream on flush and merge.
+    // otherwise in a real debugging situation we won't know whats going on!
+    if (LuceneTestCase.VERBOSE) {
+      System.out.println("forcing docvalues format to:" + format);
+    }
+    return new Lucene42Codec() {
+      @Override
+      public DocValuesFormat getDocValuesFormatForField(String field) {
+        return format;
+      }
+    };
+  }
 
   // TODO: generalize all 'test-checks-for-crazy-codecs' to
   // annotations (LUCENE-3489)
@@ -825,50 +875,20 @@ public class _TestUtil {
     for(IndexableField f : doc1.getFields()) {
       final Field field1 = (Field) f;
       final Field field2;
-      final DocValues.Type dvType = field1.fieldType().docValueType();
+      final DocValuesType dvType = field1.fieldType().docValueType();
       if (dvType != null) {
         switch(dvType) {
-        case VAR_INTS:
-          field2 = new PackedLongDocValuesField(field1.name(), field1.numericValue().longValue());
-          break;
-        case FIXED_INTS_8:
-          field2 = new ByteDocValuesField(field1.name(), field1.numericValue().byteValue());
-          break;
-        case FIXED_INTS_16:
-          field2 = new ShortDocValuesField(field1.name(), field1.numericValue().shortValue());
-          break;
-        case FIXED_INTS_32:
-          field2 = new IntDocValuesField(field1.name(), field1.numericValue().intValue());
-          break;
-        case FIXED_INTS_64:
-          field2 = new LongDocValuesField(field1.name(), field1.numericValue().longValue());
-          break;
-        case FLOAT_32:
-          field2 = new FloatDocValuesField(field1.name(), field1.numericValue().floatValue());
-          break;
-        case FLOAT_64:
-          field2 = new DoubleDocValuesField(field1.name(), field1.numericValue().doubleValue());
-          break;
-        case BYTES_FIXED_STRAIGHT:
-          field2 = new StraightBytesDocValuesField(field1.name(), field1.binaryValue(), true);
-          break;
-        case BYTES_VAR_STRAIGHT:
-          field2 = new StraightBytesDocValuesField(field1.name(), field1.binaryValue(), false);
-          break;
-        case BYTES_FIXED_DEREF:
-          field2 = new DerefBytesDocValuesField(field1.name(), field1.binaryValue(), true);
-          break;
-        case BYTES_VAR_DEREF:
-          field2 = new DerefBytesDocValuesField(field1.name(), field1.binaryValue(), false);
-          break;
-        case BYTES_FIXED_SORTED:
-          field2 = new SortedBytesDocValuesField(field1.name(), field1.binaryValue(), true);
-          break;
-        case BYTES_VAR_SORTED:
-          field2 = new SortedBytesDocValuesField(field1.name(), field1.binaryValue(), false);
+          case NUMERIC:
+            field2 = new NumericDocValuesField(field1.name(), field1.numericValue().longValue());
+            break;
+          case BINARY:
+            field2 = new BinaryDocValuesField(field1.name(), field1.binaryValue());
           break;
-        default:
-          throw new IllegalStateException("unknown Type: " + dvType);
+          case SORTED:
+            field2 = new SortedDocValuesField(field1.name(), field1.binaryValue());
+            break;
+          default:
+            throw new IllegalStateException("unknown Type: " + dvType);
         }
       } else {
         field2 = new Field(field1.name(), field1.stringValue(), field1.fieldType());

Modified: lucene/dev/trunk/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec (original)
+++ lucene/dev/trunk/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec Thu Feb  7 20:48:21 2013
@@ -14,8 +14,10 @@
 #  limitations under the License.
 
 org.apache.lucene.codecs.asserting.AssertingCodec
+org.apache.lucene.codecs.cheapbastard.CheapBastardCodec
 org.apache.lucene.codecs.compressing.FastCompressingCodec
 org.apache.lucene.codecs.compressing.FastDecompressionCompressingCodec
 org.apache.lucene.codecs.compressing.HighCompressionCompressingCodec
 org.apache.lucene.codecs.compressing.dummy.DummyCompressingCodec
+org.apache.lucene.codecs.lucene40.Lucene40RWCodec
 org.apache.lucene.codecs.lucene41.Lucene41RWCodec

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java Thu Feb  7 20:48:21 2013
@@ -55,6 +55,7 @@ public class SchemaCodecFactory extends 
         }
         return super.getPostingsFormatForField(field);
       }
+      // TODO: when dv support is added to solr, add it here too
     };
   }
 

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java Thu Feb  7 20:48:21 2013
@@ -36,6 +36,7 @@ import org.apache.lucene.analysis.util.T
 import org.apache.lucene.analysis.util.TokenizerFactory;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.analysis.util.WordlistLoader;
 import org.apache.solr.common.ResourceLoader;
 import org.apache.solr.handler.admin.CoreAdminHandler;
@@ -178,6 +179,7 @@ public class SolrResourceLoader implemen
   void reloadLuceneSPI() {
     // Codecs:
     PostingsFormat.reloadPostingsFormats(this.classLoader);
+    DocValuesFormat.reloadDocValuesFormats(this.classLoader);
     Codec.reloadCodecs(this.classLoader);
     // Analysis:
     CharFilterFactory.reloadCharFilters(this.classLoader);

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java Thu Feb  7 20:48:21 2013
@@ -16,16 +16,17 @@ package org.apache.solr.handler.componen
  * limitations under the License.
  */
 
-import org.apache.lucene.search.FieldCache;
-import org.apache.lucene.util.BytesRef;
-import org.apache.solr.schema.FieldType;
-import org.apache.solr.schema.SchemaField;
-
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.SchemaField;
+
 
 /**
  * FieldFacetStats is a utility to accumulate statistics on a set of values in one field,
@@ -39,7 +40,7 @@ import java.util.Map;
 
 public class FieldFacetStats {
   public final String name;
-  final FieldCache.DocTermsIndex si;
+  final SortedDocValues si;
   final SchemaField facet_sf;
   final SchemaField field_sf;
 
@@ -55,15 +56,15 @@ public class FieldFacetStats {
 
   private final BytesRef tempBR = new BytesRef();
 
-  public FieldFacetStats(String name, FieldCache.DocTermsIndex si, SchemaField field_sf, SchemaField facet_sf, int numStatsTerms) {
+  public FieldFacetStats(String name, SortedDocValues si, SchemaField field_sf, SchemaField facet_sf, int numStatsTerms) {
     this.name = name;
     this.si = si;
     this.field_sf = field_sf;
     this.facet_sf = facet_sf;
     this.numStatsTerms = numStatsTerms;
 
-    startTermIndex = 1;
-    endTermIndex = si.numOrd();
+    startTermIndex = 0;
+    endTermIndex = si.getValueCount();
     nTerms = endTermIndex - startTermIndex;
 
     facetStatsValues = new HashMap<String, StatsValues>();
@@ -79,10 +80,11 @@ public class FieldFacetStats {
 
   BytesRef getTermText(int docID, BytesRef ret) {
     final int ord = si.getOrd(docID);
-    if (ord == 0) {
+    if (ord == -1) {
       return null;
     } else {
-      return si.lookup(ord, ret);
+      si.lookupOrd(ord, ret);
+      return ret;
     }
   }
 
@@ -90,7 +92,14 @@ public class FieldFacetStats {
     int term = si.getOrd(docID);
     int arrIdx = term - startTermIndex;
     if (arrIdx >= 0 && arrIdx < nTerms) {
-      final BytesRef br = si.lookup(term, tempBR);
+      
+      final BytesRef br;
+      if (term == -1) {
+        br = null;
+      } else {
+        br = tempBR;
+        si.lookupOrd(term, tempBR);
+      }
       String key = (br == null)?null:facet_sf.getType().indexedToReadable(br.utf8ToString());
       StatsValues stats = facetStatsValues.get(key);
       if (stats == null) {
@@ -117,7 +126,13 @@ public class FieldFacetStats {
     int term = si.getOrd(docID);
     int arrIdx = term - startTermIndex;
     if (arrIdx >= 0 && arrIdx < nTerms) {
-      final BytesRef br = si.lookup(term, tempBR);
+      final BytesRef br;
+      if (term == -1) {
+        br = null;
+      } else {
+        br = tempBR;
+        si.lookupOrd(term, tempBR);
+      }
       String key = br == null ? null : br.utf8ToString();
       HashMap<String, Integer> statsTermCounts = facetStatsTerms.get(statsTermNum);
       Integer statsTermCount = statsTermCounts.get(key);

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java Thu Feb  7 20:48:21 2013
@@ -23,22 +23,23 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.search.FieldCache;
 import org.apache.lucene.util.BytesRef;
 import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.ShardParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.params.StatsParams;
-import org.apache.solr.common.params.ShardParams;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SimpleOrderedMap;
 import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.UnInvertedField;
 import org.apache.solr.schema.FieldType;
 import org.apache.solr.schema.SchemaField;
 import org.apache.solr.schema.TrieField;
 import org.apache.solr.search.DocIterator;
 import org.apache.solr.search.DocSet;
 import org.apache.solr.search.SolrIndexSearcher;
-import org.apache.solr.request.UnInvertedField;
 
 /**
  * Stats component calculates simple statistics on numeric field values
@@ -240,7 +241,7 @@ class SimpleStats {
   public NamedList<?> getFieldCacheStats(String fieldName, String[] facet ) {
     SchemaField sf = searcher.getSchema().getField(fieldName);
     
-    FieldCache.DocTermsIndex si;
+    SortedDocValues si;
     try {
       si = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), fieldName);
     } 
@@ -248,12 +249,12 @@ class SimpleStats {
       throw new RuntimeException( "failed to open field cache for: "+fieldName, e );
     }
     StatsValues allstats = StatsValuesFactory.createStatsValues(sf);
-    final int nTerms = si.numOrd();
+    final int nTerms = si.getValueCount();
     if ( nTerms <= 0 || docs.size() <= 0 ) return allstats.getStatsValues();
 
     // don't worry about faceting if no documents match...
     List<FieldFacetStats> facetStats = new ArrayList<FieldFacetStats>();
-    FieldCache.DocTermsIndex facetTermsIndex;
+    SortedDocValues facetTermsIndex;
     for( String facetField : facet ) {
       SchemaField fsf = searcher.getSchema().getField(facetField);
 
@@ -261,7 +262,7 @@ class SimpleStats {
         throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
           "Stats can only facet on single-valued fields, not: " + facetField );
       }
-
+      
       try {
         facetTermsIndex = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), facetField);
       }
@@ -276,11 +277,20 @@ class SimpleStats {
     DocIterator iter = docs.iterator();
     while (iter.hasNext()) {
       int docID = iter.nextDoc();
-      BytesRef raw = si.lookup(si.getOrd(docID), tempBR);
-      if( raw.length > 0 ) {
-        allstats.accumulate(raw);
-      } else {
+      int docOrd = si.getOrd(docID);
+      BytesRef raw;
+      if (docOrd == -1) {
         allstats.missing();
+        tempBR.length = 0;
+        raw = tempBR;
+      } else {
+        raw = tempBR;
+        si.lookupOrd(docOrd, tempBR);
+        if( tempBR.length > 0 ) {
+          allstats.accumulate(tempBR);
+        } else {
+          allstats.missing();
+        }
       }
 
       // now update the facets

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java Thu Feb  7 20:48:21 2013
@@ -17,15 +17,21 @@
 
 package org.apache.solr.request;
 
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.*;
+
 import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedDocValuesTermsEnum;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.DocIdSet;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.FieldCache;
 import org.apache.lucene.search.Filter;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CharsRef;
 import org.apache.lucene.util.PriorityQueue;
-import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.UnicodeUtil;
 import org.apache.lucene.util.packed.PackedInts;
 import org.apache.solr.common.SolrException;
@@ -36,10 +42,6 @@ import org.apache.solr.search.DocSet;
 import org.apache.solr.search.SolrIndexSearcher;
 import org.apache.solr.util.BoundedTreeSet;
 
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.*;
-
 
 class PerSegmentSingleValuedFaceting {
 
@@ -145,15 +147,15 @@ class PerSegmentSingleValuedFaceting {
 
 
       if (seg.startTermIndex < seg.endTermIndex) {
-        if (seg.startTermIndex==0) {
+        if (seg.startTermIndex==-1) {
           hasMissingCount=true;
           missingCount += seg.counts[0];
-          seg.pos = 1;
+          seg.pos = 0;
         } else {
           seg.pos = seg.startTermIndex;
         }
         if (seg.pos < seg.endTermIndex) {
-          seg.tenum = seg.si.getTermsEnum();          
+          seg.tenum = new SortedDocValuesTermsEnum(seg.si);
           seg.tenum.seekExact(seg.pos);
           seg.tempBR = seg.tenum.term();
           queue.add(seg);
@@ -224,7 +226,7 @@ class PerSegmentSingleValuedFaceting {
       this.context = context;
     }
     
-    FieldCache.DocTermsIndex si;
+    SortedDocValues si;
     int startTermIndex;
     int endTermIndex;
     int[] counts;
@@ -240,16 +242,16 @@ class PerSegmentSingleValuedFaceting {
 
       if (prefix!=null) {
         BytesRef prefixRef = new BytesRef(prefix);
-        startTermIndex = si.binarySearchLookup(prefixRef, tempBR);
+        startTermIndex = si.lookupTerm(prefixRef);
         if (startTermIndex<0) startTermIndex=-startTermIndex-1;
         prefixRef.append(UnicodeUtil.BIG_TERM);
         // TODO: we could constrain the lower endpoint if we had a binarySearch method that allowed passing start/end
-        endTermIndex = si.binarySearchLookup(prefixRef, tempBR);
+        endTermIndex = si.lookupTerm(prefixRef);
         assert endTermIndex < 0;
         endTermIndex = -endTermIndex-1;
       } else {
-        startTermIndex=0;
-        endTermIndex=si.numOrd();
+        startTermIndex=-1;
+        endTermIndex=si.getValueCount();
       }
 
       final int nTerms=endTermIndex-startTermIndex;
@@ -262,68 +264,19 @@ class PerSegmentSingleValuedFaceting {
 
 
         ////
-        PackedInts.Reader ordReader = si.getDocToOrd();
         int doc;
 
-        final Object arr;
-        if (ordReader.hasArray()) {
-          arr = ordReader.getArray();
-        } else {
-          arr = null;
-        }
-
-        if (arr instanceof int[]) {
-          int[] ords = (int[]) arr;
-          if (prefix==null) {
-            while ((doc = iter.nextDoc()) < DocIdSetIterator.NO_MORE_DOCS) {
-              counts[ords[doc]]++;
-            }
-          } else {
-            while ((doc = iter.nextDoc()) < DocIdSetIterator.NO_MORE_DOCS) {
-              int term = ords[doc];
-              int arrIdx = term-startTermIndex;
-              if (arrIdx>=0 && arrIdx<nTerms) counts[arrIdx]++;
-            }
-          }
-        } else if (arr instanceof short[]) {
-          short[] ords = (short[]) arr;
-          if (prefix==null) {
-            while ((doc = iter.nextDoc()) < DocIdSetIterator.NO_MORE_DOCS) {
-              counts[ords[doc] & 0xffff]++;
-            }
-          } else {
-            while ((doc = iter.nextDoc()) < DocIdSetIterator.NO_MORE_DOCS) {
-              int term = ords[doc] & 0xffff;
-              int arrIdx = term-startTermIndex;
-              if (arrIdx>=0 && arrIdx<nTerms) counts[arrIdx]++;
-            }
-          }
-        } else if (arr instanceof byte[]) {
-          byte[] ords = (byte[]) arr;
-          if (prefix==null) {
-            while ((doc = iter.nextDoc()) < DocIdSetIterator.NO_MORE_DOCS) {
-              counts[ords[doc] & 0xff]++;
-            }
-          } else {
-            while ((doc = iter.nextDoc()) < DocIdSetIterator.NO_MORE_DOCS) {
-              int term = ords[doc] & 0xff;
-              int arrIdx = term-startTermIndex;
-              if (arrIdx>=0 && arrIdx<nTerms) counts[arrIdx]++;
-            }
+        if (prefix==null) {
+          // specialized version when collecting counts for all terms
+          while ((doc = iter.nextDoc()) < DocIdSetIterator.NO_MORE_DOCS) {
+            counts[1+si.getOrd(doc)]++;
           }
         } else {
-          if (prefix==null) {
-            // specialized version when collecting counts for all terms
-            while ((doc = iter.nextDoc()) < DocIdSetIterator.NO_MORE_DOCS) {
-              counts[si.getOrd(doc)]++;
-            }
-          } else {
-            // version that adjusts term numbers because we aren't collecting the full range
-            while ((doc = iter.nextDoc()) < DocIdSetIterator.NO_MORE_DOCS) {
-              int term = si.getOrd(doc);
-              int arrIdx = term-startTermIndex;
-              if (arrIdx>=0 && arrIdx<nTerms) counts[arrIdx]++;
-            }
+          // version that adjusts term numbers because we aren't collecting the full range
+          while ((doc = iter.nextDoc()) < DocIdSetIterator.NO_MORE_DOCS) {
+            int term = si.getOrd(doc);
+            int arrIdx = term-startTermIndex;
+            if (arrIdx>=0 && arrIdx<nTerms) counts[arrIdx]++;
           }
         }
       }

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/SimpleFacets.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/SimpleFacets.java Thu Feb  7 20:48:21 2013
@@ -478,7 +478,9 @@ public class SimpleFacets {
     FieldType ft = searcher.getSchema().getFieldType(fieldName);
     NamedList<Integer> res = new NamedList<Integer>();
 
-    FieldCache.DocTermsIndex si = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), fieldName);
+    SortedDocValues si = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), fieldName);
+
+    final BytesRef br = new BytesRef();
 
     final BytesRef prefixRef;
     if (prefix == null) {
@@ -490,19 +492,17 @@ public class SimpleFacets {
       prefixRef = new BytesRef(prefix);
     }
 
-    final BytesRef br = new BytesRef();
-
     int startTermIndex, endTermIndex;
     if (prefix!=null) {
-      startTermIndex = si.binarySearchLookup(prefixRef, br);
+      startTermIndex = si.lookupTerm(prefixRef);
       if (startTermIndex<0) startTermIndex=-startTermIndex-1;
       prefixRef.append(UnicodeUtil.BIG_TERM);
-      endTermIndex = si.binarySearchLookup(prefixRef, br);
+      endTermIndex = si.lookupTerm(prefixRef);
       assert endTermIndex < 0;
       endTermIndex = -endTermIndex-1;
     } else {
-      startTermIndex=0;
-      endTermIndex=si.numOrd();
+      startTermIndex=-1;
+      endTermIndex=si.getValueCount();
     }
 
     final int nTerms=endTermIndex-startTermIndex;
@@ -516,62 +516,13 @@ public class SimpleFacets {
 
       DocIterator iter = docs.iterator();
 
-      PackedInts.Reader ordReader = si.getDocToOrd();
-      final Object arr;
-      if (ordReader.hasArray()) {
-        arr = ordReader.getArray();
-      } else {
-        arr = null;
-      }
-
-      if (arr instanceof int[]) {
-        int[] ords = (int[]) arr;
-        if (prefix==null) {
-          while (iter.hasNext()) {
-            counts[ords[iter.nextDoc()]]++;
-          }
-        } else {
-          while (iter.hasNext()) {
-            int term = ords[iter.nextDoc()];
-            int arrIdx = term-startTermIndex;
-            if (arrIdx>=0 && arrIdx<nTerms) counts[arrIdx]++;
-          }
-        }
-      } else if (arr instanceof short[]) {
-        short[] ords = (short[]) arr;
-        if (prefix==null) {
-          while (iter.hasNext()) {
-            counts[ords[iter.nextDoc()] & 0xffff]++;
-          }
-        } else {
-          while (iter.hasNext()) {
-            int term = ords[iter.nextDoc()] & 0xffff;
-            int arrIdx = term-startTermIndex;
-            if (arrIdx>=0 && arrIdx<nTerms) counts[arrIdx]++;
-          }
-        }
-      } else if (arr instanceof byte[]) {
-        byte[] ords = (byte[]) arr;
-        if (prefix==null) {
-          while (iter.hasNext()) {
-            counts[ords[iter.nextDoc()] & 0xff]++;
-          }
-        } else {
-          while (iter.hasNext()) {
-            int term = ords[iter.nextDoc()] & 0xff;
-            int arrIdx = term-startTermIndex;
-            if (arrIdx>=0 && arrIdx<nTerms) counts[arrIdx]++;
-          }
-        }
-      } else {
-        while (iter.hasNext()) {
-          int term = si.getOrd(iter.nextDoc());
-          int arrIdx = term-startTermIndex;
-          if (arrIdx>=0 && arrIdx<nTerms) counts[arrIdx]++;
-        }
+      while (iter.hasNext()) {
+        int term = si.getOrd(iter.nextDoc());
+        int arrIdx = term-startTermIndex;
+        if (arrIdx>=0 && arrIdx<nTerms) counts[arrIdx]++;
       }
 
-      if (startTermIndex == 0) {
+      if (startTermIndex == -1) {
         missingCount = counts[0];
       }
 
@@ -587,7 +538,7 @@ public class SimpleFacets {
         LongPriorityQueue queue = new LongPriorityQueue(Math.min(maxsize,1000), maxsize, Long.MIN_VALUE);
 
         int min=mincount-1;  // the smallest value in the top 'N' values
-        for (int i=(startTermIndex==0)?1:0; i<nTerms; i++) {
+        for (int i=(startTermIndex==-1)?1:0; i<nTerms; i++) {
           int c = counts[i];
           if (c>min) {
             // NOTE: we use c>min rather than c>=min as an optimization because we are going in
@@ -614,13 +565,14 @@ public class SimpleFacets {
           long pair = sorted[i];
           int c = (int)(pair >>> 32);
           int tnum = Integer.MAX_VALUE - (int)pair;
-          ft.indexedToReadable(si.lookup(startTermIndex+tnum, br), charsRef);
+          si.lookupOrd(startTermIndex+tnum, br);
+          ft.indexedToReadable(br, charsRef);
           res.add(charsRef.toString(), c);
         }
       
       } else {
         // add results in index order
-        int i=(startTermIndex==0)?1:0;
+        int i=(startTermIndex==-1)?1:0;
         if (mincount<=0) {
           // if mincount<=0, then we won't discard any terms and we know exactly
           // where to start.
@@ -632,7 +584,8 @@ public class SimpleFacets {
           int c = counts[i];
           if (c<mincount || --off>=0) continue;
           if (--lim<0) break;
-          ft.indexedToReadable(si.lookup(startTermIndex+i, br), charsRef);
+          si.lookupOrd(startTermIndex+i, br);
+          ft.indexedToReadable(br, charsRef);
           res.add(charsRef.toString(), c);
         }
       }

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/UnInvertedField.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/UnInvertedField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/UnInvertedField.java Thu Feb  7 20:48:21 2013
@@ -17,37 +17,36 @@
 
 package org.apache.solr.request;
 
-import org.apache.lucene.search.FieldCache;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
 import org.apache.lucene.index.DocTermOrds;
+import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.FieldCache;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.UnicodeUtil;
+import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.FacetParams;
 import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.SolrException;
 import org.apache.solr.core.SolrCore;
-
+import org.apache.solr.handler.component.FieldFacetStats;
+import org.apache.solr.handler.component.StatsValues;
+import org.apache.solr.handler.component.StatsValuesFactory;
 import org.apache.solr.schema.FieldType;
 import org.apache.solr.schema.SchemaField;
 import org.apache.solr.schema.TrieField;
 import org.apache.solr.search.*;
 import org.apache.solr.util.LongPriorityQueue;
 import org.apache.solr.util.PrimUtils;
-import org.apache.solr.handler.component.StatsValues;
-import org.apache.solr.handler.component.StatsValuesFactory;
-import org.apache.solr.handler.component.FieldFacetStats;
-import org.apache.lucene.util.CharsRef;
-import org.apache.lucene.util.OpenBitSet;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.UnicodeUtil;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-import java.util.concurrent.atomic.AtomicLong;
 
 /**
  *
@@ -481,7 +480,7 @@ public class UnInvertedField extends Doc
     int i = 0;
     final FieldFacetStats[] finfo = new FieldFacetStats[facet.length];
     //Initialize facetstats, if facets have been passed in
-    FieldCache.DocTermsIndex si;
+    SortedDocValues si;
     for (String f : facet) {
       SchemaField facet_sf = searcher.getSchema().getField(f);
       try {

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/BoolField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/BoolField.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/BoolField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/BoolField.java Thu Feb  7 20:48:21 2013
@@ -17,29 +17,30 @@
 
 package org.apache.solr.schema;
 
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Map;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.GeneralField;
+import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.StorableField;
-import org.apache.lucene.search.FieldCache;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.docvalues.BoolDocValues;
 import org.apache.lucene.queries.function.valuesource.OrdFieldSource;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CharsRef;
 import org.apache.lucene.util.mutable.MutableValue;
 import org.apache.lucene.util.mutable.MutableValueBool;
-import org.apache.solr.search.QParser;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.solr.response.TextResponseWriter;
 import org.apache.solr.analysis.SolrAnalyzer;
-
-import java.util.Map;
-import java.io.Reader;
-import java.io.IOException;
+import org.apache.solr.response.TextResponseWriter;
+import org.apache.solr.search.QParser;
 /**
  *
  */
@@ -167,14 +168,14 @@ class BoolFieldSource extends ValueSourc
 
   @Override
   public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
-    final FieldCache.DocTermsIndex sindex = FieldCache.DEFAULT.getTermsIndex(readerContext.reader(), field);
+    final SortedDocValues sindex = FieldCache.DEFAULT.getTermsIndex(readerContext.reader(), field);
 
     // figure out what ord maps to true
-    int nord = sindex.numOrd();
+    int nord = sindex.getValueCount();
     BytesRef br = new BytesRef();
     int tord = -1;
-    for (int i=1; i<nord; i++) {
-      sindex.lookup(i, br);
+    for (int i=0; i<nord; i++) {
+      sindex.lookupOrd(i, br);
       if (br.length==1 && br.bytes[br.offset]=='T') {
         tord = i;
         break;
@@ -191,7 +192,7 @@ class BoolFieldSource extends ValueSourc
 
       @Override
       public boolean exists(int doc) {
-        return sindex.getOrd(doc) != 0;
+        return sindex.getOrd(doc) != -1;
       }
 
       @Override
@@ -208,7 +209,7 @@ class BoolFieldSource extends ValueSourc
           public void fillValue(int doc) {
             int ord = sindex.getOrd(doc);
             mval.value = (ord == trueOrd);
-            mval.exists = (ord != 0);
+            mval.exists = (ord != -1);
           }
         };
       }

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/DateField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/DateField.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/DateField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/DateField.java Thu Feb  7 20:48:21 2013
@@ -498,21 +498,22 @@ class DateFieldSource extends FieldCache
       @Override
       public String strVal(int doc) {
         int ord=termsIndex.getOrd(doc);
-        if (ord == 0) {
+        if (ord == -1) {
           return null;
         } else {
-          final BytesRef br = termsIndex.lookup(ord, spare);
-          return ft.indexedToReadable(br, spareChars).toString();
+          termsIndex.lookupOrd(ord, spare);
+          return ft.indexedToReadable(spare, spareChars).toString();
         }
       }
 
       @Override
       public Object objectVal(int doc) {
         int ord=termsIndex.getOrd(doc);
-        if (ord == 0) {
+        if (ord == -1) {
           return null;
         } else {
-          final BytesRef br = termsIndex.lookup(ord, new BytesRef());
+          final BytesRef br = new BytesRef();
+          termsIndex.lookupOrd(ord, br);
           return ft.toObject(null, br);
         }
       }

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java Thu Feb  7 20:48:21 2013
@@ -149,7 +149,12 @@ class SortableDoubleFieldSource extends 
       @Override
       public double doubleVal(int doc) {
         int ord=termsIndex.getOrd(doc);
-        return ord==0 ? def  : NumberUtils.SortableStr2double(termsIndex.lookup(ord, spare));
+        if (ord == -1) {
+          return def;
+        } else {
+          termsIndex.lookupOrd(ord, spare);
+          return NumberUtils.SortableStr2double(spare);
+        }
       }
 
       @Override
@@ -160,7 +165,12 @@ class SortableDoubleFieldSource extends 
       @Override
       public Object objectVal(int doc) {
         int ord=termsIndex.getOrd(doc);
-        return ord==0 ? null  : NumberUtils.SortableStr2double(termsIndex.lookup(ord, spare));
+        if (ord==-1) {
+          return null;
+        } else {
+          termsIndex.lookupOrd(ord, spare);
+          return NumberUtils.SortableStr2double(spare);
+        }
       }
 
       @Override
@@ -181,11 +191,12 @@ class SortableDoubleFieldSource extends 
           @Override
           public void fillValue(int doc) {
             int ord=termsIndex.getOrd(doc);
-            if (ord == 0) {
+            if (ord == -1) {
               mval.value = def;
               mval.exists = false;
             } else {
-              mval.value = NumberUtils.SortableStr2double(termsIndex.lookup(ord, spare));
+              termsIndex.lookupOrd(ord, spare);
+              mval.value = NumberUtils.SortableStr2double(spare);
               mval.exists = true;
             }
           }

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java Thu Feb  7 20:48:21 2013
@@ -138,7 +138,12 @@ class SortableFloatFieldSource extends F
       @Override
       public float floatVal(int doc) {
         int ord=termsIndex.getOrd(doc);
-        return ord==0 ? def  : NumberUtils.SortableStr2float(termsIndex.lookup(ord, spare));
+        if (ord==-1) {
+          return def;
+        } else {
+          termsIndex.lookupOrd(ord, spare);
+          return NumberUtils.SortableStr2float(spare);
+        }
       }
 
       @Override
@@ -169,7 +174,12 @@ class SortableFloatFieldSource extends F
       @Override
       public Object objectVal(int doc) {
         int ord=termsIndex.getOrd(doc);
-        return ord==0 ? null  : NumberUtils.SortableStr2float(termsIndex.lookup(ord, spare));
+        if (ord==-1) {
+          return null;
+        } else {
+          termsIndex.lookupOrd(ord, spare);
+          return NumberUtils.SortableStr2float(spare);
+        }
       }
 
       @Override
@@ -185,11 +195,12 @@ class SortableFloatFieldSource extends F
           @Override
           public void fillValue(int doc) {
             int ord=termsIndex.getOrd(doc);
-            if (ord == 0) {
+            if (ord == -1) {
               mval.value = def;
               mval.exists = false;
             } else {
-              mval.value = NumberUtils.SortableStr2float(termsIndex.lookup(ord, spare));
+              termsIndex.lookupOrd(ord, spare);
+              mval.value = NumberUtils.SortableStr2float(spare);
               mval.exists = true;
             }
           }

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableIntField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableIntField.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableIntField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableIntField.java Thu Feb  7 20:48:21 2013
@@ -145,7 +145,12 @@ class SortableIntFieldSource extends Fie
       @Override
       public int intVal(int doc) {
         int ord=termsIndex.getOrd(doc);
-        return ord==0 ? def  : NumberUtils.SortableStr2int(termsIndex.lookup(ord, spare),0,3);
+        if (ord==-1) {
+          return def;
+        } else {
+          termsIndex.lookupOrd(ord, spare);
+          return NumberUtils.SortableStr2int(spare,0,3);
+        }
       }
 
       @Override
@@ -171,7 +176,12 @@ class SortableIntFieldSource extends Fie
       @Override
       public Object objectVal(int doc) {
         int ord=termsIndex.getOrd(doc);
-        return ord==0 ? null  : NumberUtils.SortableStr2int(termsIndex.lookup(ord, spare));
+        if (ord==-1) {
+          return null;
+        } else {
+          termsIndex.lookupOrd(ord, spare);
+          return NumberUtils.SortableStr2int(spare);
+        }
       }
 
       @Override
@@ -187,11 +197,12 @@ class SortableIntFieldSource extends Fie
           @Override
           public void fillValue(int doc) {
             int ord=termsIndex.getOrd(doc);
-            if (ord == 0) {
+            if (ord == -1) {
               mval.value = def;
               mval.exists = false;
             } else {
-              mval.value = NumberUtils.SortableStr2int(termsIndex.lookup(ord, spare),0,3);
+              termsIndex.lookupOrd(ord, spare);
+              mval.value = NumberUtils.SortableStr2int(spare,0,3);
               mval.exists = true;
             }
           }

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableLongField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableLongField.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableLongField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableLongField.java Thu Feb  7 20:48:21 2013
@@ -148,7 +148,12 @@ class SortableLongFieldSource extends Fi
       @Override
       public long longVal(int doc) {
         int ord=termsIndex.getOrd(doc);
-        return ord==0 ? def  : NumberUtils.SortableStr2long(termsIndex.lookup(ord, spare),0,5);
+        if (ord==-1) {
+          return def;
+        } else {
+          termsIndex.lookupOrd(ord, spare);
+          return NumberUtils.SortableStr2long(spare,0,5);
+        }
       }
 
       @Override
@@ -164,7 +169,12 @@ class SortableLongFieldSource extends Fi
       @Override
       public Object objectVal(int doc) {
         int ord=termsIndex.getOrd(doc);
-        return ord==0 ? null  : NumberUtils.SortableStr2long(termsIndex.lookup(ord, spare));
+        if (ord==-1) {
+          return null;
+        } else {
+          termsIndex.lookupOrd(ord, spare);
+          return NumberUtils.SortableStr2long(spare);
+        }
       }
 
       @Override
@@ -185,11 +195,12 @@ class SortableLongFieldSource extends Fi
           @Override
           public void fillValue(int doc) {
             int ord=termsIndex.getOrd(doc);
-            if (ord == 0) {
+            if (ord == -1) {
               mval.value = def;
               mval.exists = false;
             } else {
-              mval.value = NumberUtils.SortableStr2long(termsIndex.lookup(ord, spare),0,5);
+              termsIndex.lookupOrd(ord, spare);
+              mval.value = NumberUtils.SortableStr2long(spare,0,5);
               mval.exists = true;
             }
           }

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/StrFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/StrFieldSource.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/StrFieldSource.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/StrFieldSource.java Thu Feb  7 20:48:21 2013
@@ -52,7 +52,7 @@ public class StrFieldSource extends Fiel
 
       @Override
       public int numOrd() {
-        return termsIndex.numOrd();
+        return termsIndex.getValueCount();
       }
 
       @Override