You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/01/02 13:02:33 UTC

svn commit: r1649007 [10/10] - in /lucene/dev/branches/lucene6005/lucene: analysis/common/src/test/org/apache/lucene/analysis/core/ analysis/common/src/test/org/apache/lucene/collation/ benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ bench...

Modified: lucene/dev/branches/lucene6005/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/lucene6005/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Fri Jan  2 12:02:31 2015
@@ -31,6 +31,7 @@ import org.apache.lucene.analysis.tokena
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
+import org.apache.lucene.document.FieldTypes;
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.DocsAndPositionsEnum;
@@ -60,12 +61,12 @@ import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.ByteBlockPool;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefHash;
 import org.apache.lucene.util.BytesRefHash.DirectBytesStartArray;
+import org.apache.lucene.util.BytesRefHash;
 import org.apache.lucene.util.Counter;
-import org.apache.lucene.util.IntBlockPool;
 import org.apache.lucene.util.IntBlockPool.SliceReader;
 import org.apache.lucene.util.IntBlockPool.SliceWriter;
+import org.apache.lucene.util.IntBlockPool;
 import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.RecyclingByteBlockAllocator;
 import org.apache.lucene.util.RecyclingIntBlockAllocator;
@@ -756,6 +757,12 @@ public class MemoryIndex {
     }
 
     @Override
+    public FieldTypes getFieldTypes() {
+      // nocommit todo
+      return null;
+    }
+
+    @Override
     public Bits getLiveDocs() {
       return null;
     }

Modified: lucene/dev/branches/lucene6005/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java (original)
+++ lucene/dev/branches/lucene6005/lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java Fri Jan  2 12:02:31 2015
@@ -82,7 +82,6 @@ interface FieldCache {
   /** Expert: The cache used internally by sorting and range query classes. */
   public static FieldCache DEFAULT = new FieldCacheImpl();
 
-  // nocommit rename
   public static final Parser DOCUMENT_INT_PARSER = new Parser() {
     @Override
     public long parseValue(BytesRef term) {
@@ -100,7 +99,6 @@ interface FieldCache {
     }
   };
 
-  // nocommit rename
   public static final Parser DOCUMENT_FLOAT_PARSER = new Parser() {
     @Override
     public long parseValue(BytesRef term) {
@@ -118,7 +116,6 @@ interface FieldCache {
     }
   };
 
-  // nocommit rename
   public static final Parser DOCUMENT_LONG_PARSER = new Parser() {
     @Override
     public long parseValue(BytesRef term) {
@@ -136,9 +133,6 @@ interface FieldCache {
     }
   };
 
-  // nocommit half floats?
-  
-  // nocommit rename
   public static final Parser DOCUMENT_DOUBLE_PARSER = new Parser() {
     @Override
     public long parseValue(BytesRef term) {

Modified: lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCache.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCache.java (original)
+++ lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCache.java Fri Jan  2 12:02:31 2015
@@ -613,16 +613,16 @@ public class TestFieldCache extends Luce
     Directory dir = newDirectory();
     RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
     Document doc = iw.newDocument();
-    doc.addStored("bogusbytes", "bogus");
-    doc.addStored("bogusshorts", "bogus");
-    doc.addStored("bogusints", "bogus");
-    doc.addStored("boguslongs", "bogus");
-    doc.addStored("bogusfloats", "bogus");
-    doc.addStored("bogusdoubles", "bogus");
-    doc.addStored("bogusterms", "bogus");
-    doc.addStored("bogustermsindex", "bogus");
-    doc.addStored("bogusmultivalued", "bogus");
-    doc.addStored("bogusbits", "bogus");
+    doc.addStoredString("bogusbytes", "bogus");
+    doc.addStoredString("bogusshorts", "bogus");
+    doc.addStoredString("bogusints", "bogus");
+    doc.addStoredString("boguslongs", "bogus");
+    doc.addStoredString("bogusfloats", "bogus");
+    doc.addStoredString("bogusdoubles", "bogus");
+    doc.addStoredString("bogusterms", "bogus");
+    doc.addStoredString("bogustermsindex", "bogus");
+    doc.addStoredString("bogusmultivalued", "bogus");
+    doc.addStoredString("bogusbits", "bogus");
     iw.addDocument(doc);
     DirectoryReader ir = iw.getReader();
     iw.close();

Modified: lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java Fri Jan  2 12:02:31 2015
@@ -67,11 +67,6 @@ final class IDVersionPostingsWriter exte
     if (fieldInfo.getIndexOptions() != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
       throw new IllegalArgumentException("field \"" + fieldInfo.name + "\" must be indexed using IndexOptions.DOCS_AND_FREQS_AND_POSITIONS (got: " + fieldInfo.getIndexOptions() + ")");
     }
-    // LUCENE-5693: because CheckIndex cross-checks term vectors with postings even for deleted docs, and because our PF only indexes the
-    // non-deleted documents on flush, CheckIndex will see this as corruption:
-    if (fieldInfo.hasVectors()) {
-      throw new IllegalArgumentException("field cannot index term vectors: CheckIndex will report this as index corruption");
-    }
     lastState = emptyState;
     return 0;
   }
@@ -83,10 +78,6 @@ final class IDVersionPostingsWriter exte
 
   @Override
   public void startDoc(int docID, int termDocFreq) throws IOException {
-    // TODO: LUCENE-5693: we don't need this check if we fix IW to not send deleted docs to us on flush:
-    if (liveDocs != null && liveDocs.get(docID) == false) {
-      return;
-    }
     if (lastDocID != -1) {
       throw new IllegalArgumentException("term appears in more than one document");
     }

Modified: lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java Fri Jan  2 12:02:31 2015
@@ -165,6 +165,8 @@ public final class VersionBlockTreeTerms
   private final List<FieldMetaData> fields = new ArrayList<>();
   // private final String segment;
 
+  // TODO: add auto-prefix support?
+
   /** Create a new writer.  The number of items (terms or
    *  sub-blocks) per block will aim to be between
    *  minItemsPerBlock and maxItemsPerBlock, though in some
@@ -176,18 +178,7 @@ public final class VersionBlockTreeTerms
                                      int maxItemsInBlock)
     throws IOException
   {
-    if (minItemsInBlock <= 1) {
-      throw new IllegalArgumentException("minItemsInBlock must be >= 2; got " + minItemsInBlock);
-    }
-    if (maxItemsInBlock <= 0) {
-      throw new IllegalArgumentException("maxItemsInBlock must be >= 1; got " + maxItemsInBlock);
-    }
-    if (minItemsInBlock > maxItemsInBlock) {
-      throw new IllegalArgumentException("maxItemsInBlock must be >= minItemsInBlock; got maxItemsInBlock=" + maxItemsInBlock + " minItemsInBlock=" + minItemsInBlock);
-    }
-    if (2*(minItemsInBlock-1) > maxItemsInBlock) {
-      throw new IllegalArgumentException("maxItemsInBlock must be at least 2*(minItemsInBlock-1); got maxItemsInBlock=" + maxItemsInBlock + " minItemsInBlock=" + minItemsInBlock);
-    }
+    BlockTreeTermsWriter.validateSettings(minItemsInBlock, maxItemsInBlock);
 
     maxDoc = state.segmentInfo.getDocCount();
 
@@ -745,8 +736,7 @@ public final class VersionBlockTreeTerms
     public void write(BytesRef text, TermsEnum termsEnum) throws IOException {
 
       BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen);
-      // TODO: LUCENE-5693: we don't need this check if we fix IW to not send deleted docs to us on flush:
-      if (state != null && ((IDVersionPostingsWriter) postingsWriter).lastDocID != -1) {
+      if (state != null) {
         assert state.docFreq != 0;
         assert fieldInfo.getIndexOptions() == IndexOptions.DOCS || state.totalTermFreq >= state.docFreq: "postingsWriter=" + postingsWriter;
         pushTerm(text);

Modified: lucene/dev/branches/lucene6005/lucene/sandbox/src/test/org/apache/lucene/codecs/idversion/TestIDVersionPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/sandbox/src/test/org/apache/lucene/codecs/idversion/TestIDVersionPostingsFormat.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/sandbox/src/test/org/apache/lucene/codecs/idversion/TestIDVersionPostingsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/sandbox/src/test/org/apache/lucene/codecs/idversion/TestIDVersionPostingsFormat.java Fri Jan  2 12:02:31 2015
@@ -199,6 +199,11 @@ public class TestIDVersionPostingsFormat
     FieldTypes fieldTypes = w.getFieldTypes();
     fieldTypes.disableHighlighting("id");
     fieldTypes.disableExistsFilters();
+    if (random().nextBoolean()) {
+      fieldTypes.enableTermVectors("id");
+      fieldTypes.enableTermVectorPositions("id");
+      fieldTypes.enableTermVectorOffsets("id");
+    }
     //IndexWriter w = new IndexWriter(dir, iwc);
     int numDocs = atLeast(1000);
     Map<String,Long> idValues = new HashMap<String,Long>();
@@ -567,31 +572,6 @@ public class TestIDVersionPostingsFormat
     w.close();
     dir.close();
   }
-
-  // LUCENE-5693: because CheckIndex cross-checks term vectors with postings even for deleted docs, and because our PF only indexes the
-  // non-deleted documents on flush, CheckIndex will see this as corruption:
-  public void testCannotIndexTermVectors() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
-    iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
-    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
-    FieldTypes fieldTypes = w.getFieldTypes();
-    fieldTypes.disableHighlighting("id");
-    fieldTypes.disableExistsFilters();
-    fieldTypes.enableTermVectors("id");
-    Document doc = w.newDocument();
-    doc.addLargeText("id", makeIDTokenStream("foo", 17));
-    try {
-      w.addDocument(doc);
-      w.commit();
-      fail("didn't hit expected exception");
-    } catch (IllegalArgumentException iae) {
-      // expected
-      // iae.printStackTrace(System.out);
-    }
-    w.close();
-    dir.close();
-  }
 
   public void testMoreThanOnceInSingleDoc() throws IOException {
     Directory dir = newDirectory();

Modified: lucene/dev/branches/lucene6005/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java (original)
+++ lucene/dev/branches/lucene6005/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java Fri Jan  2 12:02:31 2015
@@ -488,7 +488,7 @@ public class TestTermAutomatonQuery exte
       }
       String contents = sb.toString();
       doc.addLargeText("field", contents);
-      doc.addStored("id", ""+i);
+      doc.addStoredString("id", ""+i);
       if (VERBOSE) {
         System.out.println("  doc " + i + " -> " + contents);
       }

Modified: lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java (original)
+++ lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java Fri Jan  2 12:02:31 2015
@@ -129,7 +129,7 @@ public class SpatialExample extends Luce
       //store it too; the format is up to you
       //  (assume point in this example)
       Point pt = (Point) shape;
-      doc.addStored(strategy.getFieldName() + "_stored", pt.getX()+" "+pt.getY());
+      doc.addStoredString(strategy.getFieldName() + "_stored", pt.getX()+" "+pt.getY());
     }
 
     return doc;

Modified: lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java Fri Jan  2 12:02:31 2015
@@ -96,7 +96,7 @@ public abstract class StrategyTestCase e
       if (shape != null) {
         strategy.addFields(document, shape);
         if (storeShape) {//just for diagnostics
-          document.addStored(strategy.getFieldName() + "_stored", shape.toString());
+          document.addStoredString(strategy.getFieldName() + "_stored", shape.toString());
         }
       }
       documents.add(document);
@@ -197,7 +197,7 @@ public abstract class StrategyTestCase e
     if (shape != null) {
       strategy.addFields(doc, shape);
       if (storeShape) {
-        doc.addStored(strategy.getFieldName() + "_stored", shape.toString());
+        doc.addStoredString(strategy.getFieldName() + "_stored", shape.toString());
       }
     }
     return doc;

Modified: lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/RandomSpatialOpFuzzyPrefixTreeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/RandomSpatialOpFuzzyPrefixTreeTest.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/RandomSpatialOpFuzzyPrefixTreeTest.java (original)
+++ lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/RandomSpatialOpFuzzyPrefixTreeTest.java Fri Jan  2 12:02:31 2015
@@ -214,7 +214,7 @@ public class RandomSpatialOpFuzzyPrefixT
         strategy.addFields(doc, shapei);
       }
       if (storeShape)//just for diagnostics
-        doc.addStored(strategy.getFieldName() + "_stored", shape.toString());
+        doc.addStoredString(strategy.getFieldName() + "_stored", shape.toString());
     }
     return doc;
   }

Modified: lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java (original)
+++ lucene/dev/branches/lucene6005/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java Fri Jan  2 12:02:31 2015
@@ -43,7 +43,7 @@ public class TestTermQueryPrefixGridStra
 
     fieldTypes.setMultiValued(prefixGridStrategy.getFieldName());
     prefixGridStrategy.addFields(losAngeles, point);
-    losAngeles.addStored(prefixGridStrategy.getFieldName() + "_stored", point.toString());//just for diagnostics
+    losAngeles.addStoredString(prefixGridStrategy.getFieldName() + "_stored", point.toString());//just for diagnostics
 
     addDocumentsAndCommit(Arrays.asList(losAngeles));
 

Modified: lucene/dev/branches/lucene6005/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java (original)
+++ lucene/dev/branches/lucene6005/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java Fri Jan  2 12:02:31 2015
@@ -75,7 +75,7 @@ public class DocumentDictionaryTest exte
       if (requiresPayload || usually()) {
         // usually have valid payload field in document
         if (usually()) {
-          doc.addStored(PAYLOAD_FIELD_NAME, new BytesRef("payload_" + i));
+          doc.addStoredBinary(PAYLOAD_FIELD_NAME, new BytesRef("payload_" + i));
         } else if (requiresPayload) {
           invalidDoc = true;
         }
@@ -84,7 +84,7 @@ public class DocumentDictionaryTest exte
       if (requiresContexts || usually()) {
         if (usually()) {
           for (int j = 0; j < atLeast(2); j++) {
-            doc.addStored(CONTEXT_FIELD_NAME, new BytesRef("context_" + i + "_"+ j));
+            doc.addStoredBinary(CONTEXT_FIELD_NAME, new BytesRef("context_" + i + "_"+ j));
           }
         }
         // we should allow entries without context
@@ -340,7 +340,7 @@ public class DocumentDictionaryTest exte
       BytesRef term;
 
       payloadValue = new BytesRef("payload_" + i);
-      doc.addStored(PAYLOAD_FIELD_NAME, payloadValue);
+      doc.addStoredBinary(PAYLOAD_FIELD_NAME, payloadValue);
 
       if (usually()) {
         numericValue = 100 + i;
@@ -350,14 +350,14 @@ public class DocumentDictionaryTest exte
       int numContexts = atLeast(1);
       for (int j=0; j<numContexts; j++) {
         BytesRef contextValue = new BytesRef("context_" + i + "_" + j);
-        doc.addStored(CONTEXT_FIELD_NAME, contextValue);
+        doc.addStoredBinary(CONTEXT_FIELD_NAME, contextValue);
         contextValues.add(contextValue);
       }
 
       int numSuggestions = atLeast(2);
       for (int j=0; j<numSuggestions; j++) {
         term = new BytesRef("field_" + i + "_" + j);
-        doc.addStored(FIELD_NAME, term);
+        doc.addStoredBinary(FIELD_NAME, term);
 
         Suggestion suggestionValue = new Suggestion();
         suggestionValue.payload = payloadValue;

Modified: lucene/dev/branches/lucene6005/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java (original)
+++ lucene/dev/branches/lucene6005/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java Fri Jan  2 12:02:31 2015
@@ -61,13 +61,13 @@ public class DocumentValueSourceDictiona
     for(int i = 0; i < ndocs ; i++) {
       Document doc = writer.newDocument();
       doc.addLargeText(FIELD_NAME, "field_" + i);
-      doc.addStored(PAYLOAD_FIELD_NAME, new BytesRef("payload_" + i));
+      doc.addStoredBinary(PAYLOAD_FIELD_NAME, new BytesRef("payload_" + i));
       doc.addInt(WEIGHT_FIELD_NAME_1, 10 + i);
       doc.addInt(WEIGHT_FIELD_NAME_2, 20 + i);
       doc.addInt(WEIGHT_FIELD_NAME_3, 30 + i);
-      doc.addStored(CONTEXTS_FIELD_NAME, new BytesRef("ctx_"  + i + "_0"));
+      doc.addStoredBinary(CONTEXTS_FIELD_NAME, new BytesRef("ctx_"  + i + "_0"));
       for(int j = 1; j < atLeast(3); j++) {
-        doc.addStored(CONTEXTS_FIELD_NAME, new BytesRef("ctx_" + i + "_" + j));
+        doc.addStoredBinary(CONTEXTS_FIELD_NAME, new BytesRef("ctx_" + i + "_" + j));
       }
       docs.put("field_" + i, doc);
     }

Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java Fri Jan  2 12:02:31 2015
@@ -108,7 +108,7 @@ public final class RAMOnlyPostingsFormat
 
   static class RAMField extends Terms implements Accountable {
     final String field;
-    final SortedMap<String,RAMTerm> termToDocs = new TreeMap<>();
+    final SortedMap<BytesRef,RAMTerm> termToDocs = new TreeMap<>();
     long sumTotalTermFreq;
     long sumDocFreq;
     int docCount;
@@ -175,11 +175,11 @@ public final class RAMOnlyPostingsFormat
   }
 
   static class RAMTerm implements Accountable {
-    final String term;
+    final BytesRef term;
     long totalTermFreq;
     final List<RAMDoc> docs = new ArrayList<>();
-    public RAMTerm(String term) {
-      this.term = term;
+    public RAMTerm(BytesRef term) {
+      this.term = BytesRef.deepCopyOf(term);
     }
 
     @Override
@@ -362,8 +362,7 @@ public final class RAMOnlyPostingsFormat
     }
       
     public RAMPostingsWriterImpl startTerm(BytesRef text) {
-      final String term = text.utf8ToString();
-      current = new RAMTerm(term);
+      current = new RAMTerm(text);
       postingsWriter.reset(current);
       return postingsWriter;
     }
@@ -417,8 +416,8 @@ public final class RAMOnlyPostingsFormat
   }
 
   static class RAMTermsEnum extends TermsEnum {
-    Iterator<String> it;
-    String current;
+    Iterator<BytesRef> it;
+    BytesRef current;
     private final RAMField ramField;
 
     public RAMTermsEnum(RAMField field) {
@@ -436,7 +435,7 @@ public final class RAMOnlyPostingsFormat
       }
       if (it.hasNext()) {
         current = it.next();
-        return new BytesRef(current);
+        return current;
       } else {
         return null;
       }
@@ -444,7 +443,7 @@ public final class RAMOnlyPostingsFormat
 
     @Override
     public SeekStatus seekCeil(BytesRef term) {
-      current = term.utf8ToString();
+      current = term;
       it = null;
       if (ramField.termToDocs.containsKey(current)) {
         return SeekStatus.FOUND;
@@ -469,8 +468,7 @@ public final class RAMOnlyPostingsFormat
 
     @Override
     public BytesRef term() {
-      // TODO: reuse BytesRef
-      return new BytesRef(current);
+      return current;
     }
 
     @Override

Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/document/LowSchemaField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/document/LowSchemaField.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/document/LowSchemaField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/document/LowSchemaField.java Fri Jan  2 12:02:31 2015
@@ -134,8 +134,6 @@ public class LowSchemaField implements I
     return this;
   }
 
-  // nocommit need test trying to do index field with tokenized=false, and Reader value
-
   @Override
   public TokenStream tokenStream(TokenStream reuse) throws IOException {
     if (tokenStream != null) {

Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java Fri Jan  2 12:02:31 2015
@@ -729,7 +729,7 @@ public abstract class BaseCompoundFormat
 
   @Override
   protected void addRandomFields(Document doc) {
-    doc.addStored("foobar", TestUtil.randomSimpleString(random()));
+    doc.addStoredString("foobar", TestUtil.randomSimpleString(random()));
   }
 
   @Override

Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java Fri Jan  2 12:02:31 2015
@@ -1280,7 +1280,7 @@ public abstract class BaseDocValuesForma
       }
       Arrays.sort(valueArray);
       for (int j = 0; j < valueCount; j++) {
-        doc.addStored("stored", Long.toString(valueArray[j]));
+        doc.addStoredString("stored", Long.toString(valueArray[j]));
       }
       writer.addDocument(doc);
       if (random().nextInt(31) == 0) {
@@ -1375,7 +1375,7 @@ public abstract class BaseDocValuesForma
       }
       byte buffer[] = new byte[length];
       random().nextBytes(buffer);
-      doc.addStored("stored", buffer);
+      doc.addStoredBinary("stored", buffer);
       doc.addBinary("dv", buffer);
       writer.addDocument(doc);
       if (random().nextInt(31) == 0) {
@@ -1452,7 +1452,7 @@ public abstract class BaseDocValuesForma
       }
       byte buffer[] = new byte[length];
       random().nextBytes(buffer);
-      doc.addStored("stored", buffer);
+      doc.addStoredBinary("stored", buffer);
       doc.addBinary("dv", buffer);
       writer.addDocument(doc);
       if (random().nextInt(31) == 0) {
@@ -2003,7 +2003,7 @@ public abstract class BaseDocValuesForma
       
       // add ordered to the stored field
       for (String v : values) {
-        doc.addStored("stored", v);
+        doc.addStoredString("stored", v);
       }
 
       // add in any order to the dv field
@@ -2424,11 +2424,11 @@ public abstract class BaseDocValuesForma
       int length = TestUtil.nextInt(random(), 0, 8);
       byte buffer[] = new byte[length];
       random().nextBytes(buffer);
-      doc.addStored("storedBin", buffer);
+      doc.addStoredBinary("storedBin", buffer);
       doc.addBinary("dvBin", buffer);
       doc.addBinary("dvSorted", buffer);
       long numericValue = random().nextLong();
-      doc.addStored("storedNum", Long.toString(numericValue));
+      doc.addStoredString("storedNum", Long.toString(numericValue));
       doc.addLong("dvNum", numericValue);
       writer.addDocument(doc);
       if (random().nextInt(31) == 0) {
@@ -2512,13 +2512,13 @@ public abstract class BaseDocValuesForma
       byte buffer[] = new byte[length];
       random().nextBytes(buffer);
       if (random().nextInt(4) > 0) {
-        doc.addStored("storedBin", buffer);
+        doc.addStoredBinary("storedBin", buffer);
         doc.addBinary("dvBin", buffer);
         doc.addBinary("dvSorted", buffer);
       }
       long numericValue = random().nextLong();
       if (random().nextInt(4) > 0) {
-        doc.addStored("storedNum", Long.toString(numericValue));
+        doc.addStoredString("storedNum", Long.toString(numericValue));
         doc.addLong("dvNum", numericValue);
       }
       int numSortedSetFields = random().nextInt(3);
@@ -2528,7 +2528,7 @@ public abstract class BaseDocValuesForma
       }
       for (String v : values) {
         doc.addBinary("dvSortedSet", new BytesRef(v));
-        doc.addStored("storedSortedSet", v);
+        doc.addStoredString("storedSortedSet", v);
       }
       int numSortedNumericFields = random().nextInt(3);
       Set<Long> numValues = new TreeSet<>();
@@ -2537,7 +2537,7 @@ public abstract class BaseDocValuesForma
       }
       for (Long l : numValues) {
         doc.addLong("dvSortedNumeric", l);
-        doc.addStored("storedSortedNumeric", Long.toString(l));
+        doc.addStoredString("storedSortedNumeric", Long.toString(l));
       }
       writer.addDocument(doc);
       if (random().nextInt(31) == 0) {
@@ -2757,7 +2757,7 @@ public abstract class BaseDocValuesForma
         w.addDocument(doc);
       }
       Document doc = w.newDocument();
-      doc.addStored("id", "5");
+      doc.addStoredString("id", "5");
       doc.addBinary("field", new BytesRef());
       w.addDocument(doc);
       IndexReader r = w.getReader();

Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java Fri Jan  2 12:02:31 2015
@@ -166,7 +166,7 @@ public abstract class BaseFieldInfoForma
   
   @Override
   protected void addRandomFields(Document doc) {
-    doc.addStored("foobar", TestUtil.randomSimpleString(random()));
+    doc.addStoredString("foobar", TestUtil.randomSimpleString(random()));
   }
 
   @Override

Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java Fri Jan  2 12:02:31 2015
@@ -214,7 +214,7 @@ public abstract class BaseSegmentInfoFor
   
   @Override
   protected void addRandomFields(Document doc) {
-    doc.addStored("foobar", TestUtil.randomSimpleString(random()));
+    doc.addStoredString("foobar", TestUtil.randomSimpleString(random()));
   }
 
   @Override

Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java Fri Jan  2 12:02:31 2015
@@ -62,7 +62,7 @@ public abstract class BaseStoredFieldsFo
     fieldTypes.setMultiValued("f");
     final int numValues = random().nextInt(3);
     for (int i = 0; i < numValues; ++i) {
-      d.addStored("f", TestUtil.randomSimpleString(random(), 100));
+      d.addStoredString("f", TestUtil.randomSimpleString(random(), 100));
     }
   }
 
@@ -99,7 +99,7 @@ public abstract class BaseStoredFieldsFo
         final String s;
         if (rand.nextInt(4) != 3) {
           s = TestUtil.randomUnicodeString(rand, 1000);
-          doc.addStored("f"+field, s);
+          doc.addStoredString("f"+field, s);
         } else {
           s = null;
         }
@@ -163,9 +163,9 @@ public abstract class BaseStoredFieldsFo
     FieldTypes fieldTypes = w.getFieldTypes();
     fieldTypes.setMultiValued("zzz");
 
-    doc.addStored("zzz", "a b c");
-    doc.addStored("aaa", "a b c");
-    doc.addStored("zzz", "1 2 3");
+    doc.addStoredString("zzz", "a b c");
+    doc.addStoredString("aaa", "a b c");
+    doc.addStoredString("zzz", "1 2 3");
     w.addDocument(doc);
     IndexReader r = w.getReader();
     Document doc2 = r.document(0);
@@ -221,7 +221,7 @@ public abstract class BaseStoredFieldsFo
     Directory dir = newDirectory();
     RandomIndexWriter w = new RandomIndexWriter(random(), dir);
     Document doc = w.newDocument();
-    doc.addStored("field", "value");
+    doc.addStoredString("field", "value");
     doc.addAtom("field2", "value");
     w.addDocument(doc);
     IndexReader r = w.getReader();
@@ -247,8 +247,8 @@ public abstract class BaseStoredFieldsFo
 
     for (int k = 0; k < 100; ++k) {
       Document doc = iw.newDocument();
-      doc.addStored("bytes", bytes);
-      doc.addStored("string", string);
+      doc.addStoredBinary("bytes", bytes);
+      doc.addStoredString("string", string);
       doc.addInt("int", i);
       doc.addLong("long", l);
       doc.addFloat("float", f);
@@ -428,7 +428,7 @@ public abstract class BaseStoredFieldsFo
       Document doc = iw.newDocument();
       doc.addInt("id", i);
       for (int j = 0; j < data[i].length; ++j) {
-        doc.addStored("bytes" + j, new BytesRef(data[i][j]));
+        doc.addStoredBinary("bytes" + j, new BytesRef(data[i][j]));
       }
       iw.w.addDocument(doc);
       if (random().nextBoolean() && (i % (data.length / 10) == 0)) {
@@ -509,10 +509,10 @@ public abstract class BaseStoredFieldsFo
       doc.addAtom("id", "" + i);
       if (x == 1) {
         for (int j = 0; j < numFields; ++j) {
-          doc.addStored("fld", randomByteArray(random().nextInt(10), 256));
+          doc.addStoredBinary("fld", randomByteArray(random().nextInt(10), 256));
         }
       } else {
-        doc.addStored("fld", randomByteArray(RandomInts.randomIntBetween(random(), 1000000, 5000000), 2));
+        doc.addStoredBinary("fld", randomByteArray(RandomInts.randomIntBetween(random(), 1000000, 5000000), 2));
       }
       iw.addDocument(doc);
       if (random().nextInt(numDocs) == 0) {

Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java Fri Jan  2 12:02:31 2015
@@ -156,11 +156,11 @@ class DocHelper {
     doc.addAtom(KEYWORD_FIELD_KEY, KEYWORD_TEXT);
     doc.addLargeText(NO_NORMS_KEY, NO_NORMS_TEXT);
     doc.addLargeText(NO_TF_KEY, NO_TF_TEXT);
-    doc.addStored(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT);
+    doc.addStoredString(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT);
     doc.addLargeText(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT);
     doc.addLargeText(UNSTORED_FIELD_2_KEY, UNSTORED_2_FIELD_TEXT);
     doc.addLargeText(LAZY_FIELD_KEY, LAZY_FIELD_TEXT);
-    doc.addStored(LAZY_FIELD_BINARY_KEY, LAZY_FIELD_BINARY_BYTES);
+    doc.addStoredBinary(LAZY_FIELD_BINARY_KEY, LAZY_FIELD_BINARY_BYTES);
     doc.addLargeText(LARGE_LAZY_FIELD_KEY, LARGE_LAZY_FIELD_TEXT);
     doc.addLargeText(TEXT_FIELD_UTF1_KEY, FIELD_UTF1_TEXT);
     doc.addLargeText(TEXT_FIELD_UTF2_KEY, FIELD_UTF2_TEXT);

Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterLeafReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterLeafReader.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterLeafReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterLeafReader.java Fri Jan  2 12:02:31 2015
@@ -19,9 +19,11 @@ package org.apache.lucene.index;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Set;
 
+import org.apache.lucene.document.FieldTypes;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.FilterIterator;
 
@@ -34,18 +36,22 @@ public final class FieldFilterLeafReader
   private final Set<String> fields;
   private final boolean negate;
   private final FieldInfos fieldInfos;
+  private final FieldTypes fieldTypes;
 
   public FieldFilterLeafReader(LeafReader in, Set<String> fields, boolean negate) {
     super(in);
     this.fields = fields;
     this.negate = negate;
     ArrayList<FieldInfo> filteredInfos = new ArrayList<>();
+    Set<String> actualFields = new HashSet<>();
     for (FieldInfo fi : in.getFieldInfos()) {
       if (hasField(fi.name)) {
-        filteredInfos.add(fi);
+        filteredInfos.add(fi);  
+        actualFields.add(fi.name);
       }
     }
     fieldInfos = new FieldInfos(filteredInfos.toArray(new FieldInfo[filteredInfos.size()]));
+    fieldTypes = new FieldTypes(in.getFieldTypes(), actualFields);
   }
   
   boolean hasField(String field) {
@@ -58,6 +64,11 @@ public final class FieldFilterLeafReader
   }
 
   @Override
+  public FieldTypes getFieldTypes() {
+    return fieldTypes;
+  }
+
+  @Override
   public Fields getTermVectors(int docID) throws IOException {
     Fields f = super.getTermVectors(docID);
     if (f == null) {

Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java Fri Jan  2 12:02:31 2015
@@ -105,10 +105,6 @@ public class RandomIndexWriter implement
     return w.newDocument();
   }
 
-  public ReferenceManager<DirectoryReader> getReaderManager() {
-    return w.getReaderManager();
-  }
-  
   /**
    * Adds a Document.
    * @see IndexWriter#addDocument(org.apache.lucene.index.IndexDocument)

Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1649007&r1=1649006&r2=1649007&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Fri Jan  2 12:02:31 2015
@@ -109,7 +109,9 @@ import org.apache.lucene.search.DocIdSet
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.FilterCachingPolicy;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryUtils.FCInvisibleMultiReader;
+import org.apache.lucene.search.TotalHitCountCollector;
 import org.apache.lucene.store.BaseDirectoryWrapper;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
@@ -137,7 +139,6 @@ import org.junit.Test;
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
 import org.junit.runner.RunWith;
-
 import com.carrotsearch.randomizedtesting.JUnit4MethodProvider;
 import com.carrotsearch.randomizedtesting.LifecycleScope;
 import com.carrotsearch.randomizedtesting.MixWithSuiteName;
@@ -646,6 +647,8 @@ public abstract class LuceneTestCase ext
     liveIWCFlushMode = flushMode;
   }
 
+  protected Directory dir;
+
   // -----------------------------------------------------------------
   // Suite and test case setup/ cleanup.
   // -----------------------------------------------------------------
@@ -656,6 +659,7 @@ public abstract class LuceneTestCase ext
   @Before
   public void setUp() throws Exception {
     parentChainCallRule.setupCalled = true;
+    dir = newDirectory();
   }
 
   /**
@@ -667,6 +671,10 @@ public abstract class LuceneTestCase ext
 
     // Test is supposed to call this itself, but we do this defensively in case it forgot:
     restoreIndexWriterMaxDocs();
+
+    if (dir != null) {
+      dir.close();
+    }
   }
 
   /** Tells {@link IndexWriter} to enforce the specified limit as the maximum number of documents in one index; call
@@ -882,11 +890,21 @@ public abstract class LuceneTestCase ext
   }
 
   /** create a new index writer config with random defaults, using MockAnalyzer */
+  public IndexWriter newIndexWriter() throws IOException {
+    return new IndexWriter(dir, newIndexWriterConfig());
+  }
+
+  /** create a new index writer config with random defaults, using MockAnalyzer */
   public static RandomIndexWriter newRandomIndexWriter(Directory dir) throws IOException {
     return new RandomIndexWriter(random(), dir, newIndexWriterConfig());
   }
 
   /** create a new index writer config with random defaults, using MockAnalyzer */
+  public RandomIndexWriter newRandomIndexWriter() throws IOException {
+    return new RandomIndexWriter(random(), dir, newIndexWriterConfig());
+  }
+
+  /** create a new index writer config with random defaults, using MockAnalyzer */
   public static RandomIndexWriter newRandomIndexWriter(Directory dir, Analyzer a) throws IOException {
     return new RandomIndexWriter(random(), dir, newIndexWriterConfig(a));
   }
@@ -2441,4 +2459,30 @@ public abstract class LuceneTestCase ext
     assert enabled = true; // Intentional side-effect!!!
     assertsAreEnabled = enabled;
   }
+
+  protected interface ThrowableRunnable {
+    public void run() throws Exception;
+  }
+
+  /** Only catches {@code IllegalStateException} and {@code IllegalArgumentException} and checks their message. */
+  protected static void shouldFail(ThrowableRunnable x, String message) throws Exception {
+    try {
+      x.run();
+      fail("did not hit expected exception");
+    } catch (IllegalStateException ise) {
+      assertTrue("wrong message: " + ise.getMessage(), ise.getMessage().startsWith(message));
+    } catch (IllegalArgumentException iae) {
+      assertTrue("wrong message: " + iae.getMessage(), iae.getMessage().startsWith(message));
+    }
+  }
+
+  public static int hitCount(IndexSearcher s, Query q) throws IOException {
+    if (random().nextBoolean()) {
+      return s.search(q, 1).totalHits;
+    } else {
+      TotalHitCountCollector c = new TotalHitCountCollector();
+      s.search(q, c);
+      return c.getTotalHits();
+    }
+  }
 }