You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/10/27 22:49:14 UTC

svn commit: r1190017 - in /lucene/dev/trunk: ./ dev-tools/idea/lucene/contrib/ lucene/ lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/ lucene/src/java/org/apache/lucene/index/codecs/preflex/ lucene/src/java/org/apache/lucene/st...

Author: mikemccand
Date: Thu Oct 27 20:49:13 2011
New Revision: 1190017

URL: http://svn.apache.org/viewvc?rev=1190017&view=rev
Log:
LUCENE-2205: port to trunk in preflex codec; port TestPagedBytes and PagedBytes.DataInput/Output to trunk

Added:
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java
      - copied, changed from r1189914, lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TermInfosReaderIndex.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestTermInfosReaderIndex.java
      - copied, changed from r1189914, lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermInfosReaderIndex.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestPagedBytes.java
      - copied unchanged from r1189914, lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestPagedBytes.java
Modified:
    lucene/dev/trunk/   (props changed)
    lucene/dev/trunk/dev-tools/idea/lucene/contrib/   (props changed)
    lucene/dev/trunk/lucene/   (props changed)
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java   (props changed)
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java
    lucene/dev/trunk/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java   (props changed)
    lucene/dev/trunk/solr/   (props changed)
    lucene/dev/trunk/solr/CHANGES.txt   (props changed)
    lucene/dev/trunk/solr/LICENSE.txt   (props changed)
    lucene/dev/trunk/solr/NOTICE.txt   (props changed)
    lucene/dev/trunk/solr/README.txt   (props changed)
    lucene/dev/trunk/solr/build.xml   (props changed)
    lucene/dev/trunk/solr/client/   (props changed)
    lucene/dev/trunk/solr/common-build.xml   (props changed)
    lucene/dev/trunk/solr/contrib/   (props changed)
    lucene/dev/trunk/solr/contrib/clustering/src/test-files/   (props changed)
    lucene/dev/trunk/solr/contrib/dataimporthandler-extras/src/java/   (props changed)
    lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/   (props changed)
    lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/   (props changed)
    lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/   (props changed)
    lucene/dev/trunk/solr/contrib/uima/src/java/   (props changed)
    lucene/dev/trunk/solr/contrib/uima/src/test-files/   (props changed)
    lucene/dev/trunk/solr/core/   (props changed)
    lucene/dev/trunk/solr/core/src/java/   (props changed)
    lucene/dev/trunk/solr/core/src/test/   (props changed)
    lucene/dev/trunk/solr/dev-tools/   (props changed)
    lucene/dev/trunk/solr/example/   (props changed)
    lucene/dev/trunk/solr/lib/   (props changed)
    lucene/dev/trunk/solr/scripts/   (props changed)
    lucene/dev/trunk/solr/site/   (props changed)
    lucene/dev/trunk/solr/site-src/   (props changed)
    lucene/dev/trunk/solr/solrj/   (props changed)
    lucene/dev/trunk/solr/solrj/src/java/   (props changed)
    lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/   (props changed)
    lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/   (props changed)
    lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/common/   (props changed)
    lucene/dev/trunk/solr/test-framework/   (props changed)
    lucene/dev/trunk/solr/testlogging.properties   (props changed)
    lucene/dev/trunk/solr/webapp/   (props changed)

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1190017&r1=1190016&r2=1190017&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Thu Oct 27 20:49:13 2011
@@ -723,6 +723,9 @@ Optimizations
   FilteredQuery/IndexSearcher added by LUCENE-1536 to Lucene 4.0.
   (Uwe Schindler)
 
+* LUCENE-2205: Very substantial (3-5X) RAM reduction required to hold
+  the terms index on opening an IndexReader (Aaron McCurry via Mike McCandless)
+
 Test Cases
 
 * LUCENE-3420: Disable the finalness checks in TokenStream and Analyzer

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java?rev=1190017&r1=1190016&r2=1190017&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java Thu Oct 27 20:49:13 2011
@@ -47,9 +47,8 @@ public final class TermInfosReader {
   private final SegmentTermEnum origEnum;
   private final long size;
 
-  private final Term[] indexTerms;
-  private final TermInfo[] indexInfos;
-  private final long[] indexPointers;
+  private final TermInfosReaderIndex index;
+  private final int indexLength;
   
   private final int totalIndexInterval;
 
@@ -118,37 +117,23 @@ public final class TermInfosReader {
       if (indexDivisor != -1) {
         // Load terms index
         totalIndexInterval = origEnum.indexInterval * indexDivisor;
-        final SegmentTermEnum indexEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_INDEX_EXTENSION),
-                                                                                  context), fieldInfos, true);
 
-        try {
-          int indexSize = 1+((int)indexEnum.size-1)/indexDivisor;  // otherwise read index
+        final String indexFileName = IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_INDEX_EXTENSION);
+        final SegmentTermEnum indexEnum = new SegmentTermEnum(directory.openInput(indexFileName,
+                                                                                   context), fieldInfos, true);
 
-          indexTerms = new Term[indexSize];
-          indexInfos = new TermInfo[indexSize];
-          indexPointers = new long[indexSize];
-
-          for (int i=0;indexEnum.next(); i++) {
-            indexTerms[i] = indexEnum.term();
-            assert indexTerms[i] != null;
-            assert indexTerms[i].text() != null;
-            assert indexTerms[i].field() != null;
-            indexInfos[i] = indexEnum.termInfo();
-            indexPointers[i] = indexEnum.indexPointer;
-        
-            for (int j = 1; j < indexDivisor; j++)
-              if (!indexEnum.next())
-                break;
-          }
+        try {
+          // nocommit don't cast to int..
+          index = new TermInfosReaderIndex(indexEnum, indexDivisor, (int) dir.fileLength(indexFileName), totalIndexInterval);
+          indexLength = index.length();
         } finally {
           indexEnum.close();
         }
       } else {
         // Do not load terms index:
         totalIndexInterval = -1;
-        indexTerms = null;
-        indexInfos = null;
-        indexPointers = null;
+        index = null;
+        indexLength = -1;
       }
       success = true;
     } finally {
@@ -203,31 +188,6 @@ public final class TermInfosReader {
     }
   }
 
-  /** Returns the offset of the greatest index entry which is less than or equal to term.*/
-  private int getIndexOffset(Term term) {
-    int lo = 0;					  // binary search indexTerms[]
-    int hi = indexTerms.length - 1;
-
-    while (hi >= lo) {
-      int mid = (lo + hi) >>> 1;
-      assert indexTerms[mid] != null : "indexTerms = " + indexTerms.length + " mid=" + mid;
-      int delta = compareAsUTF16(term, indexTerms[mid]);
-      if (delta < 0)
-	hi = mid - 1;
-      else if (delta > 0)
-	lo = mid + 1;
-      else
-	return mid;
-    }
-    return hi;
-  }
-
-  private void seekEnum(SegmentTermEnum enumerator, int indexOffset) throws IOException {
-    enumerator.seek(indexPointers[indexOffset],
-                    ((long) indexOffset * totalIndexInterval) - 1,
-                    indexTerms[indexOffset], indexInfos[indexOffset]);
-  }
-
   /** Returns the TermInfo for a Term in the set, or null. */
   TermInfo get(Term term) throws IOException {
     return get(term, false);
@@ -272,8 +232,8 @@ public final class TermInfosReader {
 	&& ((enumerator.prev() != null && compareAsUTF16(term, enumerator.prev())> 0)
 	    || compareAsUTF16(term, enumerator.term()) >= 0)) {
       int enumOffset = (int)(enumerator.position/totalIndexInterval)+1;
-      if (indexTerms.length == enumOffset	  // but before end of block
-          || compareAsUTF16(term, indexTerms[enumOffset]) < 0) {
+      if (indexLength == enumOffset    // but before end of block
+    || index.compareTo(term, enumOffset) < 0) {
        // no need to seek
 
         final TermInfo ti;
@@ -309,10 +269,10 @@ public final class TermInfosReader {
       indexPos = (int) (tiOrd.termOrd / totalIndexInterval);
     } else {
       // Must do binary search:
-      indexPos = getIndexOffset(term);
+      indexPos = index.getIndexOffset(term);
     }
 
-    seekEnum(enumerator, indexPos);
+    index.seekEnum(enumerator, indexPos);
     enumerator.scanTo(term);
     final TermInfo ti;
 
@@ -352,7 +312,7 @@ public final class TermInfosReader {
   }
 
   private void ensureIndexIsRead() {
-    if (indexTerms == null) {
+    if (index == null) {
       throw new IllegalStateException("terms index was not loaded when this reader was created");
     }
   }
@@ -362,10 +322,10 @@ public final class TermInfosReader {
     if (size == 0) return -1;
 
     ensureIndexIsRead();
-    int indexOffset = getIndexOffset(term);
+    int indexOffset = index.getIndexOffset(term);
     
     SegmentTermEnum enumerator = getThreadResources().termEnum;
-    seekEnum(enumerator, indexOffset);
+    index.seekEnum(enumerator, indexOffset);
 
     while(compareAsUTF16(term, enumerator.term()) > 0 && enumerator.next()) {}
 

Copied: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java (from r1189914, lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TermInfosReaderIndex.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java?p2=lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java&p1=lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TermInfosReaderIndex.java&r1=1189914&r2=1190017&rev=1190017&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TermInfosReaderIndex.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java Thu Oct 27 20:49:13 2011
@@ -1,4 +1,4 @@
-package org.apache.lucene.index;
+package org.apache.lucene.index.codecs.preflex;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
 
+import org.apache.lucene.index.Term;
 import org.apache.lucene.util.BitUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.PagedBytes.PagedBytesDataInput;
@@ -75,8 +76,8 @@ class TermInfosReaderIndex {
     int fieldCounter = -1;
     for (int i = 0; indexEnum.next(); i++) {
       Term term = indexEnum.term();
-      if (currentField != term.field) {
-        currentField = term.field;
+      if (currentField == null || !currentField.equals(term.field())) {
+        currentField = term.field();
         fieldStrs.add(currentField);
         fieldCounter++;
       }
@@ -120,7 +121,7 @@ class TermInfosReaderIndex {
     // read the term
     int fieldId = input.readVInt();
     Term field = fields[fieldId];
-    Term term = field.createTerm(input.readString());
+    Term term = new Term(field.field(), input.readString());
 
     // read the terminfo
     TermInfo termInfo = new TermInfo();
@@ -146,14 +147,14 @@ class TermInfosReaderIndex {
    *          the term to locate.
    * @throws IOException 
    */
-  int getIndexOffset(Term term, BytesRef termBytesRef) throws IOException {
+  int getIndexOffset(Term term) throws IOException {
     int lo = 0;
     int hi = indexSize - 1;
     PagedBytesDataInput input = (PagedBytesDataInput) dataInput.clone();
     BytesRef scratch = new BytesRef();
     while (hi >= lo) {
       int mid = (lo + hi) >>> 1;
-      int delta = compareTo(term, termBytesRef, mid, input, scratch);
+      int delta = compareTo(term, mid, input, scratch);
       if (delta < 0)
         hi = mid - 1;
       else if (delta > 0)
@@ -179,7 +180,7 @@ class TermInfosReaderIndex {
     // read the term
     int fieldId = input.readVInt();
     Term field = fields[fieldId];
-    return field.createTerm(input.readString());
+    return new Term(field.field(), input.readString());
   }
 
   /**
@@ -202,8 +203,8 @@ class TermInfosReaderIndex {
    * @return int.
    * @throws IOException 
    */
-  int compareTo(Term term, BytesRef termBytesRef, int termIndex) throws IOException {
-    return compareTo(term, termBytesRef, termIndex, (PagedBytesDataInput) dataInput.clone(), new BytesRef());
+  int compareTo(Term term, int termIndex) throws IOException {
+    return compareTo(term, termIndex, (PagedBytesDataInput) dataInput.clone(), new BytesRef());
   }
 
   /**
@@ -219,7 +220,7 @@ class TermInfosReaderIndex {
    * @return int.
    * @throws IOException 
    */
-  private int compareTo(Term term, BytesRef termBytesRef, int termIndex, PagedBytesDataInput input, BytesRef reuse) throws IOException {
+  private int compareTo(Term term, int termIndex, PagedBytesDataInput input, BytesRef reuse) throws IOException {
     // if term field does not equal mid's field index, then compare fields
     // else if they are equal, compare term's string values...
     int c = compareField(term, termIndex, input);
@@ -227,7 +228,7 @@ class TermInfosReaderIndex {
       reuse.length = input.readVInt();
       reuse.grow(reuse.length);
       input.readBytes(reuse.bytes, 0, reuse.length);
-      return comparator.compare(termBytesRef, reuse);
+      return comparator.compare(term.bytes(), reuse);
     }
     return c;
   }
@@ -246,6 +247,6 @@ class TermInfosReaderIndex {
    */
   private int compareField(Term term, int termIndex, PagedBytesDataInput input) throws IOException {
     input.setPosition(indexToDataOffset.get(termIndex));
-    return term.field.compareTo(fields[input.readVInt()].field);
+    return term.field().compareTo(fields[input.readVInt()].field());
   }
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java?rev=1190017&r1=1190016&r2=1190017&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java Thu Oct 27 20:49:13 2011
@@ -52,7 +52,7 @@ public final class ByteArrayDataInput ex
   public int getPosition() {
     return pos;
   }
-
+  
   public void setPosition(int pos) {
     this.pos = pos;
   }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java?rev=1190017&r1=1190016&r2=1190017&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java Thu Oct 27 20:49:13 2011
@@ -17,12 +17,14 @@ package org.apache.lucene.util;
  * limitations under the License.
  */
 
-import org.apache.lucene.store.IndexInput;
-
-import java.util.List;
-import java.util.ArrayList;
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.IndexInput;
 
 /** Represents a logical byte[] as a series of pages.  You
  *  can write-once into the logical byte[] (append only),
@@ -37,6 +39,8 @@ public final class PagedBytes {
   private final int blockSize;
   private final int blockBits;
   private final int blockMask;
+  private boolean didSkipBytes;
+  private boolean frozen;
   private int upto;
   private byte[] currentBlock;
 
@@ -320,6 +324,7 @@ public final class PagedBytes {
       if (currentBlock != null) {
         blocks.add(currentBlock);
         blockEnd.add(upto);
+        didSkipBytes = true;
       }
       currentBlock = new byte[blockSize];
       upto = 0;
@@ -338,6 +343,12 @@ public final class PagedBytes {
 
   /** Commits final byte[], trimming it if necessary and if trim=true */
   public Reader freeze(boolean trim) {
+    if (frozen) {
+      throw new IllegalStateException("already frozen");
+    }
+    if (didSkipBytes) {
+      throw new IllegalStateException("cannot freeze when copy(BytesRef, BytesRef) was used");
+    }
     if (trim && upto < blockSize) {
       final byte[] newBlock = new byte[upto];
       System.arraycopy(currentBlock, 0, newBlock, 0, upto);
@@ -348,6 +359,7 @@ public final class PagedBytes {
     }
     blocks.add(currentBlock);
     blockEnd.add(upto); 
+    frozen = true;
     currentBlock = null;
     return new Reader(this);
   }
@@ -389,4 +401,150 @@ public final class PagedBytes {
 
     return pointer;
   }
+
+  public final class PagedBytesDataInput extends DataInput {
+    private int currentBlockIndex;
+    private int currentBlockUpto;
+    private byte[] currentBlock;
+
+    PagedBytesDataInput() {
+      currentBlock = blocks.get(0);
+    }
+
+    @Override
+    public Object clone() {
+      PagedBytesDataInput clone = getDataInput();
+      clone.setPosition(getPosition());
+      return clone;
+    }
+
+    /** Returns the current byte position. */
+    public long getPosition() {
+      return currentBlockIndex * blockSize + currentBlockUpto;
+    }
+  
+    /** Seek to a position previously obtained from
+     *  {@link #getPosition}. */
+    public void setPosition(long pos) {
+      currentBlockIndex = (int) (pos >> blockBits);
+      currentBlock = blocks.get(currentBlockIndex);
+      currentBlockUpto = (int) (pos & blockMask);
+    }
+
+    @Override
+    public byte readByte() {
+      if (currentBlockUpto == blockSize) {
+        nextBlock();
+      }
+      return currentBlock[currentBlockUpto++];
+    }
+
+    @Override
+    public void readBytes(byte[] b, int offset, int len) {
+      final int offsetEnd = offset + len;
+      while (true) {
+        final int blockLeft = blockSize - currentBlockUpto;
+        final int left = offsetEnd - offset;
+        if (blockLeft < left) {
+          System.arraycopy(currentBlock, currentBlockUpto,
+                           b, offset,
+                           blockLeft);
+          nextBlock();
+          offset += blockLeft;
+        } else {
+          // Last block
+          System.arraycopy(currentBlock, currentBlockUpto,
+                           b, offset,
+                           left);
+          currentBlockUpto += left;
+          break;
+        }
+      }
+    }
+
+    private void nextBlock() {
+      currentBlockIndex++;
+      currentBlockUpto = 0;
+      currentBlock = blocks.get(currentBlockIndex);
+    }
+  }
+
+  public final class PagedBytesDataOutput extends DataOutput {
+    @Override
+    public void writeByte(byte b) {
+      if (upto == blockSize) {
+        if (currentBlock != null) {
+          blocks.add(currentBlock);
+          blockEnd.add(upto);
+        }
+        currentBlock = new byte[blockSize];
+        upto = 0;
+      }
+      currentBlock[upto++] = b;
+    }
+
+    @Override
+    public void writeBytes(byte[] b, int offset, int length) throws IOException {
+      if (length == 0) {
+        return;
+      }
+
+      if (upto == blockSize) {
+        if (currentBlock != null) {
+          blocks.add(currentBlock);
+          blockEnd.add(upto);
+        }
+        currentBlock = new byte[blockSize];
+        upto = 0;
+      }
+          
+      final int offsetEnd = offset + length;
+      while(true) {
+        final int left = offsetEnd - offset;
+        final int blockLeft = blockSize - upto;
+        if (blockLeft < left) {
+          System.arraycopy(b, offset, currentBlock, upto, blockLeft);
+          blocks.add(currentBlock);
+          blockEnd.add(blockSize);
+          currentBlock = new byte[blockSize];
+          upto = 0;
+          offset += blockLeft;
+        } else {
+          // Last block
+          System.arraycopy(b, offset, currentBlock, upto, left);
+          upto += left;
+          break;
+        }
+      }
+    }
+
+    /** Return the current byte position. */
+    public long getPosition() {
+      if (currentBlock == null) {
+        return 0;
+      } else {
+        return blocks.size() * blockSize + upto;
+      }
+    }
+  }
+
+  /** Returns a DataInput to read values from this
+   *  PagedBytes instance. */
+  public PagedBytesDataInput getDataInput() {
+    if (!frozen) {
+      throw new IllegalStateException("must call freeze() before getDataInput");
+    }
+    return new PagedBytesDataInput();
+  }
+
+  /** Returns a DataOutput that you may use to write into
+   *  this PagedBytes instance.  If you do this, you should
+   *  not call the other writing methods (eg, copy);
+   *  results are undefined. */
+  public PagedBytesDataOutput getDataOutput() {
+    if (frozen) {
+      throw new IllegalStateException("cannot get DataOutput after freeze()");
+    }
+    return new PagedBytesDataOutput();
+  }
 }

Copied: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestTermInfosReaderIndex.java (from r1189914, lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermInfosReaderIndex.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestTermInfosReaderIndex.java?p2=lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestTermInfosReaderIndex.java&p1=lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermInfosReaderIndex.java&r1=1189914&r2=1190017&rev=1190017&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermInfosReaderIndex.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestTermInfosReaderIndex.java Thu Oct 27 20:49:13 2011
@@ -1,4 +1,4 @@
-package org.apache.lucene.index;
+package org.apache.lucene.index.codecs.preflex;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -26,8 +26,23 @@ import java.util.Random;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.index.codecs.CoreCodecProvider;
+import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
@@ -47,7 +62,6 @@ public class TestTermInfosReaderIndex ex
   private SegmentTermEnum termEnum;
   private int indexDivisor;
   private int termIndexInterval;
-  private int readBufferSize = 1024;
   private IndexReader reader;
   private List<Term> sampleTerms;
   
@@ -57,16 +71,17 @@ public class TestTermInfosReaderIndex ex
     indexDivisor = _TestUtil.nextInt(random, 1, 10);
     directory = newDirectory();
     termIndexInterval = populate(directory);
-    
-    SegmentReader r = SegmentReader.getOnlySegmentReader(directory);
+
+    IndexReader r0 = IndexReader.open(directory);
+    SegmentReader r = (SegmentReader) r0.getSequentialSubReaders()[0];
     String segment = r.getSegmentName();
     r.close();
 
-    FieldInfos fieldInfos = new FieldInfos(directory, IndexFileNames.segmentFileName(segment, IndexFileNames.FIELD_INFOS_EXTENSION));
-    String segmentFileName = IndexFileNames.segmentFileName(segment, IndexFileNames.TERMS_INDEX_EXTENSION);
+    FieldInfos fieldInfos = new FieldInfos(directory, IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELD_INFOS_EXTENSION));
+    String segmentFileName = IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_INDEX_EXTENSION);
     long tiiFileLength = directory.fileLength(segmentFileName);
-    IndexInput input = directory.openInput(segmentFileName, readBufferSize);
-    termEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, IndexFileNames.TERMS_EXTENSION), readBufferSize), fieldInfos, false);
+    IndexInput input = directory.openInput(segmentFileName, newIOContext(random));
+    termEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_EXTENSION), newIOContext(random)), fieldInfos, false);
     int totalIndexInterval = termEnum.indexInterval * indexDivisor;
     
     SegmentTermEnum indexEnum = new SegmentTermEnum(input, fieldInfos, true);
@@ -91,19 +106,18 @@ public class TestTermInfosReaderIndex ex
     int indexPosition = 3;
     SegmentTermEnum clone = (SegmentTermEnum) termEnum.clone();
     Term term = findTermThatWouldBeAtIndex(clone, indexPosition);
-    clone.close();
     SegmentTermEnum enumerator = clone;
     index.seekEnum(enumerator, indexPosition);
     assertEquals(term, enumerator.term());
+    clone.close();
   }
   
   public void testCompareTo() throws IOException {
     Term term = new Term("field" + random.nextInt(NUMBER_OF_FIELDS) ,getText());
-    BytesRef termBytesRef = new BytesRef(term.text);
     for (int i = 0; i < index.length(); i++) {
       Term t = index.getTerm(i);
       int compareTo = term.compareTo(t);
-      assertEquals(compareTo, index.compareTo(term, termBytesRef, i));
+      assertEquals(compareTo, index.compareTo(term, i));
     }
   }
   
@@ -120,16 +134,19 @@ public class TestTermInfosReaderIndex ex
   private List<Term> sample(IndexReader reader, int size) throws IOException {
     List<Term> sample = new ArrayList<Term>();
     Random random = new Random();
-    TermEnum terms = reader.terms();
-    while (terms.next()) {
-      if (sample.size() >= size) {
-        int pos = random.nextInt(size);
-        sample.set(pos, terms.term());
-      } else {
-        sample.add(terms.term());
+    FieldsEnum fieldsEnum = MultiFields.getFields(reader).iterator();
+    String field;
+    while((field = fieldsEnum.next()) != null) {
+      TermsEnum terms = fieldsEnum.terms();
+      while (terms.next() != null) {
+        if (sample.size() >= size) {
+          int pos = random.nextInt(size);
+          sample.set(pos, new Term(field, terms.term()));
+        } else {
+          sample.add(new Term(field, terms.term()));
+        }
       }
     }
-    terms.close();
     Collections.shuffle(sample);
     return sample;
   }
@@ -147,6 +164,11 @@ public class TestTermInfosReaderIndex ex
   private int populate(Directory directory) throws CorruptIndexException, LockObtainFailedException, IOException {
     IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, 
         new MockAnalyzer(random, MockTokenizer.KEYWORD, false));
+    CoreCodecProvider cp = new CoreCodecProvider();
+    cp.unregister(cp.lookup("PreFlex"));
+    cp.register(new PreFlexRWCodec());
+    cp.setDefaultFieldCodec("PreFlex");
+    config.setCodecProvider(cp);
     // turn off compound file, this test will open some index files directly.
     LogMergePolicy mp = newLogMergePolicy();
     mp.setUseCompoundFile(false);
@@ -156,7 +178,7 @@ public class TestTermInfosReaderIndex ex
     for (int i = 0; i < NUMBER_OF_DOCUMENTS; i++) {
       Document document = new Document();
       for (int f = 0; f < NUMBER_OF_FIELDS; f++) {
-        document.add(newField("field" + f,getText(),Store.NO,Index.NOT_ANALYZED_NO_NORMS));
+        document.add(newField("field" + f, getText(), StringField.TYPE_UNSTORED));
       }
       writer.addDocument(document);
     }