svn commit: r1440716 - /lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java

[rm...@apache.org]

Author: rmuir
Date: Wed Jan 30 22:30:10 2013
New Revision: 1440716

URL: http://svn.apache.org/viewvc?rev=1440716&view=rev
Log:
clear bitsets earlier for less RAM during merge, prepare to not have bytes in ram

Modified:
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java?rev=1440716&r1=1440715&r2=1440716&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java Wed Jan 30 22:30:10 2013
@@ -244,6 +244,7 @@ public abstract class DocValuesConsumer 
     final List<SegmentState> segStates = new ArrayList<SegmentState>();
 
     private static class SegmentState {
+      int segmentID;
       AtomicReader reader;
       FixedBitSet liveTerms;
       int ord = -1;
@@ -293,6 +294,7 @@ public abstract class DocValuesConsumer 
         int maxDoc = reader.maxDoc();
 
         SegmentState state = new SegmentState();
+        state.segmentID = readerIDX;
         state.reader = reader;
         state.values = toMerge.get(readerIDX);
 
@@ -327,11 +329,22 @@ public abstract class DocValuesConsumer 
         }
       }
 
+      int lastOrds[] = new int[segStates.size()];
       BytesRef lastTerm = null;
       int ord = 0;
       while (q.size() != 0) {
         SegmentState top = q.top();
         if (lastTerm == null || !lastTerm.equals(top.scratch)) {
+          // a new unique term: record its segment ID / sourceOrd pair
+          int readerId = top.segmentID;
+          int sourceOrd = top.ord;
+          // nocommit: do this
+          //   ordToReaderID.add(readerId);
+          int delta = sourceOrd - lastOrds[readerId];
+          lastOrds[readerId] = sourceOrd;
+          // nocommit: do this
+          //   top.ordDeltas.add(delta);
+          
           lastTerm = BytesRef.deepCopyOf(top.scratch);
           // nocommit we could spill this to disk instead of
           // RAM, and replay on finish...
@@ -348,6 +361,10 @@ public abstract class DocValuesConsumer 
       }
 
       numMergedTerms = ord;
+      // clear our bitsets for GC: we dont need them anymore (e.g. while flushing merged stuff to codec)
+      for (SegmentState state : segStates) {
+        state.liveTerms = null;
+      }
     }
 
     /*