You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/05/28 21:28:18 UTC

svn commit: r1343365 [4/8] - in /lucene/dev/trunk: ./ lucene/ lucene/analysis/ lucene/analysis/common/ lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/ lucene/core/src/java/org/apache/lucene/codecs/appending/ lucene/core/src/java/org/apache/...

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java Mon May 28 19:28:10 2012
@@ -121,9 +121,9 @@ class BufferedDeletesStream {
     public final long gen;
 
     // If non-null, contains segments that are 100% deleted
-    public final List<SegmentInfo> allDeleted;
+    public final List<SegmentInfoPerCommit> allDeleted;
 
-    ApplyDeletesResult(boolean anyDeletes, long gen, List<SegmentInfo> allDeleted) {
+    ApplyDeletesResult(boolean anyDeletes, long gen, List<SegmentInfoPerCommit> allDeleted) {
       this.anyDeletes = anyDeletes;
       this.gen = gen;
       this.allDeleted = allDeleted;
@@ -131,9 +131,9 @@ class BufferedDeletesStream {
   }
 
   // Sorts SegmentInfos from smallest to biggest bufferedDelGen:
-  private static final Comparator<SegmentInfo> sortSegInfoByDelGen = new Comparator<SegmentInfo>() {
+  private static final Comparator<SegmentInfoPerCommit> sortSegInfoByDelGen = new Comparator<SegmentInfoPerCommit>() {
     @Override
-    public int compare(SegmentInfo si1, SegmentInfo si2) {
+    public int compare(SegmentInfoPerCommit si1, SegmentInfoPerCommit si2) {
       final long cmp = si1.getBufferedDeletesGen() - si2.getBufferedDeletesGen();
       if (cmp > 0) {
         return 1;
@@ -148,7 +148,7 @@ class BufferedDeletesStream {
   /** Resolves the buffered deleted Term/Query/docIDs, into
    *  actual deleted docIDs in the liveDocs MutableBits for
    *  each SegmentReader. */
-  public synchronized ApplyDeletesResult applyDeletes(IndexWriter.ReaderPool readerPool, List<SegmentInfo> infos) throws IOException {
+  public synchronized ApplyDeletesResult applyDeletes(IndexWriter.ReaderPool readerPool, List<SegmentInfoPerCommit> infos) throws IOException {
     final long t0 = System.currentTimeMillis();
 
     if (infos.size() == 0) {
@@ -168,7 +168,7 @@ class BufferedDeletesStream {
       infoStream.message("BD", "applyDeletes: infos=" + infos + " packetCount=" + deletes.size());
     }
 
-    List<SegmentInfo> infos2 = new ArrayList<SegmentInfo>();
+    List<SegmentInfoPerCommit> infos2 = new ArrayList<SegmentInfoPerCommit>();
     infos2.addAll(infos);
     Collections.sort(infos2, sortSegInfoByDelGen);
 
@@ -178,13 +178,13 @@ class BufferedDeletesStream {
     int infosIDX = infos2.size()-1;
     int delIDX = deletes.size()-1;
 
-    List<SegmentInfo> allDeleted = null;
+    List<SegmentInfoPerCommit> allDeleted = null;
 
     while (infosIDX >= 0) {
       //System.out.println("BD: cycle delIDX=" + delIDX + " infoIDX=" + infosIDX);
 
       final FrozenBufferedDeletes packet = delIDX >= 0 ? deletes.get(delIDX) : null;
-      final SegmentInfo info = infos2.get(infosIDX);
+      final SegmentInfoPerCommit info = infos2.get(infosIDX);
       final long segGen = info.getBufferedDeletesGen();
 
       if (packet != null && segGen < packet.delGen()) {
@@ -225,8 +225,8 @@ class BufferedDeletesStream {
           // already did that on flush:
           delCount += applyQueryDeletes(packet.queriesIterable(), rld, reader);
           final int fullDelCount = rld.info.getDelCount() + rld.getPendingDeleteCount();
-          assert fullDelCount <= rld.info.docCount;
-          segAllDeletes = fullDelCount == rld.info.docCount;
+          assert fullDelCount <= rld.info.info.getDocCount();
+          segAllDeletes = fullDelCount == rld.info.info.getDocCount();
         } finally {
           rld.release(reader);
           readerPool.release(rld);
@@ -235,7 +235,7 @@ class BufferedDeletesStream {
 
         if (segAllDeletes) {
           if (allDeleted == null) {
-            allDeleted = new ArrayList<SegmentInfo>();
+            allDeleted = new ArrayList<SegmentInfoPerCommit>();
           }
           allDeleted.add(info);
         }
@@ -271,8 +271,8 @@ class BufferedDeletesStream {
             delCount += applyTermDeletes(coalescedDeletes.termsIterable(), rld, reader);
             delCount += applyQueryDeletes(coalescedDeletes.queriesIterable(), rld, reader);
             final int fullDelCount = rld.info.getDelCount() + rld.getPendingDeleteCount();
-            assert fullDelCount <= rld.info.docCount;
-            segAllDeletes = fullDelCount == rld.info.docCount;
+            assert fullDelCount <= rld.info.info.getDocCount();
+            segAllDeletes = fullDelCount == rld.info.info.getDocCount();
           } finally {   
             rld.release(reader);
             readerPool.release(rld);
@@ -281,7 +281,7 @@ class BufferedDeletesStream {
 
           if (segAllDeletes) {
             if (allDeleted == null) {
-              allDeleted = new ArrayList<SegmentInfo>();
+              allDeleted = new ArrayList<SegmentInfoPerCommit>();
             }
             allDeleted.add(info);
           }
@@ -316,7 +316,7 @@ class BufferedDeletesStream {
   public synchronized void prune(SegmentInfos segmentInfos) {
     assert checkDeleteStats();
     long minGen = Long.MAX_VALUE;
-    for(SegmentInfo info : segmentInfos) {
+    for(SegmentInfoPerCommit info : segmentInfos) {
       minGen = Math.min(info.getBufferedDeletesGen(), minGen);
     }
 

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Mon May 28 19:28:10 2012
@@ -88,9 +88,6 @@ public class CheckIndex {
     /** Number of segments in the index. */
     public int numSegments;
 
-    /** String description of the version of the index. */
-    public String segmentFormat;
-
     /** Empty unless you passed specific segments list to check as optional 3rd argument.
      *  @see CheckIndex#checkIndex(List) */
     public List<String> segmentsChecked = new ArrayList<String>();
@@ -185,11 +182,6 @@ public class CheckIndex {
       /** Number of fields in this segment. */
       int numFields;
 
-      /** True if at least one of the fields in this segment
-       *  has position data
-       *  @see FieldType#setIndexOptions(org.apache.lucene.index.FieldInfo.IndexOptions) */
-      public boolean hasProx;
-
       /** Map that includes certain
        *  debugging details that IndexWriter records into
        *  each segment it creates */
@@ -369,8 +361,8 @@ public class CheckIndex {
     String oldSegs = null;
     boolean foundNonNullVersion = false;
     Comparator<String> versionComparator = StringHelper.getVersionComparator();
-    for (SegmentInfo si : sis) {
-      String version = si.getVersion();
+    for (SegmentInfoPerCommit si : sis) {
+      String version = si.info.getVersion();
       if (version == null) {
         // pre-3.1 segment
         oldSegs = "pre-3.1";
@@ -415,27 +407,8 @@ public class CheckIndex {
     String sFormat = "";
     boolean skip = false;
 
-    if (format == SegmentInfos.FORMAT_DIAGNOSTICS) {
-      sFormat = "FORMAT_DIAGNOSTICS [Lucene 2.9]";
-    } else if (format == SegmentInfos.FORMAT_HAS_VECTORS) {
-      sFormat = "FORMAT_HAS_VECTORS [Lucene 3.1]";
-    } else if (format == SegmentInfos.FORMAT_3_1) {
-      sFormat = "FORMAT_3_1 [Lucene 3.1+]";
-    } else if (format == SegmentInfos.FORMAT_4_0) {
-      sFormat = "FORMAT_4_0 [Lucene 4.0]";
-    } else if (format == SegmentInfos.FORMAT_CURRENT) {
-      throw new RuntimeException("BUG: You should update this tool!");
-    } else if (format < SegmentInfos.FORMAT_CURRENT) {
-      sFormat = "int=" + format + " [newer version of Lucene than this tool supports]";
-      skip = true;
-    } else if (format > SegmentInfos.FORMAT_MINIMUM) {
-      sFormat = "int=" + format + " [older version of Lucene than this tool supports]";
-      skip = true;
-    }
-
     result.segmentsFileName = segmentsFileName;
     result.numSegments = numSegments;
-    result.segmentFormat = sFormat;
     result.userData = sis.getUserData();
     String userDataString;
     if (sis.getUserData().size() > 0) {
@@ -482,50 +455,41 @@ public class CheckIndex {
     result.maxSegmentName = -1;
 
     for(int i=0;i<numSegments;i++) {
-      final SegmentInfo info = sis.info(i);
-      int segmentName = Integer.parseInt(info.name.substring(1), Character.MAX_RADIX);
+      final SegmentInfoPerCommit info = sis.info(i);
+      int segmentName = Integer.parseInt(info.info.name.substring(1), Character.MAX_RADIX);
       if (segmentName > result.maxSegmentName) {
         result.maxSegmentName = segmentName;
       }
-      if (onlySegments != null && !onlySegments.contains(info.name))
+      if (onlySegments != null && !onlySegments.contains(info.info.name)) {
         continue;
+      }
       Status.SegmentInfoStatus segInfoStat = new Status.SegmentInfoStatus();
       result.segmentInfos.add(segInfoStat);
-      msg("  " + (1+i) + " of " + numSegments + ": name=" + info.name + " docCount=" + info.docCount);
-      segInfoStat.name = info.name;
-      segInfoStat.docCount = info.docCount;
+      msg("  " + (1+i) + " of " + numSegments + ": name=" + info.info.name + " docCount=" + info.info.getDocCount());
+      segInfoStat.name = info.info.name;
+      segInfoStat.docCount = info.info.getDocCount();
 
-      int toLoseDocCount = info.docCount;
+      int toLoseDocCount = info.info.getDocCount();
 
       SegmentReader reader = null;
 
       try {
-        final Codec codec = info.getCodec();
+        final Codec codec = info.info.getCodec();
         msg("    codec=" + codec);
         segInfoStat.codec = codec;
-        msg("    compound=" + info.getUseCompoundFile());
-        segInfoStat.compound = info.getUseCompoundFile();
-        msg("    hasProx=" + info.getHasProx());
-        segInfoStat.hasProx = info.getHasProx();
+        msg("    compound=" + info.info.getUseCompoundFile());
+        segInfoStat.compound = info.info.getUseCompoundFile();
         msg("    numFiles=" + info.files().size());
         segInfoStat.numFiles = info.files().size();
         segInfoStat.sizeMB = info.sizeInBytes()/(1024.*1024.);
         msg("    size (MB)=" + nf.format(segInfoStat.sizeMB));
-        Map<String,String> diagnostics = info.getDiagnostics();
+        Map<String,String> diagnostics = info.info.getDiagnostics();
         segInfoStat.diagnostics = diagnostics;
         if (diagnostics.size() > 0) {
           msg("    diagnostics = " + diagnostics);
         }
 
-        final int docStoreOffset = info.getDocStoreOffset();
-        if (docStoreOffset != -1) {
-          msg("    docStoreOffset=" + docStoreOffset);
-          segInfoStat.docStoreOffset = docStoreOffset;
-          msg("    docStoreSegment=" + info.getDocStoreSegment());
-          segInfoStat.docStoreSegment = info.getDocStoreSegment();
-          msg("    docStoreIsCompoundFile=" + info.getDocStoreIsCompoundFile());
-          segInfoStat.docStoreCompoundFile = info.getDocStoreIsCompoundFile();
-        }
+        // TODO: we could append the info attributes() to the msg?
 
         if (info.hasDeletions()) {
           msg("    no deletions");
@@ -545,14 +509,14 @@ public class CheckIndex {
         final int numDocs = reader.numDocs();
         toLoseDocCount = numDocs;
         if (reader.hasDeletions()) {
-          if (reader.numDocs() != info.docCount - info.getDelCount()) {
-            throw new RuntimeException("delete count mismatch: info=" + (info.docCount - info.getDelCount()) + " vs reader=" + reader.numDocs());
+          if (reader.numDocs() != info.info.getDocCount() - info.getDelCount()) {
+            throw new RuntimeException("delete count mismatch: info=" + (info.info.getDocCount() - info.getDelCount()) + " vs reader=" + reader.numDocs());
           }
-          if ((info.docCount-reader.numDocs()) > reader.maxDoc()) {
-            throw new RuntimeException("too many deleted docs: maxDoc()=" + reader.maxDoc() + " vs del count=" + (info.docCount-reader.numDocs()));
+          if ((info.info.getDocCount()-reader.numDocs()) > reader.maxDoc()) {
+            throw new RuntimeException("too many deleted docs: maxDoc()=" + reader.maxDoc() + " vs del count=" + (info.info.getDocCount()-reader.numDocs()));
           }
-          if (info.docCount - numDocs != info.getDelCount()) {
-            throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs reader=" + (info.docCount - numDocs));
+          if (info.info.getDocCount() - numDocs != info.getDelCount()) {
+            throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs reader=" + (info.info.getDocCount() - numDocs));
           }
           Bits liveDocs = reader.getLiveDocs();
           if (liveDocs == null) {
@@ -569,11 +533,11 @@ public class CheckIndex {
             }
           }
           
-          segInfoStat.numDeleted = info.docCount - numDocs;
+          segInfoStat.numDeleted = info.info.getDocCount() - numDocs;
           msg("OK [" + (segInfoStat.numDeleted) + " deleted docs]");
         } else {
           if (info.getDelCount() != 0) {
-            throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs reader=" + (info.docCount - numDocs));
+            throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs reader=" + (info.info.getDocCount() - numDocs));
           }
           Bits liveDocs = reader.getLiveDocs();
           if (liveDocs != null) {
@@ -586,8 +550,9 @@ public class CheckIndex {
           }
           msg("OK");
         }
-        if (reader.maxDoc() != info.docCount)
-          throw new RuntimeException("SegmentReader.maxDoc() " + reader.maxDoc() + " != SegmentInfos.docCount " + info.docCount);
+        if (reader.maxDoc() != info.info.getDocCount()) {
+          throw new RuntimeException("SegmentReader.maxDoc() " + reader.maxDoc() + " != SegmentInfos.docCount " + info.info.getDocCount());
+        }
 
         // Test getFieldInfos()
         if (infoStream != null) {
@@ -609,7 +574,7 @@ public class CheckIndex {
         // Test Term Vectors
         segInfoStat.termVectorStatus = testTermVectors(fieldInfos, info, reader, nf);
         
-        segInfoStat.docValuesStatus = testDocValues(info, reader);
+        segInfoStat.docValuesStatus = testDocValues(info, fieldInfos, reader);
 
         // Rethrow the first exception we encountered
         //  This will cause stats for failed segments to be incremented properly
@@ -706,8 +671,7 @@ public class CheckIndex {
    * checks Fields api is consistent with itself.
    * searcher is optional, to verify with queries. Can be null.
    */
-  // TODO: cutover term vectors to this!
-  private Status.TermIndexStatus checkFields(Fields fields, Bits liveDocs, int maxDoc, FieldInfos fieldInfos, IndexSearcher searcher) throws IOException {
+  private Status.TermIndexStatus checkFields(Fields fields, Bits liveDocs, int maxDoc, FieldInfos fieldInfos, IndexSearcher searcher, boolean doPrint) throws IOException {
     // TODO: we should probably return our own stats thing...?!
     
     final Status.TermIndexStatus status = new Status.TermIndexStatus();
@@ -741,7 +705,7 @@ public class CheckIndex {
       if (fi == null) {
         throw new RuntimeException("fieldsEnum inconsistent with fieldInfos, no fieldInfos for: " + field);
       }
-      if (!fi.isIndexed) {
+      if (!fi.isIndexed()) {
         throw new RuntimeException("fieldsEnum inconsistent with fieldInfos, isIndexed == false for: " + field);
       }
       
@@ -1131,8 +1095,10 @@ public class CheckIndex {
     if (status.termCount != uniqueTermCountAllFields) {
       throw new RuntimeException("termCount mismatch " + uniqueTermCountAllFields + " vs " + (status.termCount));
     }
-    
-    msg("OK [" + status.termCount + " terms; " + status.totFreq + " terms/docs pairs; " + status.totPos + " tokens]");
+
+    if (doPrint) {
+      msg("OK [" + status.termCount + " terms; " + status.totFreq + " terms/docs pairs; " + status.totPos + " tokens]");
+    }
     
     if (verbose && status.blockTreeStats != null && infoStream != null && status.termCount > 0) {
       for(Map.Entry<String,BlockTreeTermsReader.Stats> ent : status.blockTreeStats.entrySet()) {
@@ -1163,13 +1129,13 @@ public class CheckIndex {
       }
 
       final Fields fields = reader.fields();
-      status = checkFields(fields, liveDocs, maxDoc, fieldInfos, is);
+      status = checkFields(fields, liveDocs, maxDoc, fieldInfos, is, true);
       if (liveDocs != null) {
         if (infoStream != null) {
           infoStream.print("    test (ignoring deletes): terms, freq, prox...");
         }
         // TODO: can we make a IS that ignores all deletes?
-        checkFields(fields, null, maxDoc, fieldInfos, null);
+        checkFields(fields, null, maxDoc, fieldInfos, null, true);
       }
     } catch (Throwable e) {
       msg("ERROR: " + e);
@@ -1186,7 +1152,7 @@ public class CheckIndex {
   /**
    * Test stored fields for a segment.
    */
-  private Status.StoredFieldStatus testStoredFields(SegmentInfo info, SegmentReader reader, NumberFormat format) {
+  private Status.StoredFieldStatus testStoredFields(SegmentInfoPerCommit info, SegmentReader reader, NumberFormat format) {
     final Status.StoredFieldStatus status = new Status.StoredFieldStatus();
 
     try {
@@ -1196,7 +1162,7 @@ public class CheckIndex {
 
       // Scan stored fields for all documents
       final Bits liveDocs = reader.getLiveDocs();
-      for (int j = 0; j < info.docCount; ++j) {
+      for (int j = 0; j < info.info.getDocCount(); ++j) {
         // Intentionally pull even deleted documents to
         // make sure they too are not corrupt:
         Document doc = reader.document(j);
@@ -1310,14 +1276,14 @@ public class CheckIndex {
     }
   }
   
-  private Status.DocValuesStatus testDocValues(SegmentInfo info,
-      SegmentReader reader) {
+  private Status.DocValuesStatus testDocValues(SegmentInfoPerCommit info,
+                                               FieldInfos fieldInfos,
+                                               SegmentReader reader) {
     final Status.DocValuesStatus status = new Status.DocValuesStatus();
     try {
       if (infoStream != null) {
         infoStream.print("    test: DocValues........");
       }
-      final FieldInfos fieldInfos = info.getFieldInfos();
       for (FieldInfo fieldInfo : fieldInfos) {
         if (fieldInfo.hasDocValues()) {
           status.totalValueFields++;
@@ -1345,7 +1311,7 @@ public class CheckIndex {
   /**
    * Test term vectors for a segment.
    */
-  private Status.TermVectorStatus testTermVectors(FieldInfos fieldInfos, SegmentInfo info, SegmentReader reader, NumberFormat format) {
+  private Status.TermVectorStatus testTermVectors(FieldInfos fieldInfos, SegmentInfoPerCommit info, SegmentReader reader, NumberFormat format) {
     final Status.TermVectorStatus status = new Status.TermVectorStatus();
 
     final Bits onlyDocIsDeleted = new FixedBitSet(1);
@@ -1375,7 +1341,7 @@ public class CheckIndex {
       TermsEnum termsEnum = null;
       TermsEnum postingsTermsEnum = null;
 
-      for (int j = 0; j < info.docCount; ++j) {
+      for (int j = 0; j < info.info.getDocCount(); ++j) {
         // Intentionally pull/visit (but don't count in
         // stats) deleted documents to make sure they too
         // are not corrupt:
@@ -1386,10 +1352,10 @@ public class CheckIndex {
 
         if (tfv != null) {
           // First run with no deletions:
-          checkFields(tfv, null, 1, fieldInfos, null);
+          checkFields(tfv, null, 1, fieldInfos, null, false);
 
           // Again, with the one doc deleted:
-          checkFields(tfv, onlyDocIsDeleted, 1, fieldInfos, null);
+          checkFields(tfv, onlyDocIsDeleted, 1, fieldInfos, null, false);
 
           // Only agg stats if the doc is live:
           final boolean doStats = liveDocs == null || liveDocs.get(j);
@@ -1406,7 +1372,7 @@ public class CheckIndex {
 
             // Make sure FieldInfo thinks this field is vector'd:
             final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
-            if (!fieldInfo.storeTermVector) {
+            if (!fieldInfo.hasVectors()) {
               throw new RuntimeException("docID=" + j + " has term vectors for field=" + field + " but FieldInfo has storeTermVector=false");
             }
 
@@ -1594,7 +1560,7 @@ public class CheckIndex {
     if (result.partial)
       throw new IllegalArgumentException("can only fix an index that was fully checked (this status checked a subset of segments)");
     result.newSegments.changed();
-    result.newSegments.commit(result.dir, codec);
+    result.newSegments.commit(result.dir);
   }
 
   private static boolean assertsOn;

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java Mon May 28 19:28:10 2012
@@ -20,7 +20,7 @@ package org.apache.lucene.index;
 import java.io.IOException;
 
 abstract class DocConsumer {
-  abstract void processDocument(FieldInfos fieldInfos) throws IOException;
+  abstract void processDocument(FieldInfos.Builder fieldInfos) throws IOException;
   abstract void finishDocument() throws IOException;
   abstract void flush(final SegmentWriteState state) throws IOException;
   abstract void abort();

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java Mon May 28 19:28:10 2012
@@ -23,7 +23,7 @@ import java.util.Map;
 abstract class DocFieldConsumer {
   /** Called when DocumentsWriterPerThread decides to create a new
    *  segment */
-  abstract void flush(Map<FieldInfo, DocFieldConsumerPerField> fieldsToFlush, SegmentWriteState state) throws IOException;
+  abstract void flush(Map<String, DocFieldConsumerPerField> fieldsToFlush, SegmentWriteState state) throws IOException;
 
   /** Called when an aborting exception is hit */
   abstract void abort();

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java Mon May 28 19:28:10 2012
@@ -71,28 +71,28 @@ final class DocFieldProcessor extends Do
   @Override
   public void flush(SegmentWriteState state) throws IOException {
 
-    Map<FieldInfo, DocFieldConsumerPerField> childFields = new HashMap<FieldInfo, DocFieldConsumerPerField>();
+    Map<String,DocFieldConsumerPerField> childFields = new HashMap<String,DocFieldConsumerPerField>();
     Collection<DocFieldConsumerPerField> fields = fields();
     for (DocFieldConsumerPerField f : fields) {
-      childFields.put(f.getFieldInfo(), f);
+      childFields.put(f.getFieldInfo().name, f);
     }
 
     fieldsWriter.flush(state);
     consumer.flush(childFields, state);
 
     for (DocValuesConsumerAndDocID consumer : docValues.values()) {
-      consumer.docValuesConsumer.finish(state.numDocs);
+      consumer.docValuesConsumer.finish(state.segmentInfo.getDocCount());
     }
+    
+    // close perDocConsumer during flush to ensure all files are flushed due to PerCodec CFS
+    IOUtils.close(perDocConsumer);
 
     // Important to save after asking consumer to flush so
     // consumer can alter the FieldInfo* if necessary.  EG,
     // FreqProxTermsWriter does this with
     // FieldInfo.storePayload.
     FieldInfosWriter infosWriter = codec.fieldInfosFormat().getFieldInfosWriter();
-    infosWriter.write(state.directory, state.segmentName, state.fieldInfos, IOContext.DEFAULT);
-
-    // close perDocConsumer during flush to ensure all files are flushed due to PerCodec CFS
-    IOUtils.close(perDocConsumer);
+    infosWriter.write(state.directory, state.segmentInfo.name, state.fieldInfos, IOContext.DEFAULT);
   }
 
   @Override
@@ -203,7 +203,7 @@ final class DocFieldProcessor extends Do
   }
 
   @Override
-  public void processDocument(FieldInfos fieldInfos) throws IOException {
+  public void processDocument(FieldInfos.Builder fieldInfos) throws IOException {
 
     consumer.startDocument();
     fieldsWriter.startDocument();
@@ -342,7 +342,8 @@ final class DocFieldProcessor extends Do
       }
     }
     DocValuesConsumer docValuesConsumer = perDocConsumer.addValuesField(valueType, fieldInfo);
-    fieldInfo.setDocValuesType(valueType, false);
+    assert fieldInfo.getDocValuesType() == null || fieldInfo.getDocValuesType() == valueType;
+    fieldInfo.setDocValuesType(valueType);
 
     docValuesConsumerAndDocID = new DocValuesConsumerAndDocID(docValuesConsumer);
     docValuesConsumerAndDocID.docID = docState.docID;

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverter.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverter.java Mon May 28 19:28:10 2012
@@ -39,12 +39,12 @@ final class DocInverter extends DocField
   }
 
   @Override
-  void flush(Map<FieldInfo, DocFieldConsumerPerField> fieldsToFlush, SegmentWriteState state) throws IOException {
+  void flush(Map<String, DocFieldConsumerPerField> fieldsToFlush, SegmentWriteState state) throws IOException {
 
-    Map<FieldInfo, InvertedDocConsumerPerField> childFieldsToFlush = new HashMap<FieldInfo, InvertedDocConsumerPerField>();
-    Map<FieldInfo, InvertedDocEndConsumerPerField> endChildFieldsToFlush = new HashMap<FieldInfo, InvertedDocEndConsumerPerField>();
+    Map<String, InvertedDocConsumerPerField> childFieldsToFlush = new HashMap<String, InvertedDocConsumerPerField>();
+    Map<String, InvertedDocEndConsumerPerField> endChildFieldsToFlush = new HashMap<String, InvertedDocEndConsumerPerField>();
 
-    for (Map.Entry<FieldInfo, DocFieldConsumerPerField> fieldToFlush : fieldsToFlush.entrySet()) {
+    for (Map.Entry<String, DocFieldConsumerPerField> fieldToFlush : fieldsToFlush.entrySet()) {
       DocInverterPerField perField = (DocInverterPerField) fieldToFlush.getValue();
       childFieldsToFlush.put(fieldToFlush.getKey(), perField.consumer);
       endChildFieldsToFlush.put(fieldToFlush.getKey(), perField.endConsumer);

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java Mon May 28 19:28:10 2012
@@ -28,7 +28,7 @@ import org.apache.lucene.index.Documents
 import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment;
 import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain;
 import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState;
-import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
+import org.apache.lucene.index.FieldInfos.FieldNumbers;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.store.AlreadyClosedException;
@@ -133,7 +133,7 @@ final class DocumentsWriter {
   final DocumentsWriterFlushControl flushControl;
   
   final Codec codec;
-  DocumentsWriter(Codec codec, IndexWriterConfig config, Directory directory, IndexWriter writer, FieldNumberBiMap globalFieldNumbers,
+  DocumentsWriter(Codec codec, IndexWriterConfig config, Directory directory, IndexWriter writer, FieldNumbers globalFieldNumbers,
       BufferedDeletesStream bufferedDeletesStream) throws IOException {
     this.codec = codec;
     this.directory = directory;
@@ -494,7 +494,8 @@ final class DocumentsWriter {
   private void publishFlushedSegment(FlushedSegment newSegment, FrozenBufferedDeletes globalPacket)
       throws IOException {
     assert newSegment != null;
-    final SegmentInfo segInfo = indexWriter.prepareFlushedSegment(newSegment);
+    assert newSegment.segmentInfo != null;
+    final SegmentInfoPerCommit segInfo = indexWriter.prepareFlushedSegment(newSegment);
     final BufferedDeletes deletes = newSegment.segmentDeletes;
     if (infoStream.isEnabled("DW")) {
       infoStream.message("DW", Thread.currentThread().getName() + ": publishFlushedSegment seg-private deletes=" + deletes);  

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java Mon May 28 19:28:10 2012
@@ -170,7 +170,7 @@ class DocumentsWriterFlushQueue {
     protected abstract boolean canPublish();
   }
   
-  static final class GlobalDeletesTicket extends FlushTicket{
+  static final class GlobalDeletesTicket extends FlushTicket {
 
     protected GlobalDeletesTicket(FrozenBufferedDeletes frozenDeletes) {
       super(frozenDeletes);

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Mon May 28 19:28:10 2012
@@ -17,11 +17,9 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
-import static org.apache.lucene.util.ByteBlockPool.BYTE_BLOCK_MASK;
-import static org.apache.lucene.util.ByteBlockPool.BYTE_BLOCK_SIZE;
-
 import java.io.IOException;
 import java.text.NumberFormat;
+import java.util.HashSet;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.codecs.Codec;
@@ -30,12 +28,17 @@ import org.apache.lucene.search.similari
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FlushInfo;
 import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.Counter;
+import org.apache.lucene.store.TrackingDirectoryWrapper;
 import org.apache.lucene.util.ByteBlockPool.Allocator;
 import org.apache.lucene.util.ByteBlockPool.DirectTrackingAllocator;
+import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.Counter;
 import org.apache.lucene.util.InfoStream;
-import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.MutableBits;
+import org.apache.lucene.util.RamUsageEstimator;
+
+import static org.apache.lucene.util.ByteBlockPool.BYTE_BLOCK_MASK;
+import static org.apache.lucene.util.ByteBlockPool.BYTE_BLOCK_SIZE;
 
 class DocumentsWriterPerThread {
 
@@ -112,14 +115,16 @@ class DocumentsWriterPerThread {
   }
 
   static class FlushedSegment {
-    final SegmentInfo segmentInfo;
+    final SegmentInfoPerCommit segmentInfo;
+    final FieldInfos fieldInfos;
     final BufferedDeletes segmentDeletes;
     final MutableBits liveDocs;
     final int delCount;
 
-    private FlushedSegment(SegmentInfo segmentInfo,
+    private FlushedSegment(SegmentInfoPerCommit segmentInfo, FieldInfos fieldInfos,
                            BufferedDeletes segmentDeletes, MutableBits liveDocs, int delCount) {
       this.segmentInfo = segmentInfo;
+      this.fieldInfos = fieldInfos;
       this.segmentDeletes = segmentDeletes;
       this.liveDocs = liveDocs;
       this.delCount = delCount;
@@ -157,7 +162,8 @@ class DocumentsWriterPerThread {
   final DocumentsWriter parent;
   final Codec codec;
   final IndexWriter writer;
-  final Directory directory;
+  final TrackingDirectoryWrapper directory;
+  final Directory directoryOrig;
   final DocState docState;
   final DocConsumer consumer;
   final Counter bytesUsed;
@@ -165,11 +171,11 @@ class DocumentsWriterPerThread {
   SegmentWriteState flushState;
   //Deletes for our still-in-RAM (to be flushed next) segment
   BufferedDeletes pendingDeletes;  
-  String segment;     // Current segment we are working on
+  SegmentInfo segmentInfo;     // Current segment we are working on
   boolean aborting = false;   // True if an abort is pending
   boolean hasAborted = false; // True if the last exception throws by #updateDocument was aborting
 
-  private FieldInfos fieldInfos;
+  private FieldInfos.Builder fieldInfos;
   private final InfoStream infoStream;
   private int numDocsInRAM;
   private int flushedDocCount;
@@ -180,8 +186,9 @@ class DocumentsWriterPerThread {
 
   
   public DocumentsWriterPerThread(Directory directory, DocumentsWriter parent,
-      FieldInfos fieldInfos, IndexingChain indexingChain) {
-    this.directory = directory;
+      FieldInfos.Builder fieldInfos, IndexingChain indexingChain) {
+    this.directoryOrig = directory;
+    this.directory = new TrackingDirectoryWrapper(directory);
     this.parent = parent;
     this.fieldInfos = fieldInfos;
     this.writer = parent.indexWriter;
@@ -196,8 +203,8 @@ class DocumentsWriterPerThread {
     initialize();
   }
   
-  public DocumentsWriterPerThread(DocumentsWriterPerThread other, FieldInfos fieldInfos) {
-    this(other.directory, other.parent, fieldInfos, other.parent.chain);
+  public DocumentsWriterPerThread(DocumentsWriterPerThread other, FieldInfos.Builder fieldInfos) {
+    this(other.directoryOrig, other.parent, fieldInfos, other.parent.chain);
   }
   
   void initialize() {
@@ -223,17 +230,11 @@ class DocumentsWriterPerThread {
     docState.doc = doc;
     docState.analyzer = analyzer;
     docState.docID = numDocsInRAM;
-    if (segment == null) {
-      // this call is synchronized on IndexWriter.segmentInfos
-      segment = writer.newSegmentName();
-      assert numDocsInRAM == 0;
-      if (INFO_VERBOSE && infoStream.isEnabled("DWPT")) {
-        infoStream.message("DWPT", Thread.currentThread().getName() + " init seg=" + segment + " delQueue=" + deleteQueue);  
-      }
-      
+    if (segmentInfo == null) {
+      initSegmentInfo();
     }
     if (INFO_VERBOSE && infoStream.isEnabled("DWPT")) {
-      infoStream.message("DWPT", Thread.currentThread().getName() + " update delTerm=" + delTerm + " docID=" + docState.docID + " seg=" + segment);
+      infoStream.message("DWPT", Thread.currentThread().getName() + " update delTerm=" + delTerm + " docID=" + docState.docID + " seg=" + segmentInfo.name);
     }
     boolean success = false;
     try {
@@ -265,21 +266,26 @@ class DocumentsWriterPerThread {
     }
     finishDocument(delTerm);
   }
+
+  private void initSegmentInfo() {
+    String segment = writer.newSegmentName();
+    segmentInfo = new SegmentInfo(directoryOrig, Constants.LUCENE_MAIN_VERSION, segment, -1,
+                                  false, codec, null, null);
+    assert numDocsInRAM == 0;
+    if (INFO_VERBOSE && infoStream.isEnabled("DWPT")) {
+      infoStream.message("DWPT", Thread.currentThread().getName() + " init seg=" + segment + " delQueue=" + deleteQueue);  
+    }
+  }
   
   public int updateDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs, Analyzer analyzer, Term delTerm) throws IOException {
     assert writer.testPoint("DocumentsWriterPerThread addDocuments start");
     assert deleteQueue != null;
     docState.analyzer = analyzer;
-    if (segment == null) {
-      // this call is synchronized on IndexWriter.segmentInfos
-      segment = writer.newSegmentName();
-      assert numDocsInRAM == 0;
-      if (INFO_VERBOSE && infoStream.isEnabled("DWPT")) {
-        infoStream.message("DWPT", Thread.currentThread().getName() + " init seg=" + segment + " delQueue=" + deleteQueue);  
-      }
+    if (segmentInfo == null) {
+      initSegmentInfo();
     }
     if (INFO_VERBOSE && infoStream.isEnabled("DWPT")) {
-      infoStream.message("DWPT", Thread.currentThread().getName() + " update delTerm=" + delTerm + " docID=" + docState.docID + " seg=" + segment);
+      infoStream.message("DWPT", Thread.currentThread().getName() + " update delTerm=" + delTerm + " docID=" + docState.docID + " seg=" + segmentInfo.name);
     }
     int docCount = 0;
     try {
@@ -405,15 +411,12 @@ class DocumentsWriterPerThread {
     return numDocsInRAM;
   }
 
-  Codec getCodec() {
-    return flushState.codec;
-  }
-
   /** Reset after a flush */
   private void doAfterFlush() throws IOException {
-    segment = null;
+    segmentInfo = null;
     consumer.doAfterFlush();
-    fieldInfos = FieldInfos.from(fieldInfos);
+    directory.getCreatedFiles().clear();
+    fieldInfos = new FieldInfos.Builder(fieldInfos.globalFieldNumbers);
     parent.subtractFlushedNumDocs(numDocsInRAM);
     numDocsInRAM = 0;
   }
@@ -441,10 +444,12 @@ class DocumentsWriterPerThread {
   FlushedSegment flush() throws IOException {
     assert numDocsInRAM > 0;
     assert deleteSlice == null : "all deletes must be applied in prepareFlush";
-    flushState = new SegmentWriteState(infoStream, directory, segment, fieldInfos,
-        numDocsInRAM, writer.getConfig().getTermIndexInterval(),
-        codec, pendingDeletes, new IOContext(new FlushInfo(numDocsInRAM, bytesUsed())));
+    segmentInfo.setDocCount(numDocsInRAM);
+    flushState = new SegmentWriteState(infoStream, directory, segmentInfo, fieldInfos.finish(),
+        writer.getConfig().getTermIndexInterval(),
+        pendingDeletes, new IOContext(new FlushInfo(numDocsInRAM, bytesUsed())));
     final double startMBUsed = parent.flushControl.netBytes() / 1024. / 1024.;
+
     // Apply delete-by-docID now (delete-byDocID only
     // happens when an exception is hit processing that
     // doc, eg if analyzer has some problem w/ the text):
@@ -459,7 +464,7 @@ class DocumentsWriterPerThread {
     }
 
     if (infoStream.isEnabled("DWPT")) {
-      infoStream.message("DWPT", "flush postings as segment " + flushState.segmentName + " numDocs=" + numDocsInRAM);
+      infoStream.message("DWPT", "flush postings as segment " + flushState.segmentInfo.name + " numDocs=" + numDocsInRAM);
     }
 
     if (aborting) {
@@ -474,14 +479,22 @@ class DocumentsWriterPerThread {
     try {
       consumer.flush(flushState);
       pendingDeletes.terms.clear();
-      final SegmentInfo newSegment = new SegmentInfo(segment, flushState.numDocs, directory, false, flushState.codec, fieldInfos.asReadOnly());
+      segmentInfo.setFiles(new HashSet<String>(directory.getCreatedFiles()));
+
+      final SegmentInfoPerCommit segmentInfoPerCommit = new SegmentInfoPerCommit(segmentInfo, 0, -1L);
       if (infoStream.isEnabled("DWPT")) {
-        infoStream.message("DWPT", "new segment has " + (flushState.liveDocs == null ? 0 : (flushState.numDocs - flushState.delCountOnFlush)) + " deleted docs");
-        infoStream.message("DWPT", "new segment has " + (newSegment.getHasVectors() ? "vectors" : "no vectors"));
-        infoStream.message("DWPT", "flushedFiles=" + newSegment.files());
-        infoStream.message("DWPT", "flushed codec=" + newSegment.getCodec());
+        infoStream.message("DWPT", "new segment has " + (flushState.liveDocs == null ? 0 : (flushState.segmentInfo.getDocCount() - flushState.delCountOnFlush)) + " deleted docs");
+        infoStream.message("DWPT", "new segment has " +
+                           (flushState.fieldInfos.hasVectors() ? "vectors" : "no vectors") + "; " +
+                           (flushState.fieldInfos.hasNorms() ? "norms" : "no norms") + "; " + 
+                           (flushState.fieldInfos.hasDocValues() ? "docValues" : "no docValues") + "; " + 
+                           (flushState.fieldInfos.hasProx() ? "prox" : "no prox") + "; " + 
+                           (flushState.fieldInfos.hasFreq() ? "freqs" : "no freqs"));
+        infoStream.message("DWPT", "flushedFiles=" + segmentInfoPerCommit.files());
+        infoStream.message("DWPT", "flushed codec=" + codec);
       }
-      flushedDocCount += flushState.numDocs;
+
+      flushedDocCount += flushState.segmentInfo.getDocCount();
 
       final BufferedDeletes segmentDeletes;
       if (pendingDeletes.queries.isEmpty()) {
@@ -493,21 +506,26 @@ class DocumentsWriterPerThread {
       }
 
       if (infoStream.isEnabled("DWPT")) {
-        final double newSegmentSize = newSegment.sizeInBytes()/1024./1024.;
-        infoStream.message("DWPT", "flushed: segment=" + newSegment + 
+        final double newSegmentSize = segmentInfo.sizeInBytes()/1024./1024.;
+        infoStream.message("DWPT", "flushed: segment=" + segmentInfo.name + 
                 " ramUsed=" + nf.format(startMBUsed) + " MB" +
                 " newFlushedSize(includes docstores)=" + nf.format(newSegmentSize) + " MB" +
                 " docs/MB=" + nf.format(flushedDocCount / newSegmentSize));
       }
+
+      assert segmentInfo != null;
+
+      FlushedSegment fs = new FlushedSegment(segmentInfoPerCommit, flushState.fieldInfos,
+                                             segmentDeletes, flushState.liveDocs, flushState.delCountOnFlush);
       doAfterFlush();
       success = true;
 
-      return new FlushedSegment(newSegment, segmentDeletes, flushState.liveDocs, flushState.delCountOnFlush);
+      return fs;
     } finally {
       if (!success) {
-        if (segment != null) {
+        if (segmentInfo != null) {
           synchronized(parent.indexWriter) {
-            parent.indexWriter.deleter.refresh(segment);
+            parent.indexWriter.deleter.refresh(segmentInfo.name);
           }
         }
         abort();
@@ -515,9 +533,9 @@ class DocumentsWriterPerThread {
     }
   }
 
-  /** Get current segment name we are writing. */
-  String getSegment() {
-    return segment;
+  /** Get current segment info we are writing. */
+  SegmentInfo getSegmentInfo() {
+    return segmentInfo;
   }
 
   long bytesUsed() {
@@ -550,14 +568,14 @@ class DocumentsWriterPerThread {
   }
 
   PerDocWriteState newPerDocWriteState(String segmentSuffix) {
-    assert segment != null;
-    return new PerDocWriteState(infoStream, directory, segment, fieldInfos, bytesUsed, segmentSuffix, IOContext.DEFAULT);
+    assert segmentInfo != null;
+    return new PerDocWriteState(infoStream, directory, segmentInfo, bytesUsed, segmentSuffix, IOContext.DEFAULT);
   }
   
   @Override
   public String toString() {
     return "DocumentsWriterPerThread [pendingDeletes=" + pendingDeletes
-        + ", segment=" + segment + ", aborting=" + aborting + ", numDocsInRAM="
+      + ", segment=" + (segmentInfo != null ? segmentInfo.name : "null") + ", aborting=" + aborting + ", numDocsInRAM="
         + numDocsInRAM + ", deleteQueue=" + deleteQueue + "]";
   }
 }

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java Mon May 28 19:28:10 2012
@@ -18,7 +18,7 @@ package org.apache.lucene.index;
 
 import java.util.concurrent.locks.ReentrantLock;
 
-import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
+import org.apache.lucene.index.FieldInfos.FieldNumbers;
 import org.apache.lucene.util.SetOnce;
 
 /**
@@ -121,7 +121,7 @@ abstract class DocumentsWriterPerThreadP
 
   private final ThreadState[] threadStates;
   private volatile int numThreadStatesActive;
-  private final SetOnce<FieldNumberBiMap> globalFieldMap = new SetOnce<FieldNumberBiMap>();
+  private final SetOnce<FieldNumbers> globalFieldMap = new SetOnce<FieldNumbers>();
   private final SetOnce<DocumentsWriter> documentsWriter = new SetOnce<DocumentsWriter>();
   
   /**
@@ -135,11 +135,11 @@ abstract class DocumentsWriterPerThreadP
     numThreadStatesActive = 0;
   }
 
-  void initialize(DocumentsWriter documentsWriter, FieldNumberBiMap globalFieldMap, IndexWriterConfig config) {
+  void initialize(DocumentsWriter documentsWriter, FieldNumbers globalFieldMap, IndexWriterConfig config) {
     this.documentsWriter.set(documentsWriter); // thread pool is bound to DW
     this.globalFieldMap.set(globalFieldMap);
     for (int i = 0; i < threadStates.length; i++) {
-      final FieldInfos infos = new FieldInfos(globalFieldMap);
+      final FieldInfos.Builder infos = new FieldInfos.Builder(globalFieldMap);
       threadStates[i] = new ThreadState(new DocumentsWriterPerThread(documentsWriter.directory, documentsWriter, infos, documentsWriter.chain));
     }
   }
@@ -228,7 +228,7 @@ abstract class DocumentsWriterPerThreadP
     assert globalFieldMap.get() != null;
     final DocumentsWriterPerThread dwpt = threadState.dwpt;
     if (!closed) {
-      final FieldInfos infos = new FieldInfos(globalFieldMap.get());
+      final FieldInfos.Builder infos = new FieldInfos.Builder(globalFieldMap.get());
       final DocumentsWriterPerThread newDwpt = new DocumentsWriterPerThread(dwpt, infos);
       newDwpt.initialize();
       threadState.resetWriter(newDwpt);

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java Mon May 28 19:28:10 2012
@@ -1,8 +1,5 @@
 package org.apache.lucene.index;
 
-import org.apache.lucene.index.DocValues.Type;
-
-
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -20,6 +17,11 @@ import org.apache.lucene.index.DocValues
  * limitations under the License.
  */
 
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.index.DocValues.Type;
+
 /**
  *  Access to the Fieldable Info file that describes document fields and whether or
  *  not they are indexed. Each segment has a separate Fieldable Info file. Objects
@@ -27,20 +29,23 @@ import org.apache.lucene.index.DocValues
  *  be adding documents at a time, with no other reader or writer threads
  *  accessing this object.
  **/
+
 public final class FieldInfo {
   public final String name;
   public final int number;
 
-  public boolean isIndexed;
+  private boolean indexed;
   private DocValues.Type docValueType;
 
   // True if any document indexed term vectors
-  public boolean storeTermVector;
+  private boolean storeTermVector;
 
   private DocValues.Type normType;
-  public boolean omitNorms; // omit norms associated with indexed fields  
-  public IndexOptions indexOptions;
-  public boolean storePayloads; // whether this field stores payloads together with term positions
+  private boolean omitNorms; // omit norms associated with indexed fields  
+  private IndexOptions indexOptions;
+  private boolean storePayloads; // whether this field stores payloads together with term positions
+
+  private Map<String,String> attributes;
 
   /**
    * Controls how much information is stored in the postings lists.
@@ -64,13 +69,13 @@ public final class FieldInfo {
   /**
    * @lucene.experimental
    */
-  public FieldInfo(String name, boolean isIndexed, int number, boolean storeTermVector, 
-            boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues, DocValues.Type normsType) {
+  public FieldInfo(String name, boolean indexed, int number, boolean storeTermVector, 
+            boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues, DocValues.Type normsType, Map<String,String> attributes) {
     this.name = name;
-    this.isIndexed = isIndexed;
+    this.indexed = indexed;
     this.number = number;
     this.docValueType = docValues;
-    if (isIndexed) {
+    if (indexed) {
       this.storeTermVector = storeTermVector;
       this.storePayloads = storePayloads;
       this.omitNorms = omitNorms;
@@ -83,22 +88,38 @@ public final class FieldInfo {
       this.indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
       this.normType = null;
     }
+    this.attributes = attributes;
+    assert checkConsistency();
     assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !storePayloads;
   }
-  
-  @Override
-  public FieldInfo clone() {
-    return new FieldInfo(name, isIndexed, number, storeTermVector,
-                         omitNorms, storePayloads, indexOptions, docValueType, normType);
+
+  private boolean checkConsistency() {
+    if (!indexed) {
+      assert !storeTermVector;
+      assert !storePayloads;
+      assert !omitNorms;
+      assert normType == null;
+      assert indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
+    } else {
+      assert indexOptions != null;
+      if (omitNorms) {
+        assert normType == null;
+      }
+    }
+
+    // Cannot store payloads unless positions are indexed:
+    assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !this.storePayloads;
+
+    return true;
   }
 
   // should only be called by FieldInfos#addOrUpdate
-  void update(boolean isIndexed, boolean storeTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions) {
+  void update(boolean indexed, boolean storeTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions) {
 
-    if (this.isIndexed != isIndexed) {
-      this.isIndexed = true;                      // once indexed, always index
+    if (this.indexed != indexed) {
+      this.indexed = true;                      // once indexed, always index
     }
-    if (isIndexed) { // if updated field data is not for indexing, leave the updates out
+    if (indexed) { // if updated field data is not for indexing, leave the updates out
       if (this.storeTermVector != storeTermVector) {
         this.storeTermVector = true;                // once vector, always vector
       }
@@ -107,6 +128,7 @@ public final class FieldInfo {
       }
       if (this.omitNorms != omitNorms) {
         this.omitNorms = true;                // if one require omitNorms at least once, it remains off for life
+        this.normType = null;
       }
       if (this.indexOptions != indexOptions) {
         // downgrade
@@ -118,14 +140,17 @@ public final class FieldInfo {
       }
     }
     assert this.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !this.storePayloads;
+    assert checkConsistency();
   }
 
-  void setDocValuesType(DocValues.Type type, boolean force) {
-    if (docValueType == null || force) {
-      docValueType = type;
-    } else if (type != docValueType) {
-      throw new IllegalArgumentException("DocValues type already set to " + docValueType + " but was: " + type);
-    }
+  void setDocValuesType(DocValues.Type type) {
+    docValueType = type;
+    assert checkConsistency();
+  }
+  
+  /** @return IndexOptions for the field */
+  public IndexOptions getIndexOptions() {
+    return indexOptions;
   }
   
   /**
@@ -149,22 +174,27 @@ public final class FieldInfo {
     return normType;
   }
 
-  public void setStoreTermVectors() {
+  void setStoreTermVectors() {
     storeTermVector = true;
+    assert checkConsistency();
   }
-
-  public void setNormValueType(Type type, boolean force) {
-    if (normType == null || force) {
-      normType = type;
-    } else if (type != normType) {
-      throw new IllegalArgumentException("Norm type already set to " + normType);
+  
+  void setStorePayloads() {
+    if (indexed && indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) {
+      storePayloads = true;
     }
+    assert checkConsistency();
+  }
+
+  void setNormValueType(Type type) {
+    normType = type;
+    assert checkConsistency();
   }
   
   /**
    * @return true if norms are explicitly omitted for this field
    */
-  public boolean omitNorms() {
+  public boolean omitsNorms() {
     return omitNorms;
   }
   
@@ -172,7 +202,62 @@ public final class FieldInfo {
    * @return true if this field actually has any norms.
    */
   public boolean hasNorms() {
-    return isIndexed && !omitNorms && normType != null;
+    return normType != null;
+  }
+  
+  /**
+   * @return true if this field is indexed.
+   */
+  public boolean isIndexed() {
+    return indexed;
+  }
+  
+  /**
+   * @return true if any payloads exist for this field.
+   */
+  public boolean hasPayloads() {
+    return storePayloads;
   }
   
+  /**
+   * @return true if any term vectors exist for this field.
+   */
+  public boolean hasVectors() {
+    return storeTermVector;
+  }
+  
+  /**
+   * Get a codec attribute value, or null if it does not exist
+   */
+  public String getAttribute(String key) {
+    if (attributes == null) {
+      return null;
+    } else {
+      return attributes.get(key);
+    }
+  }
+  
+  /**
+   * Puts a codec attribute value.
+   * <p>
+   * This is a key-value mapping for the field that the codec can use
+   * to store additional metadata, and will be available to the codec
+   * when reading the segment via {@link #getAttribute(String)}
+   * <p>
+   * If a value already exists for the field, it will be replaced with 
+   * the new value.
+   */
+  public String putAttribute(String key, String value) {
+    if (attributes == null) {
+      attributes = new HashMap<String,String>();
+    }
+    return attributes.put(key, value);
+  }
+  
+  /**
+   * @return internal codec attributes map. May be null if no mappings exist.
+   */
+  public Map<String,String> attributes() {
+    return attributes;
+  }
 }

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java Mon May 28 19:28:10 2012
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
  */
 
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
@@ -30,14 +31,119 @@ import org.apache.lucene.index.FieldInfo
  * Collection of {@link FieldInfo}s (accessible by number or by name).
  *  @lucene.experimental
  */
-public final class FieldInfos implements Iterable<FieldInfo> {
-  static final class FieldNumberBiMap {
+public class FieldInfos implements Iterable<FieldInfo> {
+  private final boolean hasFreq;
+  private final boolean hasProx;
+  private final boolean hasVectors;
+  private final boolean hasNorms;
+  private final boolean hasDocValues;
+  
+  private final SortedMap<Integer,FieldInfo> byNumber = new TreeMap<Integer,FieldInfo>();
+  private final HashMap<String,FieldInfo> byName = new HashMap<String,FieldInfo>();
+  private final Collection<FieldInfo> values; // for an unmodifiable iterator
+  
+  public FieldInfos(FieldInfo[] infos) {
+    boolean hasVectors = false;
+    boolean hasProx = false;
+    boolean hasFreq = false;
+    boolean hasNorms = false;
+    boolean hasDocValues = false;
+    
+    for (FieldInfo info : infos) {
+      assert !byNumber.containsKey(info.number);
+      byNumber.put(info.number, info);
+      assert !byName.containsKey(info.name);
+      byName.put(info.name, info);
+      
+      hasVectors |= info.hasVectors();
+      hasProx |= info.isIndexed() && info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+      hasFreq |= info.isIndexed() && info.getIndexOptions() != IndexOptions.DOCS_ONLY;
+      hasNorms |= info.hasNorms();
+      hasDocValues |= info.hasDocValues();
+    }
+    
+    this.hasVectors = hasVectors;
+    this.hasProx = hasProx;
+    this.hasFreq = hasFreq;
+    this.hasNorms = hasNorms;
+    this.hasDocValues = hasDocValues;
+    this.values = Collections.unmodifiableCollection(byNumber.values());
+  }
+  
+  /** Returns true if any fields have freqs */
+  public boolean hasFreq() {
+    return hasFreq;
+  }
+  
+  /** Returns true if any fields have positions */
+  public boolean hasProx() {
+    return hasProx;
+  }
+  
+  /**
+   * @return true if at least one field has any vectors
+   */
+  public boolean hasVectors() {
+    return hasVectors;
+  }
+  
+  /**
+   * @return true if at least one field has any norms
+   */
+  public boolean hasNorms() {
+    return hasNorms;
+  }
+  
+  /**
+   * @return true if at least one field has doc values
+   */
+  public boolean hasDocValues() {
+    return hasDocValues;
+  }
+  
+  /**
+   * @return number of fields
+   */
+  public int size() {
+    assert byNumber.size() == byName.size();
+    return byNumber.size();
+  }
+  
+  /**
+   * Returns an iterator over all the fieldinfo objects present,
+   * ordered by ascending field number
+   */
+  // TODO: what happens if in fact a different order is used?
+  public Iterator<FieldInfo> iterator() {
+    return values.iterator();
+  }
+
+  /**
+   * Return the fieldinfo object referenced by the field name
+   * @return the FieldInfo object or null when the given fieldName
+   * doesn't exist.
+   */  
+  public FieldInfo fieldInfo(String fieldName) {
+    return byName.get(fieldName);
+  }
+
+  /**
+   * Return the fieldinfo object referenced by the fieldNumber.
+   * @param fieldNumber
+   * @return the FieldInfo object or null when the given fieldNumber
+   * doesn't exist.
+   */  
+  public FieldInfo fieldInfo(int fieldNumber) {
+    return (fieldNumber >= 0) ? byNumber.get(fieldNumber) : null;
+  }
+  
+  static final class FieldNumbers {
     
     private final Map<Integer,String> numberToName;
     private final Map<String,Integer> nameToNumber;
     private int lowestUnassignedFieldNumber = -1;
     
-    FieldNumberBiMap() {
+    FieldNumbers() {
       this.nameToNumber = new HashMap<String, Integer>();
       this.numberToName = new HashMap<Integer, String>();
     }
@@ -54,8 +160,8 @@ public final class FieldInfos implements
         final Integer preferredBoxed = Integer.valueOf(preferredFieldNumber);
 
         if (preferredFieldNumber != -1 && !numberToName.containsKey(preferredBoxed)) {
-            // cool - we can use this number globally
-            fieldNumber = preferredBoxed;
+          // cool - we can use this number globally
+          fieldNumber = preferredBoxed;
         } else {
           // find a new FieldNumber
           while (numberToName.containsKey(++lowestUnassignedFieldNumber)) {
@@ -66,7 +172,6 @@ public final class FieldInfos implements
         
         numberToName.put(fieldNumber, fieldName);
         nameToNumber.put(fieldName, fieldNumber);
-        
       }
 
       return fieldNumber.intValue();
@@ -93,386 +198,115 @@ public final class FieldInfos implements
     }
   }
   
-  private final SortedMap<Integer,FieldInfo> byNumber = new TreeMap<Integer,FieldInfo>();
-  private final HashMap<String,FieldInfo> byName = new HashMap<String,FieldInfo>();
-  private final FieldNumberBiMap globalFieldNumbers;
-  
-  private boolean hasFreq; // only set if readonly
-  private boolean hasProx; // only set if readonly
-  private boolean hasVectors; // only set if readonly
-  private long version; // internal use to track changes
-
-  /**
-   * Creates a new read-only FieldInfos: only public to be accessible
-   * from the codecs package
-   * 
-   * @lucene.internal
-   */
-  public FieldInfos(FieldInfo[] infos, boolean hasFreq, boolean hasProx, boolean hasVectors) {
-    this(null);
-    this.hasFreq = hasFreq;
-    this.hasProx = hasProx;
-    this.hasVectors = hasVectors;
-    for (FieldInfo info : infos) {
-      putInternal(info);
-    }
-  }
-
-  public FieldInfos() {
-    this(new FieldNumberBiMap());
-  }
+  static final class Builder {
+    private final HashMap<String,FieldInfo> byName = new HashMap<String,FieldInfo>();
+    final FieldNumbers globalFieldNumbers;
 
-  public void add(FieldInfos other) {
-    for(FieldInfo fieldInfo : other){ 
-      add(fieldInfo);
+    Builder() {
+      this(new FieldNumbers());
     }
-  }
-
-  /**
-   * Creates a new FieldInfos instance with the given {@link FieldNumberBiMap}. 
-   * If the {@link FieldNumberBiMap} is <code>null</code> this instance will be read-only.
-   * @see #isReadOnly()
-   */
-  FieldInfos(FieldNumberBiMap globalFieldNumbers) {
-    this.globalFieldNumbers = globalFieldNumbers;
-  }
-  
-  /**
-   * adds the given field to this FieldInfos name / number mapping. The given FI
-   * must be present in the global field number mapping before this method it
-   * called
-   */
-  private void putInternal(FieldInfo fi) {
-    assert !byNumber.containsKey(fi.number);
-    assert !byName.containsKey(fi.name);
-    assert globalFieldNumbers == null || globalFieldNumbers.containsConsistent(Integer.valueOf(fi.number), fi.name);
-    byNumber.put(fi.number, fi);
-    byName.put(fi.name, fi);
-  }
-  
-  private int nextFieldNumber(String name, int preferredFieldNumber) {
-    // get a global number for this field
-    final int fieldNumber = globalFieldNumbers.addOrGet(name,
-        preferredFieldNumber);
-    assert byNumber.get(fieldNumber) == null : "field number " + fieldNumber
-        + " already taken";
-    return fieldNumber;
-  }
-
-  /**
-   * Returns a deep clone of this FieldInfos instance.
-   */
-  @Override
-  synchronized public FieldInfos clone() {
-    FieldInfos fis = new FieldInfos(globalFieldNumbers);
-    fis.hasFreq = hasFreq;
-    fis.hasProx = hasProx;
-    fis.hasVectors = hasVectors;
-    for (FieldInfo fi : this) {
-      FieldInfo clone = fi.clone();
-      fis.putInternal(clone);
+    
+    /**
+     * Creates a new instance with the given {@link FieldNumbers}. 
+     */
+    Builder(FieldNumbers globalFieldNumbers) {
+      assert globalFieldNumbers != null;
+      this.globalFieldNumbers = globalFieldNumbers;
     }
-    return fis;
-  }
 
-  /** Returns true if any fields have positions */
-  public boolean hasProx() {
-    if (isReadOnly()) {
-      return hasProx;
-    }
-    // mutable FIs must check!
-    for (FieldInfo fi : this) {
-      if (fi.isIndexed && fi.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) {
-        return true;
+    public void add(FieldInfos other) {
+      for(FieldInfo fieldInfo : other){ 
+        add(fieldInfo);
       }
     }
-    return false;
-  }
-  
-  /** Returns true if any fields have freqs */
-  public boolean hasFreq() {
-    if (isReadOnly()) {
-      return hasFreq;
-    }
-    // mutable FIs must check!
-    for (FieldInfo fi : this) {
-      if (fi.isIndexed && fi.indexOptions != IndexOptions.DOCS_ONLY) {
-        return true;
-      }
-    }
-    return false;
-  }
-  
-  /**
-   * Adds or updates fields that are indexed. Whether they have termvectors has to be specified.
-   * 
-   * @param names The names of the fields
-   * @param storeTermVectors Whether the fields store term vectors or not
-   */
-  synchronized public void addOrUpdateIndexed(Collection<String> names, boolean storeTermVectors) {
-    for (String name : names) {
-      addOrUpdate(name, true, storeTermVectors);
-    }
-  }
-
-  /**
-   * Assumes the fields are not storing term vectors.
-   * 
-   * @param names The names of the fields
-   * @param isIndexed Whether the fields are indexed or not
-   * 
-   * @see #addOrUpdate(String, boolean)
-   */
-  synchronized public void addOrUpdate(Collection<String> names, boolean isIndexed) {
-    for (String name : names) {
-      addOrUpdate(name, isIndexed);
+   
+    /**
+     * adds the given field to this FieldInfos name / number mapping. The given FI
+     * must be present in the global field number mapping before this method it
+     * called
+     */
+    private void putInternal(FieldInfo fi) {
+      assert !byName.containsKey(fi.name);
+      assert globalFieldNumbers.containsConsistent(Integer.valueOf(fi.number), fi.name);
+      byName.put(fi.name, fi);
     }
-  }
-
-  /**
-   * Calls 5 parameter add with false for all TermVector parameters.
-   * 
-   * @param name The name of the IndexableField
-   * @param isIndexed true if the field is indexed
-   * @see #addOrUpdate(String, boolean, boolean)
-   */
-  synchronized public void addOrUpdate(String name, boolean isIndexed) {
-    addOrUpdate(name, isIndexed, false, false);
-  }
-
-  /** If the field is not yet known, adds it. If it is known, checks to make
-   *  sure that the isIndexed flag is the same as was given previously for this
-   *  field. If not - marks it as being indexed.  Same goes for the TermVector
-   * parameters.
-   * 
-   * @param name The name of the field
-   * @param isIndexed true if the field is indexed
-   * @param storeTermVector true if the term vector should be stored
-   */
-  synchronized public void addOrUpdate(String name, boolean isIndexed, boolean storeTermVector) {
-    addOrUpdate(name, isIndexed, storeTermVector, false);
-  }
-
+    
     /** If the field is not yet known, adds it. If it is known, checks to make
-   *  sure that the isIndexed flag is the same as was given previously for this
-   *  field. If not - marks it as being indexed.  Same goes for the TermVector
-   * parameters.
-   *
-   * @param name The name of the field
-   * @param isIndexed true if the field is indexed
-   * @param storeTermVector true if the term vector should be stored
-   * @param omitNorms true if the norms for the indexed field should be omitted
-   */
-  synchronized public void addOrUpdate(String name, boolean isIndexed, boolean storeTermVector,
-                  boolean omitNorms) {
-    addOrUpdate(name, isIndexed, storeTermVector, omitNorms, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null, null);
-  }
-  
-  /** If the field is not yet known, adds it. If it is known, checks to make
-   *  sure that the isIndexed flag is the same as was given previously for this
-   *  field. If not - marks it as being indexed.  Same goes for the TermVector
-   * parameters.
-   *
-   * @param name The name of the field
-   * @param isIndexed true if the field is indexed
-   * @param storeTermVector true if the term vector should be stored
-   * @param omitNorms true if the norms for the indexed field should be omitted
-   * @param storePayloads true if payloads should be stored for this field
-   * @param indexOptions if term freqs should be omitted for this field
-   */
-  synchronized public FieldInfo addOrUpdate(String name, boolean isIndexed, boolean storeTermVector,
-                       boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues, DocValues.Type normType) {
-    return addOrUpdateInternal(name, -1, isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, normType);
-  }
-
-  // NOTE: this method does not carry over termVector
-  // booleans nor docValuesType; the indexer chain
-  // (TermVectorsConsumerPerField, DocFieldProcessor) must
-  // set these fields when they succeed in consuming
-  // the document:
-  public FieldInfo addOrUpdate(String name, IndexableFieldType fieldType) {
-    // TODO: really, indexer shouldn't even call this
-    // method (it's only called from DocFieldProcessor);
-    // rather, each component in the chain should update
-    // what it "owns".  EG fieldType.indexOptions() should
-    // be updated by maybe FreqProxTermsWriterPerField:
-    return addOrUpdateInternal(name, -1, fieldType.indexed(), false,
-                               fieldType.omitNorms(), false,
-                               fieldType.indexOptions(), null, null);
-  }
-
-  synchronized private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean isIndexed,
-      boolean storeTermVector,
-      boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues, DocValues.Type normType) {
-    if (globalFieldNumbers == null) {
-      throw new IllegalStateException("FieldInfos are read-only, create a new instance with a global field map to make modifications to FieldInfos");
-    }
-    FieldInfo fi = fieldInfo(name);
-    if (fi == null) {
-      final int fieldNumber = nextFieldNumber(name, preferredFieldNumber);
-      fi = addInternal(name, fieldNumber, isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, normType);
-    } else {
-      fi.update(isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions);
-      if (docValues != null) {
-        fi.setDocValuesType(docValues, true);
-      }
-      if (normType != null) {
-        fi.setNormValueType(normType, true);
+     *  sure that the isIndexed flag is the same as was given previously for this
+     *  field. If not - marks it as being indexed.  Same goes for the TermVector
+     * parameters.
+     *
+     * @param name The name of the field
+     * @param isIndexed true if the field is indexed
+     * @param storeTermVector true if the term vector should be stored
+     * @param omitNorms true if the norms for the indexed field should be omitted
+     * @param storePayloads true if payloads should be stored for this field
+     * @param indexOptions if term freqs should be omitted for this field
+     */
+    // TODO: fix testCodecs to do this another way, its the only user of this
+    FieldInfo addOrUpdate(String name, boolean isIndexed, boolean storeTermVector,
+                         boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues, DocValues.Type normType) {
+      return addOrUpdateInternal(name, -1, isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, normType);
+    }
+
+    // NOTE: this method does not carry over termVector
+    // booleans nor docValuesType; the indexer chain
+    // (TermVectorsConsumerPerField, DocFieldProcessor) must
+    // set these fields when they succeed in consuming
+    // the document:
+    public FieldInfo addOrUpdate(String name, IndexableFieldType fieldType) {
+      // TODO: really, indexer shouldn't even call this
+      // method (it's only called from DocFieldProcessor);
+      // rather, each component in the chain should update
+      // what it "owns".  EG fieldType.indexOptions() should
+      // be updated by maybe FreqProxTermsWriterPerField:
+      return addOrUpdateInternal(name, -1, fieldType.indexed(), false,
+                                 fieldType.omitNorms(), false,
+                                 fieldType.indexOptions(), null, null);
+    }
+
+    private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean isIndexed,
+        boolean storeTermVector,
+        boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues, DocValues.Type normType) {
+      FieldInfo fi = fieldInfo(name);
+      if (fi == null) {
+        // get a global number for this field
+        final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber);
+        fi = addInternal(name, fieldNumber, isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, normType);
+      } else {
+        fi.update(isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions);
+        if (docValues != null) {
+          fi.setDocValuesType(docValues);
+        }
+        if (!fi.omitsNorms() && normType != null) {
+          fi.setNormValueType(normType);
+        }
       }
+      return fi;
     }
-    version++;
-    return fi;
-  }
-  
-  synchronized public FieldInfo add(FieldInfo fi) {
-    // IMPORTANT - reuse the field number if possible for consistent field numbers across segments
-    return addOrUpdateInternal(fi.name, fi.number, fi.isIndexed, fi.storeTermVector,
-               fi.omitNorms, fi.storePayloads,
-               fi.indexOptions, fi.getDocValuesType(), fi.getNormType());
-  }
-  
-  /*
-   * NOTE: if you call this method from a public method make sure you check if we are modifiable and throw an exception otherwise
-   */
-  private FieldInfo addInternal(String name, int fieldNumber, boolean isIndexed,
-                                boolean storeTermVector, boolean omitNorms, boolean storePayloads,
-                                IndexOptions indexOptions, DocValues.Type docValuesType, DocValues.Type normType) {
-    // don't check modifiable here since we use that to initially build up FIs
-    if (globalFieldNumbers != null) {
-      globalFieldNumbers.setIfNotSet(fieldNumber, name);
-    } 
-    final FieldInfo fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, normType);
-    putInternal(fi);
-    return fi;
-  }
-
-  /**
-   * lookup the number of a field by name.
-   * 
-   * @param fieldName field's name
-   * @return number of field, or -1 if it does not exist.
-   */
-  public int fieldNumber(String fieldName) {
-    FieldInfo fi = fieldInfo(fieldName);
-    return (fi != null) ? fi.number : -1;
-  }
-
-  public FieldInfo fieldInfo(String fieldName) {
-    return byName.get(fieldName);
-  }
-
-  /**
-   * Return the fieldName identified by its number.
-   * 
-   * @param fieldNumber
-   * @return the fieldName or an empty string when the field
-   * with the given number doesn't exist.
-   */  
-  public String fieldName(int fieldNumber) {
-  	FieldInfo fi = fieldInfo(fieldNumber);
-  	return (fi != null) ? fi.name : "";
-  }
-
-  /**
-   * Return the fieldinfo object referenced by the fieldNumber.
-   * @param fieldNumber
-   * @return the FieldInfo object or null when the given fieldNumber
-   * doesn't exist.
-   */  
-  public FieldInfo fieldInfo(int fieldNumber) {
-    return (fieldNumber >= 0) ? byNumber.get(fieldNumber) : null;
-  }
-
-  public Iterator<FieldInfo> iterator() {
-    return byNumber.values().iterator();
-  }
-
-  /**
-   * @return number of fields
-   */
-  public int size() {
-    assert byNumber.size() == byName.size();
-    return byNumber.size();
-  }
-
-  /**
-   * @return true if at least one field has any vectors
-   */
-  public boolean hasVectors() {
-    if (isReadOnly()) {
-      return hasVectors;
-    }
-    // mutable FIs must check
-    for (FieldInfo fi : this) {
-      if (fi.storeTermVector) {
-        return true;
-      }
+    
+    public FieldInfo add(FieldInfo fi) {
+      // IMPORTANT - reuse the field number if possible for consistent field numbers across segments
+      return addOrUpdateInternal(fi.name, fi.number, fi.isIndexed(), fi.hasVectors(),
+                 fi.omitsNorms(), fi.hasPayloads(),
+                 fi.getIndexOptions(), fi.getDocValuesType(), fi.getNormType());
     }
-    return false;
-  }
-
-  /**
-   * @return true if at least one field has any norms
-   */
-  public boolean hasNorms() {
-    for (FieldInfo fi : this) {
-      if (fi.hasNorms()) {
-        return true;
-      }
+    
+    private FieldInfo addInternal(String name, int fieldNumber, boolean isIndexed,
+                                  boolean storeTermVector, boolean omitNorms, boolean storePayloads,
+                                  IndexOptions indexOptions, DocValues.Type docValuesType, DocValues.Type normType) {
+      globalFieldNumbers.setIfNotSet(fieldNumber, name);
+      final FieldInfo fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, normType, null);
+      putInternal(fi);
+      return fi;
     }
-    return false;
-  }
 
-  /**
-   * Returns <code>true</code> iff this instance is not backed by a
-   * {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap}. Instances read from a directory via
-   * {@link FieldInfos#FieldInfos(FieldInfo[], boolean, boolean, boolean)} will always be read-only
-   * since no {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap} is supplied, otherwise 
-   * <code>false</code>.
-   */
-  public final boolean isReadOnly() {
-    return globalFieldNumbers == null;
-  }
-  
-  synchronized final long getVersion() {
-    return version;
-  }
-  
-  final FieldInfos asReadOnly() {
-    if (isReadOnly()) {
-      return this;
+    public FieldInfo fieldInfo(String fieldName) {
+      return byName.get(fieldName);
     }
-    final FieldInfos roFis = new FieldInfos((FieldNumberBiMap)null);
-    for (FieldInfo fieldInfo : this) {
-      FieldInfo clone = fieldInfo.clone();
-      roFis.putInternal(clone);
-      roFis.hasVectors |= clone.storeTermVector;
-      roFis.hasProx |= clone.isIndexed && clone.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
-      roFis.hasFreq |= clone.isIndexed && clone.indexOptions != IndexOptions.DOCS_ONLY;
-    }
-    return roFis;
-  }
-
-  /**
-   * @return true if at least one field has docValues
-   */
-  public boolean hasDocValues() {
-    for (FieldInfo fi : this) {
-      if (fi.hasDocValues()) { 
-        return true;
-      }
+    
+    final FieldInfos finish() {
+      return new FieldInfos(byName.values().toArray(new FieldInfo[byName.size()]));
     }
-
-    return false;
-  }
-  
-  /**
-   * Creates a new {@link FieldInfo} instance from the given instance. If the given instance is
-   * read-only this instance will be read-only too.
-   * 
-   * @see #isReadOnly()
-   */
-  static FieldInfos from(FieldInfos other) {
-    return new FieldInfos(other.globalFieldNumbers);
   }
 }

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java Mon May 28 19:28:10 2012
@@ -23,7 +23,6 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.lucene.codecs.FieldsConsumer;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CollectionUtil;
 import org.apache.lucene.util.IOUtils;
@@ -39,7 +38,7 @@ final class FreqProxTermsWriter extends 
   // Other writers would presumably share alot of this...
 
   @Override
-  public void flush(Map<FieldInfo, TermsHashConsumerPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
+  public void flush(Map<String,TermsHashConsumerPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
 
     // Gather all FieldData's that have postings, across all
     // ThreadStates
@@ -57,7 +56,7 @@ final class FreqProxTermsWriter extends 
     // Sort by field name
     CollectionUtil.quickSort(allFields);
 
-    final FieldsConsumer consumer = state.codec.postingsFormat().fieldsConsumer(state);
+    final FieldsConsumer consumer = state.segmentInfo.getCodec().postingsFormat().fieldsConsumer(state);
 
     boolean success = false;
 
@@ -80,13 +79,7 @@ final class FreqProxTermsWriter extends 
         final FieldInfo fieldInfo = allFields.get(fieldNumber).fieldInfo;
         
         final FreqProxTermsWriterPerField fieldWriter = allFields.get(fieldNumber);
-        
-        // Aggregate the storePayload as seen by the same
-        // field across multiple threads
-        if (fieldInfo.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) {
-          fieldInfo.storePayloads |= fieldWriter.hasPayloads;
-        }
-        
+
         // If this field has postings then add them to the
         // segment
         fieldWriter.flush(fieldInfo.name, consumer, state);

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Mon May 28 19:28:10 2012
@@ -55,7 +55,7 @@ final class FreqProxTermsWriterPerField 
     this.fieldInfo = fieldInfo;
     docState = termsHashPerField.docState;
     fieldState = termsHashPerField.fieldState;
-    setIndexOptions(fieldInfo.indexOptions);
+    setIndexOptions(fieldInfo.getIndexOptions());
   }
 
   @Override
@@ -68,7 +68,11 @@ final class FreqProxTermsWriterPerField 
   }
 
   @Override
-  void finish() {}
+  void finish() {
+    if (hasPayloads) {
+      fieldInfo.setStorePayloads();
+    }
+  }
 
   boolean hasPayloads;
 
@@ -83,7 +87,7 @@ final class FreqProxTermsWriterPerField 
   void reset() {
     // Record, up front, whether our in-RAM format will be
     // with or without term freqs:
-    setIndexOptions(fieldInfo.indexOptions);
+    setIndexOptions(fieldInfo.getIndexOptions());
     payloadAttribute = null;
   }
 
@@ -330,7 +334,7 @@ final class FreqProxTermsWriterPerField 
     // according to this.indexOptions, but then write the
     // new segment to the directory according to
     // currentFieldIndexOptions:
-    final IndexOptions currentFieldIndexOptions = fieldInfo.indexOptions;
+    final IndexOptions currentFieldIndexOptions = fieldInfo.getIndexOptions();
 
     final boolean writeTermFreq = currentFieldIndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
     final boolean writePositions = currentFieldIndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
@@ -363,7 +367,7 @@ final class FreqProxTermsWriterPerField 
     final ByteSliceReader freq = new ByteSliceReader();
     final ByteSliceReader prox = new ByteSliceReader();
 
-    FixedBitSet visitedDocs = new FixedBitSet(state.numDocs);
+    FixedBitSet visitedDocs = new FixedBitSet(state.segmentInfo.getDocCount());
     long sumTotalTermFreq = 0;
     long sumDocFreq = 0;
 
@@ -441,7 +445,7 @@ final class FreqProxTermsWriterPerField 
         }
 
         numDocs++;
-        assert docID < state.numDocs: "doc=" + docID + " maxDoc=" + state.numDocs;
+        assert docID < state.segmentInfo.getDocCount(): "doc=" + docID + " maxDoc=" + state.segmentInfo.getDocCount();
 
         // NOTE: we could check here if the docID was
         // deleted, and skip it.  However, this is somewhat
@@ -463,7 +467,7 @@ final class FreqProxTermsWriterPerField 
           
           // TODO: can we do this reach-around in a cleaner way????
           if (state.liveDocs == null) {
-            state.liveDocs = docState.docWriter.codec.liveDocsFormat().newLiveDocs(state.numDocs);
+            state.liveDocs = docState.docWriter.codec.liveDocsFormat().newLiveDocs(state.segmentInfo.getDocCount());
           }
           if (state.liveDocs.get(docID)) {
             state.delCountOnFlush++;

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java Mon May 28 19:28:10 2012
@@ -150,7 +150,7 @@ final class IndexFileDeleter {
 
     for (String fileName : files) {
 
-      if ((IndexFileNameFilter.INSTANCE.accept(null, fileName)) && !fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
+      if (!fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
 
         // Add this file to refCounts with initial count 0:
         getRefCount(fileName);
@@ -189,30 +189,6 @@ final class IndexFileDeleter {
             }
           }
           if (sis != null) {
-            final SegmentInfos infos = sis;
-            for (SegmentInfo segmentInfo : infos) {
-              try {
-                /*
-                 * Force FI to load for each segment since we could see a
-                 * segments file and load successfully above if the files are
-                 * still referenced when they are deleted and the os doesn't let
-                 * you delete them. Yet its likely that fnm files are removed
-                 * while seg file is still around Since LUCENE-2984 we need FI
-                 * to find out if a seg has vectors and prox so we need those
-                 * files to be opened for a commit point.
-                 */
-                segmentInfo.getFieldInfos();
-              } catch (FileNotFoundException e) {
-                refresh(segmentInfo.name);
-                sis = null;
-                if (infoStream.isEnabled("IFD")) {
-                  infoStream.message("IFD", "init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
-                }
-              }
-            }
-           
-          }
-          if (sis != null) {
             final CommitPoint commitPoint = new CommitPoint(commitsToDelete, directory, sis);
             if (sis.getGeneration() == segmentInfos.getGeneration()) {
               currentCommitPoint = commitPoint;
@@ -356,7 +332,7 @@ final class IndexFileDeleter {
     for(int i=0;i<files.length;i++) {
       String fileName = files[i];
       if ((segmentName == null || fileName.startsWith(segmentPrefix1) || fileName.startsWith(segmentPrefix2)) &&
-          IndexFileNameFilter.INSTANCE.accept(null, fileName) &&
+          !fileName.endsWith("write.lock") &&
           !refCounts.containsKey(fileName) &&
           !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
         // Unreferenced file, so remove it
@@ -488,7 +464,7 @@ final class IndexFileDeleter {
     assert locked();
     // If this is a commit point, also incRef the
     // segments_N file:
-    for( final String fileName: segmentInfos.files(directory, isCommit) ) {
+    for(final String fileName: segmentInfos.files(directory, isCommit)) {
       incRef(fileName);
     }
   }