You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/03/22 12:12:10 UTC

svn commit: r1084134 [2/2] - in /lucene/dev/trunk/lucene: ./ src/java/org/apache/lucene/index/ src/java/org/apache/lucene/index/codecs/ src/java/org/apache/lucene/index/codecs/preflex/ src/test-framework/org/apache/lucene/util/ src/test/org/apache/luce...

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java Tue Mar 22 11:12:09 2011
@@ -25,10 +25,4 @@ abstract class TermsHashConsumer {
   abstract TermsHashConsumerPerThread addThread(TermsHashPerThread perThread);
   abstract void flush(Map<TermsHashConsumerPerThread,Collection<TermsHashConsumerPerField>> threadsAndFields, final SegmentWriteState state) throws IOException;
   abstract void abort();
-
-  FieldInfos fieldInfos;
-
-  void setFieldInfos(FieldInfos fieldInfos) {
-    this.fieldInfos = fieldInfos;
   }
-}

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java Tue Mar 22 11:12:09 2011
@@ -56,7 +56,9 @@ public class DefaultSegmentInfosReader e
   
       infos.version = input.readLong(); // read version
       infos.counter = input.readInt(); // read counter
-  
+      if (infos.getFormat() <= DefaultSegmentInfosWriter.FORMAT_4_0) {
+        infos.setGlobalFieldMapVersion(input.readLong());
+      }
       for (int i = input.readInt(); i > 0; i--) { // read segmentInfos
         SegmentInfo si = new SegmentInfo(directory, format, input, codecs);
         if (si.getVersion() == null) {

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java Tue Mar 22 11:12:09 2011
@@ -59,6 +59,7 @@ public class DefaultSegmentInfosWriter e
     out.writeInt(FORMAT_CURRENT); // write FORMAT
     out.writeLong(infos.version);
     out.writeInt(infos.counter); // write counter
+    out.writeLong(infos.getGlobalFieldMapVersion());
     out.writeInt(infos.size()); // write infos
     for (SegmentInfo si : infos) {
       si.write(out);

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java Tue Mar 22 11:12:09 2011
@@ -19,14 +19,15 @@ package org.apache.lucene.index.codecs.p
 
 import java.io.IOException;
 import java.util.Collection;
-import java.util.Iterator;
-import java.util.TreeMap;
+import java.util.Comparator;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
-import java.util.Comparator;
+import java.util.TreeMap;
 
-import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.CompoundFileReader;
 import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.FieldsEnum;
@@ -35,7 +36,6 @@ import org.apache.lucene.index.SegmentIn
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.CompoundFileReader;
 import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
@@ -94,13 +94,11 @@ public class PreFlexFields extends Field
       // so that if an index update removes them we'll still have them
       freqStream = dir.openInput(info.name + ".frq", readBufferSize);
       boolean anyProx = false;
-      final int numFields = fieldInfos.size();
-      for(int i=0;i<numFields;i++) {
-        final FieldInfo fieldInfo = fieldInfos.fieldInfo(i);
-        if (fieldInfo.isIndexed) {
-          fields.put(fieldInfo.name, fieldInfo);
-          preTerms.put(fieldInfo.name, new PreTerms(fieldInfo));
-          if (!fieldInfo.omitTermFreqAndPositions) {
+      for (FieldInfo fi : fieldInfos) {
+        if (fi.isIndexed) {
+          fields.put(fi.name, fi);
+          preTerms.put(fi.name, new PreTerms(fi));
+          if (!fi.omitTermFreqAndPositions) {
             anyProx = true;
           }
         }

Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java Tue Mar 22 11:12:09 2011
@@ -27,6 +27,7 @@ import java.io.OutputStream;
 import java.io.PrintStream;
 import java.lang.reflect.Method;
 import java.util.Enumeration;
+import java.util.List;
 import java.util.Random;
 import java.util.Map;
 import java.util.HashMap;
@@ -35,8 +36,11 @@ import java.util.zip.ZipFile;
 
 import org.junit.Assert;
 
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.index.CheckIndex;
 import org.apache.lucene.index.ConcurrentMergeScheduler;
+import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.LogMergePolicy;
 import org.apache.lucene.index.MergeScheduler;
@@ -325,4 +329,13 @@ public class _TestUtil {
       throw new RuntimeException(e);
     }
   }
+  
+  /** Adds field info for a Document. */
+  public static void add(Document doc, FieldInfos fieldInfos) {
+    List<Fieldable> fields = doc.getFields();
+    for (Fieldable field : fields) {
+      fieldInfos.add(field.name(), field.isIndexed(), field.isTermVectorStored(), field.isStorePositionWithTermVector(),
+              field.isStoreOffsetWithTermVector(), field.getOmitNorms(), false, field.getOmitTermFreqAndPositions());
+    }
+  }
 }

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Tue Mar 22 11:12:09 2011
@@ -1045,8 +1045,8 @@ public class TestAddIndexes extends Luce
     IndexWriter w3 = new IndexWriter(dir, conf);
     w3.addIndexes(readers);
     w3.close();
-    
-    assertEquals("Only one compound segment should exist", 3, dir.listAll().length);
+    // we should now see segments_X, segments.gen,_Y.cfs, _Z.fnx
+    assertEquals("Only one compound segment should exist", 4, dir.listAll().length);
   }
   
 }

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Tue Mar 22 11:12:09 2011
@@ -529,10 +529,9 @@ public class TestBackwardsCompatibility 
       CompoundFileReader cfsReader = new CompoundFileReader(dir, "_0.cfs");
       FieldInfos fieldInfos = new FieldInfos(cfsReader, "_0.fnm");
       int contentFieldIndex = -1;
-      for(int i=0;i<fieldInfos.size();i++) {
-        FieldInfo fi = fieldInfos.fieldInfo(i);
+      for (FieldInfo fi : fieldInfos) {
         if (fi.name.equals("content")) {
-          contentFieldIndex = i;
+          contentFieldIndex = fi.number;
           break;
         }
       }
@@ -544,7 +543,8 @@ public class TestBackwardsCompatibility 
                                "_0_1.del",
                                "_0_1.s" + contentFieldIndex,
                                "segments_2",
-                               "segments.gen"};
+                               "segments.gen",
+                               "1.fnx"};
 
       String[] actual = dir.listAll();
       Arrays.sort(expected);

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestCodecs.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestCodecs.java Tue Mar 22 11:12:09 2011
@@ -23,14 +23,14 @@ import java.util.HashSet;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.index.codecs.PostingsConsumer;
-import org.apache.lucene.index.codecs.TermsConsumer;
 import org.apache.lucene.index.codecs.TermStats;
+import org.apache.lucene.index.codecs.TermsConsumer;
 import org.apache.lucene.index.codecs.mocksep.MockSepCodec;
 import org.apache.lucene.index.codecs.preflex.PreFlexCodec;
 import org.apache.lucene.search.DocIdSetIterator;
@@ -238,8 +238,9 @@ public class TestCodecs extends LuceneTe
     final FieldData[] fields = new FieldData[] {field};
 
     final Directory dir = newDirectory();
+    FieldInfos clonedFieldInfos = (FieldInfos) fieldInfos.clone();
     this.write(fieldInfos, dir, fields, true);
-    final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()), fieldInfos.hasVectors());
+    final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecs.build(clonedFieldInfos, CodecProvider.getDefault()), clonedFieldInfos.hasVectors(), clonedFieldInfos);
     si.setHasProx(false);
 
     final FieldsProducer reader = si.getSegmentCodecs().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 64, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR));
@@ -289,8 +290,10 @@ public class TestCodecs extends LuceneTe
     if (VERBOSE) {
       System.out.println("TEST: now write postings");
     }
+
+    FieldInfos clonedFieldInfos = (FieldInfos) fieldInfos.clone();
     this.write(fieldInfos, dir, fields, false);
-    final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()), fieldInfos.hasVectors());
+    final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecs.build(clonedFieldInfos, CodecProvider.getDefault()), clonedFieldInfos.hasVectors(), clonedFieldInfos);
 
     if (VERBOSE) {
       System.out.println("TEST: now read postings");
@@ -440,7 +443,7 @@ public class TestCodecs extends LuceneTe
         final FieldData field = fields[TestCodecs.random.nextInt(fields.length)];
         final TermsEnum termsEnum = termsDict.terms(field.fieldInfo.name).iterator();
 
-        if (si.getSegmentCodecs().codecs[field.fieldInfo.codecId] instanceof PreFlexCodec) {
+        if (si.getSegmentCodecs().codecs[field.fieldInfo.getCodecId()] instanceof PreFlexCodec) {
           // code below expects unicode sort order
           continue;
         }
@@ -594,7 +597,7 @@ public class TestCodecs extends LuceneTe
     final FieldsConsumer consumer = state.segmentCodecs.codec().fieldsConsumer(state);
     Arrays.sort(fields);
     for (final FieldData field : fields) {
-      if (!allowPreFlex && codecInfo.codecs[field.fieldInfo.codecId] instanceof PreFlexCodec) {
+      if (!allowPreFlex && codecInfo.codecs[field.fieldInfo.getCodecId()] instanceof PreFlexCodec) {
         // code below expects unicode sort order
         continue;
       }

Added: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java?rev=1084134&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java (added)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java Tue Mar 22 11:12:09 2011
@@ -0,0 +1,338 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.Field.TermVector;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Test;
+
+public class TestConsistentFieldNumbers extends LuceneTestCase {
+
+  @Test
+  public void testSameFieldNumbersAcrossSegments() throws Exception {
+    for (int i = 0; i < 2; i++) {
+      Directory dir = newDirectory();
+      IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(NoMergePolicy.COMPOUND_FILES));
+
+      Document d1 = new Document();
+      d1.add(new Field("f1", "first field", Store.YES, Index.ANALYZED, TermVector.NO));
+      d1.add(new Field("f2", "second field", Store.YES, Index.ANALYZED, TermVector.NO));
+      writer.addDocument(d1);
+
+      if (i == 1) {
+        writer.close();
+        writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(NoMergePolicy.COMPOUND_FILES));
+      } else {
+        writer.commit();
+      }
+
+      Document d2 = new Document();
+      d2.add(new Field("f2", "second field", Store.YES, Index.ANALYZED, TermVector.NO));
+      d2.add(new Field("f1", "first field", Store.YES, Index.ANALYZED, TermVector.YES));
+      d2.add(new Field("f3", "third field", Store.YES, Index.ANALYZED, TermVector.NO));
+      d2.add(new Field("f4", "fourth field", Store.YES, Index.ANALYZED, TermVector.NO));
+      writer.addDocument(d2);
+
+      writer.close();
+
+      SegmentInfos sis = new SegmentInfos();
+      sis.read(dir);
+      assertEquals(2, sis.size());
+
+      FieldInfos fis1 = sis.info(0).getFieldInfos();
+      FieldInfos fis2 = sis.info(1).getFieldInfos();
+
+      assertEquals("f1", fis1.fieldInfo(0).name);
+      assertEquals("f2", fis1.fieldInfo(1).name);
+      assertEquals("f1", fis2.fieldInfo(0).name);
+      assertEquals("f2", fis2.fieldInfo(1).name);
+      assertEquals("f3", fis2.fieldInfo(2).name);
+      assertEquals("f4", fis2.fieldInfo(3).name);
+
+      writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()));
+      writer.optimize();
+      writer.close();
+
+      sis = new SegmentInfos();
+      sis.read(dir);
+      assertEquals(1, sis.size());
+
+      FieldInfos fis3 = sis.info(0).getFieldInfos();
+
+      assertEquals("f1", fis3.fieldInfo(0).name);
+      assertEquals("f2", fis3.fieldInfo(1).name);
+      assertEquals("f3", fis3.fieldInfo(2).name);
+      assertEquals("f4", fis3.fieldInfo(3).name);
+
+
+      dir.close();
+    }
+  }
+
+  @Test
+  public void testAddIndexes() throws Exception {
+    Directory dir1 = newDirectory();
+    Directory dir2 = newDirectory();
+    IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(NoMergePolicy.COMPOUND_FILES));
+
+    Document d1 = new Document();
+    d1.add(new Field("f1", "first field", Store.YES, Index.ANALYZED, TermVector.NO));
+    d1.add(new Field("f2", "second field", Store.YES, Index.ANALYZED, TermVector.NO));
+    writer.addDocument(d1);
+
+    writer.close();
+    writer = new IndexWriter(dir2, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(NoMergePolicy.COMPOUND_FILES));
+
+    Document d2 = new Document();
+    d2.add(new Field("f2", "second field", Store.YES, Index.ANALYZED, TermVector.NO));
+    d2.add(new Field("f1", "first field", Store.YES, Index.ANALYZED, TermVector.YES));
+    d2.add(new Field("f3", "third field", Store.YES, Index.ANALYZED, TermVector.NO));
+    d2.add(new Field("f4", "fourth field", Store.YES, Index.ANALYZED, TermVector.NO));
+    writer.addDocument(d2);
+
+    writer.close();
+
+    writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(NoMergePolicy.COMPOUND_FILES));
+    writer.addIndexes(dir2);
+    writer.close();
+
+    SegmentInfos sis = new SegmentInfos();
+    sis.read(dir1);
+    assertEquals(2, sis.size());
+
+    FieldInfos fis1 = sis.info(0).getFieldInfos();
+    FieldInfos fis2 = sis.info(1).getFieldInfos();
+
+    assertEquals("f1", fis1.fieldInfo(0).name);
+    assertEquals("f2", fis1.fieldInfo(1).name);
+    // make sure the ordering of the "external" segment is preserved
+    assertEquals("f2", fis2.fieldInfo(0).name);
+    assertEquals("f1", fis2.fieldInfo(1).name);
+    assertEquals("f3", fis2.fieldInfo(2).name);
+    assertEquals("f4", fis2.fieldInfo(3).name);
+
+    writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()));
+    writer.optimize();
+    writer.close();
+
+    sis = new SegmentInfos();
+    sis.read(dir1);
+    assertEquals(1, sis.size());
+
+    FieldInfos fis3 = sis.info(0).getFieldInfos();
+
+    // after merging the ordering should be identical to the first segment
+    assertEquals("f1", fis3.fieldInfo(0).name);
+    assertEquals("f2", fis3.fieldInfo(1).name);
+    assertEquals("f3", fis3.fieldInfo(2).name);
+    assertEquals("f4", fis3.fieldInfo(3).name);
+
+    dir1.close();
+    dir2.close();
+  }
+  
+  public void testFieldNumberGaps() throws IOException {
+    for (int i = 0; i < 39; i++) {
+      Directory dir = newDirectory();
+      {
+        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+            TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(
+            NoMergePolicy.NO_COMPOUND_FILES));
+        Document d = new Document();
+        d.add(new Field("f1", "d1 first field", Store.YES, Index.ANALYZED,
+            TermVector.NO));
+        d.add(new Field("f2", "d1 second field", Store.YES, Index.ANALYZED,
+            TermVector.NO));
+        writer.addDocument(d);
+        writer.close();
+        SegmentInfos sis = new SegmentInfos();
+        sis.read(dir);
+        assertEquals(1, sis.size());
+        FieldInfos fis1 = sis.info(0).getFieldInfos();
+        assertEquals("f1", fis1.fieldInfo(0).name);
+        assertEquals("f2", fis1.fieldInfo(1).name);
+        assertTrue(dir.fileExists("1.fnx"));
+      }
+      
+
+      {
+        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+            TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(
+            random.nextBoolean() ? NoMergePolicy.NO_COMPOUND_FILES
+                : NoMergePolicy.COMPOUND_FILES));
+        Document d = new Document();
+        d.add(new Field("f1", "d2 first field", Store.YES, Index.ANALYZED,
+            TermVector.NO));
+        d.add(new Field("f3", new byte[] { 1, 2, 3 }));
+        writer.addDocument(d);
+        writer.close();
+        SegmentInfos sis = new SegmentInfos();
+        sis.read(dir);
+        assertEquals(2, sis.size());
+        FieldInfos fis1 = sis.info(0).getFieldInfos();
+        FieldInfos fis2 = sis.info(1).getFieldInfos();
+        assertEquals("f1", fis1.fieldInfo(0).name);
+        assertEquals("f2", fis1.fieldInfo(1).name);
+        assertEquals("f1", fis2.fieldInfo(0).name);
+        assertNull(fis2.fieldInfo(1));
+        assertEquals("f3", fis2.fieldInfo(2).name);
+        assertFalse(dir.fileExists("1.fnx"));
+        assertTrue(dir.fileExists("2.fnx"));
+      }
+
+      {
+        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+            TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(
+            random.nextBoolean() ? NoMergePolicy.NO_COMPOUND_FILES
+                : NoMergePolicy.COMPOUND_FILES));
+        Document d = new Document();
+        d.add(new Field("f1", "d3 first field", Store.YES, Index.ANALYZED,
+            TermVector.NO));
+        d.add(new Field("f2", "d3 second field", Store.YES, Index.ANALYZED,
+            TermVector.NO));
+        d.add(new Field("f3", new byte[] { 1, 2, 3, 4, 5 }));
+        writer.addDocument(d);
+        writer.close();
+        SegmentInfos sis = new SegmentInfos();
+        sis.read(dir);
+        assertEquals(3, sis.size());
+        FieldInfos fis1 = sis.info(0).getFieldInfos();
+        FieldInfos fis2 = sis.info(1).getFieldInfos();
+        FieldInfos fis3 = sis.info(2).getFieldInfos();
+        assertEquals("f1", fis1.fieldInfo(0).name);
+        assertEquals("f2", fis1.fieldInfo(1).name);
+        assertEquals("f1", fis2.fieldInfo(0).name);
+        assertNull(fis2.fieldInfo(1));
+        assertEquals("f3", fis2.fieldInfo(2).name);
+        assertEquals("f1", fis3.fieldInfo(0).name);
+        assertEquals("f2", fis3.fieldInfo(1).name);
+        assertEquals("f3", fis3.fieldInfo(2).name);
+        assertFalse(dir.fileExists("1.fnx"));
+        assertTrue(dir.fileExists("2.fnx"));
+        assertFalse(dir.fileExists("3.fnx"));
+      }
+
+      {
+        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+            TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(
+            random.nextBoolean() ? NoMergePolicy.NO_COMPOUND_FILES
+                : NoMergePolicy.COMPOUND_FILES));
+        writer.deleteDocuments(new Term("f1", "d1"));
+        // nuke the first segment entirely so that the segment with gaps is
+        // loaded first!
+        writer.expungeDeletes();
+        writer.close();
+      }
+
+      IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+          TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(
+          new LogByteSizeMergePolicy()));
+      writer.optimize();
+      assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges);
+      writer.close();
+
+      SegmentInfos sis = new SegmentInfos();
+      sis.read(dir);
+      assertEquals(1, sis.size());
+      FieldInfos fis1 = sis.info(0).getFieldInfos();
+      assertEquals("f1", fis1.fieldInfo(0).name);
+      assertEquals("f2", fis1.fieldInfo(1).name);
+      assertEquals("f3", fis1.fieldInfo(2).name);
+      assertFalse(dir.fileExists("1.fnx"));
+      assertTrue(dir.fileExists("2.fnx"));
+      assertFalse(dir.fileExists("3.fnx"));
+      dir.close();
+    }
+  }
+
+  @Test
+  public void testManyFields() throws Exception {
+    final int NUM_DOCS = 2000;
+    final int MAX_FIELDS = 50;
+
+    int[][] docs = new int[NUM_DOCS][4];
+    for (int i = 0; i < docs.length; i++) {
+      for (int j = 0; j < docs[i].length;j++) {
+        docs[i][j] = random.nextInt(MAX_FIELDS);
+      }
+    }
+
+    Directory dir = newDirectory();
+    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()));
+
+    for (int i = 0; i < NUM_DOCS; i++) {
+      Document d = new Document();
+      for (int j = 0; j < docs[i].length; j++) {
+        d.add(getField(docs[i][j]));
+      }
+
+      writer.addDocument(d);
+    }
+
+    writer.optimize();
+    writer.close();
+
+    SegmentInfos sis = new SegmentInfos();
+    sis.read(dir);
+    for (SegmentInfo si : sis) {
+      FieldInfos fis = si.getFieldInfos();
+
+      for (FieldInfo fi : fis) {
+        Field expected = getField(Integer.parseInt(fi.name));
+        assertEquals(expected.isIndexed(), fi.isIndexed);
+        assertEquals(expected.isTermVectorStored(), fi.storeTermVector);
+        assertEquals(expected.isStorePositionWithTermVector(), fi.storePositionWithTermVector);
+        assertEquals(expected.isStoreOffsetWithTermVector(), fi.storeOffsetWithTermVector);
+      }
+    }
+
+    dir.close();
+  }
+
+  private Field getField(int number) {
+    int mode = number % 16;
+    String fieldName = "" + number;
+    switch (mode) {
+      case 0: return new Field(fieldName, "some text", Store.YES, Index.ANALYZED, TermVector.NO);
+      case 1: return new Field(fieldName, "some text", Store.NO, Index.ANALYZED, TermVector.NO);
+      case 2: return new Field(fieldName, "some text", Store.YES, Index.NOT_ANALYZED, TermVector.NO);
+      case 3: return new Field(fieldName, "some text", Store.NO, Index.NOT_ANALYZED, TermVector.NO);
+      case 4: return new Field(fieldName, "some text", Store.YES, Index.ANALYZED, TermVector.WITH_OFFSETS);
+      case 5: return new Field(fieldName, "some text", Store.NO, Index.ANALYZED, TermVector.WITH_OFFSETS);
+      case 6: return new Field(fieldName, "some text", Store.YES, Index.NOT_ANALYZED, TermVector.WITH_OFFSETS);
+      case 7: return new Field(fieldName, "some text", Store.NO, Index.NOT_ANALYZED, TermVector.WITH_OFFSETS);
+      case 8: return new Field(fieldName, "some text", Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS);
+      case 9: return new Field(fieldName, "some text", Store.NO, Index.ANALYZED, TermVector.WITH_POSITIONS);
+      case 10: return new Field(fieldName, "some text", Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS);
+      case 11: return new Field(fieldName, "some text", Store.NO, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS);
+      case 12: return new Field(fieldName, "some text", Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS);
+      case 13: return new Field(fieldName, "some text", Store.NO, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS);
+      case 14: return new Field(fieldName, "some text", Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS);
+      case 15: return new Field(fieldName, "some text", Store.NO, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS);
+      default: return null;
+    }
+  }
+}

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDoc.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDoc.java Tue Mar 22 11:12:09 2011
@@ -202,10 +202,10 @@ public class TestDoc extends LuceneTestC
       merger.merge();
       r1.close();
       r2.close();
-      
+      final FieldInfos fieldInfos =  merger.fieldInfos();
       final SegmentInfo info = new SegmentInfo(merged, si1.docCount + si2.docCount, si1.dir,
-                                               false, merger.fieldInfos().hasProx(), merger.getSegmentCodecs(),
-                                               merger.fieldInfos().hasVectors());
+                                               false, fieldInfos.hasProx(), merger.getSegmentCodecs(),
+                                               fieldInfos.hasVectors(), fieldInfos);
       
       if (useCompoundFile) {
         Collection<String> filesToDelete = merger.createCompoundFile(merged + ".cfs", info);

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java Tue Mar 22 11:12:09 2011
@@ -25,20 +25,20 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.document.Field.Index;
 import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.Field.TermVector;
+import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.util.BytesRef;
 
 public class TestDocumentWriter extends LuceneTestCase {
   private Directory dir;
@@ -98,8 +98,7 @@ public class TestDocumentWriter extends 
 
     // test that the norms are not present in the segment if
     // omitNorms is true
-    for (int i = 0; i < reader.core.fieldInfos.size(); i++) {
-      FieldInfo fi = reader.core.fieldInfos.fieldInfo(i);
+    for (FieldInfo fi : reader.core.fieldInfos) {
       if (fi.isIndexed) {
         assertTrue(fi.omitNorms == !reader.hasNorms(fi.name));
       }

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java Tue Mar 22 11:12:09 2011
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
  */
 
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexOutput;
@@ -40,7 +41,7 @@ public class TestFieldInfos extends Luce
     //Positive test of FieldInfos
     assertTrue(testDoc != null);
     FieldInfos fieldInfos = new FieldInfos();
-    fieldInfos.add(testDoc);
+    _TestUtil.add(testDoc, fieldInfos);
     //Since the complement is stored as well in the fields map
     assertTrue(fieldInfos.size() == DocHelper.all.size()); //this is all b/c we are using the no-arg constructor
     Directory dir = newDirectory();

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java Tue Mar 22 11:12:09 2011
@@ -49,7 +49,7 @@ public class TestFieldsReader extends Lu
     super.setUp();
     fieldInfos = new FieldInfos();
     DocHelper.setupDoc(testDoc);
-    fieldInfos.add(testDoc);
+    _TestUtil.add(testDoc, fieldInfos);
     dir = newDirectory();
     IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newLogMergePolicy());
     ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(false);

Added: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java?rev=1084134&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java (added)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java Tue Mar 22 11:12:09 2011
@@ -0,0 +1,537 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.Field.TermVector;
+import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
+import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+
+public class TestGlobalFieldNumbers extends LuceneTestCase {
+
+  public void testGlobalFieldNumberFiles() throws IOException {
+    for (int i = 0; i < 39; i++) {
+      Directory dir = newDirectory();
+      {
+        IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
+            new MockAnalyzer());
+        IndexWriter writer = new IndexWriter(dir, config);
+        Document d = new Document();
+        d.add(new Field("f1", "d1 first field", Store.YES, Index.ANALYZED,
+            TermVector.NO));
+        d.add(new Field("f2", "d1 second field", Store.YES, Index.ANALYZED,
+            TermVector.NO));
+        writer.addDocument(d);
+        Collection<String> files = writer.segmentInfos.files(dir, true);
+        for (String string : files) {
+          assertFalse(string.endsWith(".fnx"));
+        }
+        writer.commit();
+        files = writer.segmentInfos.files(dir, true);
+        files.remove("1.fnx");
+        for (String string : files) {
+          assertFalse(string.endsWith(".fnx"));
+        }
+
+        assertFNXFiles(dir, "1.fnx");
+        d = new Document();
+        d.add(new Field("f1", "d2 first field", Store.YES, Index.ANALYZED,
+            TermVector.NO));
+        d.add(new Field("f3", new byte[] { 1, 2, 3 }));
+        writer.addDocument(d);
+        writer.commit();
+        files = writer.segmentInfos.files(dir, true);
+        files.remove("2.fnx");
+        for (String string : files) {
+          assertFalse(string.endsWith(".fnx"));
+        }
+        assertFNXFiles(dir, "2.fnx");
+        writer.close();
+        assertFNXFiles(dir, "2.fnx");
+      }
+
+      {
+        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+            TEST_VERSION_CURRENT, new MockAnalyzer()));
+        Document d = new Document();
+        d.add(new Field("f1", "d3 first field", Store.YES, Index.ANALYZED,
+            TermVector.NO));
+        d.add(new Field("f2", "d3 second field", Store.YES, Index.ANALYZED,
+            TermVector.NO));
+        d.add(new Field("f3", new byte[] { 1, 2, 3, 4, 5 }));
+        writer.addDocument(d);
+        writer.close();
+        Collection<String> files = writer.segmentInfos.files(dir, true);
+        files.remove("2.fnx");
+        for (String string : files) {
+          assertFalse(string.endsWith(".fnx"));
+        }
+
+        assertFNXFiles(dir, "2.fnx");
+      }
+
+      IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+          TEST_VERSION_CURRENT, new MockAnalyzer()));
+      writer.optimize();
+      assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges);
+      writer.close();
+      assertFNXFiles(dir, "2.fnx");
+
+      dir.close();
+    }
+  }
+
+  public void testIndexReaderCommit() throws IOException {
+    for (int i = 0; i < 39; i++) {
+      Directory dir = newDirectory();
+      {
+        IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
+            new MockAnalyzer());
+        IndexWriter writer = new IndexWriter(dir, config);
+        Document d = new Document();
+        d.add(new Field("f1", "d1 first field", Store.YES, Index.ANALYZED,
+            TermVector.NO));
+        d.add(new Field("f2", "d1 second field", Store.YES, Index.ANALYZED,
+            TermVector.NO));
+        writer.addDocument(d);
+        writer.commit();
+        assertFNXFiles(dir, "1.fnx");
+        d = new Document();
+        d.add(new Field("f1", "d2 first field", Store.YES, Index.ANALYZED,
+            TermVector.NO));
+        d.add(new Field("f3", new byte[] { 1, 2, 3 }));
+        writer.addDocument(d);
+        writer.commit();
+        assertFNXFiles(dir, "2.fnx");
+        writer.close();
+        assertFNXFiles(dir, "2.fnx");
+      }
+      IndexReader reader = IndexReader.open(dir, false);
+      reader.deleteDocument(0);
+      reader.commit();
+      reader.close();
+      // make sure this reader can not modify the field map
+      assertFNXFiles(dir, "2.fnx");
+
+      IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+          TEST_VERSION_CURRENT, new MockAnalyzer()));
+      writer.optimize();
+      assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges);
+      writer.close();
+      assertFNXFiles(dir, "2.fnx");
+
+      dir.close();
+    }
+  }
+
+  public void testGlobalFieldNumberFilesAcrossCommits() throws IOException {
+    for (int i = 0; i < 39; i++) {
+      Directory dir = newDirectory();
+      {
+        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+            TEST_VERSION_CURRENT, new MockAnalyzer()).setIndexDeletionPolicy(
+            new KeepAllDeletionPolicy()));
+        Document d = new Document();
+        d.add(new Field("f1", "d1 first field", Store.YES, Index.ANALYZED,
+            TermVector.NO));
+        d.add(new Field("f2", "d1 second field", Store.YES, Index.ANALYZED,
+            TermVector.NO));
+        writer.addDocument(d);
+        writer.commit();
+        assertFNXFiles(dir, "1.fnx");
+        d = new Document();
+        d.add(new Field("f1", "d2 first field", Store.YES, Index.ANALYZED,
+            TermVector.NO));
+        d.add(new Field("f3", new byte[] { 1, 2, 3 }));
+        writer.addDocument(d);
+        writer.commit();
+        writer.commit();
+        writer.commit();
+        assertFNXFiles(dir, "1.fnx", "2.fnx");
+        writer.close();
+        assertFNXFiles(dir, "1.fnx", "2.fnx");
+      }
+
+      {
+        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+            TEST_VERSION_CURRENT, new MockAnalyzer()));
+        Document d = new Document();
+        d.add(new Field("f1", "d3 first field", Store.YES, Index.ANALYZED,
+            TermVector.NO));
+        d.add(new Field("f2", "d3 second field", Store.YES, Index.ANALYZED,
+            TermVector.NO));
+        d.add(new Field("f3", new byte[] { 1, 2, 3, 4, 5 }));
+        writer.addDocument(d);
+        writer.close();
+        assertFNXFiles(dir, "2.fnx");
+      }
+      IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+          TEST_VERSION_CURRENT, new MockAnalyzer()));
+      writer.optimize();
+      assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges);
+      writer.close();
+      assertFNXFiles(dir, "2.fnx");
+      dir.close();
+    }
+  }
+
+  public void testGlobalFieldNumberOnOldCommit() throws IOException {
+    for (int i = 0; i < 39; i++) {
+      Directory dir = newDirectory();
+      IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+          TEST_VERSION_CURRENT, new MockAnalyzer()).setIndexDeletionPolicy(
+          new KeepAllDeletionPolicy()));
+      Document d = new Document();
+      d.add(new Field("f1", "d1 first field", Store.YES, Index.ANALYZED,
+          TermVector.NO));
+      d.add(new Field("f2", "d1 second field", Store.YES, Index.ANALYZED,
+          TermVector.NO));
+      writer.addDocument(d);
+      writer.commit();
+      assertFNXFiles(dir, "1.fnx");
+      d = new Document();
+      d.add(new Field("f1", "d2 first field", Store.YES, Index.ANALYZED,
+          TermVector.NO));
+      d.add(new Field("f3", new byte[] { 1, 2, 3 }));
+      writer.addDocument(d);
+      assertFNXFiles(dir, "1.fnx");
+      writer.close();
+      assertFNXFiles(dir, "1.fnx", "2.fnx");
+      // open first commit
+      List<IndexCommit> listCommits = IndexReader.listCommits(dir);
+      assertEquals(2, listCommits.size());
+      writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
+          new MockAnalyzer()).setIndexDeletionPolicy(
+          new KeepAllDeletionPolicy()).setIndexCommit(listCommits.get(0)));
+
+      d = new Document();
+      d.add(new Field("f1", "d2 first field", Store.YES, Index.ANALYZED,
+          TermVector.NO));
+      d.add(new Field("f3", new byte[] { 1, 2, 3 }));
+      writer.addDocument(d);
+      writer.commit();
+      // now we have 3 files since f3 is not present in the first commit
+      assertFNXFiles(dir, "1.fnx", "2.fnx", "3.fnx");
+      writer.close();
+      assertFNXFiles(dir, "1.fnx", "2.fnx", "3.fnx");
+
+      writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
+          new MockAnalyzer()));
+      writer.commit();
+      listCommits = IndexReader.listCommits(dir);
+      assertEquals(1, listCommits.size());
+      assertFNXFiles(dir, "3.fnx");
+      writer.close();
+      assertFNXFiles(dir, "3.fnx");
+      dir.close();
+    }
+  }
+
+  private final Directory buildRandomIndex(String[] fieldNames, int numDocs,
+      IndexWriterConfig conf) throws CorruptIndexException,
+      LockObtainFailedException, IOException {
+    Directory dir = newDirectory();
+    IndexWriter writer = new IndexWriter(dir, conf);
+    for (int i = 0; i < numDocs; i++) {
+      Document doc = new Document();
+      final int numFields = 1 + random.nextInt(fieldNames.length);
+      for (int j = 0; j < numFields; j++) {
+        doc.add(newField(fieldNames[random.nextInt(fieldNames.length)],
+            _TestUtil.randomRealisticUnicodeString(random),
+            Index.toIndex(true, random.nextBoolean(), random.nextBoolean())));
+
+      }
+      writer.addDocument(doc);
+      if (random.nextInt(20) == 0) {
+        writer.commit();
+      }
+    }
+    writer.close();
+    return dir;
+  }
+
+  public void testOptimize() throws IOException {
+    for (int i = 0; i < 2; i++) {
+      Set<String> fieldNames = new HashSet<String>();
+      final int numFields = 2 + random.nextInt(200);
+      for (int j = 0; j < numFields; j++) {
+        fieldNames.add("field_" + j);
+      }
+      Directory base = buildRandomIndex(fieldNames.toArray(new String[0]),
+          20 + random.nextInt(100),
+          newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+      IndexWriter writer = new IndexWriter(base, newIndexWriterConfig(
+          TEST_VERSION_CURRENT, new MockAnalyzer()));
+      FieldNumberBiMap globalFieldMap = writer.segmentInfos
+          .getOrLoadGlobalFieldNumberMap(base);
+      Set<Entry<String, Integer>> entries = globalFieldMap.entries();
+      writer.optimize();
+      writer.commit();
+      writer.close();
+      Set<Entry<String, Integer>> afterOptmize = globalFieldMap.entries();
+      assertEquals(entries, afterOptmize);
+      base.close();
+    }
+  }
+
+  public void testAddIndexesStableFieldNumbers() throws IOException {
+    for (int i = 0; i < 2; i++) {
+      Set<String> fieldNames = new HashSet<String>();
+      final int numFields = 2 + random.nextInt(50);
+      for (int j = 0; j < numFields; j++) {
+        fieldNames.add("field_" + j);
+      }
+
+      Directory base = newDirectory();
+      IndexWriter writer = new IndexWriter(base, newIndexWriterConfig(
+          TEST_VERSION_CURRENT, new MockAnalyzer()));
+      Document doc = new Document();
+      for (String string : fieldNames) {
+        doc.add(newField(string,
+            _TestUtil.randomRealisticUnicodeString(random),
+            Index.toIndex(true, random.nextBoolean(), random.nextBoolean())));
+
+      }
+      writer.addDocument(doc);
+      writer.commit();
+      FieldNumberBiMap globalFieldMap = writer.segmentInfos
+          .getOrLoadGlobalFieldNumberMap(base);
+      final Set<Entry<String, Integer>> entries = globalFieldMap.entries();
+      assertEquals(entries.size(), fieldNames.size());
+      for (Entry<String, Integer> entry : entries) {
+        // all fields are in this fieldMap
+        assertTrue(fieldNames.contains(entry.getKey()));
+      }
+      writer.close();
+
+      int numIndexes = 1 + random.nextInt(10);
+      for (int j = 0; j < numIndexes; j++) {
+        Directory toAdd = buildRandomIndex(fieldNames.toArray(new String[0]),
+            1 + random.nextInt(50),
+            newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+        IndexWriter w = new IndexWriter(base, newIndexWriterConfig(
+            TEST_VERSION_CURRENT, new MockAnalyzer()));
+        if (random.nextBoolean()) {
+          IndexReader open = IndexReader.open(toAdd);
+          w.addIndexes(open);
+          open.close();
+        } else {
+          w.addIndexes(toAdd);
+        }
+
+        w.close();
+        FieldNumberBiMap map = w.segmentInfos
+            .getOrLoadGlobalFieldNumberMap(toAdd);
+        assertEquals(entries, map.entries());
+        toAdd.close();
+      }
+      IndexWriter w = new IndexWriter(base, newIndexWriterConfig(
+          TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(
+          new LogByteSizeMergePolicy()));
+      w.optimize();
+      w.close();
+      SegmentInfos sis = new SegmentInfos();
+      sis.read(base);
+      SegmentInfo segmentInfo = sis.get(sis.size() - 1);// last segment must
+                                                        // have all fields with
+                                                        // consistent numbers
+      FieldInfos fieldInfos = segmentInfo.getFieldInfos();
+      assertEquals(fieldInfos.size(), entries.size());
+      for (Entry<String, Integer> entry : entries) {
+        assertEquals(entry.getValue(),
+            Integer.valueOf(fieldInfos.fieldNumber(entry.getKey())));
+        assertEquals(entry.getKey(), fieldInfos.fieldName(entry.getValue()));
+      }
+      base.close();
+    }
+  }
+
+  final String[] oldNames = { "30.cfs", "30.nocfs", "31.cfs", "31.nocfs", };
+
+  public void testAddOldIndex() throws IOException {
+    int i = random.nextInt(oldNames.length);
+    File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
+    Directory dir = null;
+    try {
+      _TestUtil
+          .unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
+      dir = newFSDirectory(oldIndxeDir);
+      SegmentInfos infos = new SegmentInfos();
+      infos.read(dir);
+      SortedMap<Integer, String> sortedMap = new TreeMap<Integer, String>();
+
+      FieldNumberBiMap biMap = new FieldNumberBiMap();
+      int maxFieldNum = Integer.MIN_VALUE;
+      for (SegmentInfo segmentInfo : infos) {
+        for (FieldInfo fieldInfo : segmentInfo.getFieldInfos()) {
+          int globNumber = biMap.addOrGet(fieldInfo.name, fieldInfo.number);
+          maxFieldNum = Math.max(maxFieldNum, globNumber);
+          sortedMap.put(globNumber, fieldInfo.name);
+        }
+      }
+      Directory base = newDirectory();
+      IndexWriter writer = new IndexWriter(base, newIndexWriterConfig(
+          TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(
+          NoMergePolicy.NO_COMPOUND_FILES));
+
+      SortedMap<Integer, String> copySortedMap = new TreeMap<Integer, String>(
+          sortedMap);
+      while (!sortedMap.isEmpty()) { // add every field at least once
+        Document doc = new Document();
+        int nextField = random.nextInt(maxFieldNum + 1);
+        sortedMap.remove(nextField);
+
+        String name = copySortedMap.get(nextField);
+        assertNotNull(name);
+
+        doc.add(newField(name, _TestUtil.randomRealisticUnicodeString(random),
+            Index.toIndex(true, random.nextBoolean(), random.nextBoolean())));
+        writer.addDocument(doc);
+        if (random.nextInt(10) == 0) {
+          writer.commit();
+        }
+      }
+      Set<Entry<String, Integer>> expectedEntries = writer.segmentInfos
+          .getOrLoadGlobalFieldNumberMap(base).entries();
+      writer.addIndexes(dir); // add the old index
+      writer.close();
+
+      writer = new IndexWriter(base, newIndexWriterConfig(TEST_VERSION_CURRENT,
+          new MockAnalyzer()).setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES));
+      writer.commit(); // make sure the old index is the latest segment
+      writer.close();
+
+      // we don't merge here since we use NoMergePolicy
+      SegmentInfos sis = new SegmentInfos();
+      sis.read(base);
+      // check that the latest global field numbers are consistent and carried
+      // over from the 4.0 index
+      FieldNumberBiMap actualGlobalMap = sis
+          .getOrLoadGlobalFieldNumberMap(base);
+      assertEquals(expectedEntries, actualGlobalMap.entries());
+      base.close();
+    } finally {
+      if (dir != null)
+        dir.close();
+      _TestUtil.rmDir(oldIndxeDir);
+    }
+  }
+  
+  public void testFilesOnOldIndex() throws IOException {
+    int i = random.nextInt(oldNames.length);
+    File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
+    Directory dir = null;
+    
+    MergePolicy policy = random.nextBoolean() ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES;
+    try {
+      _TestUtil
+          .unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
+      dir = newFSDirectory(oldIndxeDir);
+      IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+          TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(policy));
+      SegmentInfos segmentInfos = writer.segmentInfos;
+      assertTrue(DefaultSegmentInfosWriter.FORMAT_4_0 < segmentInfos.getFormat());
+      assertEquals(0, segmentInfos.getGlobalFieldMapVersion());
+      Collection<String> files = segmentInfos.files(dir, true);
+      for (String string : files) {
+        assertFalse(string.endsWith(".fnx"));
+      }
+      writer.commit();
+      
+      assertTrue(DefaultSegmentInfosWriter.FORMAT_4_0 < segmentInfos.getFormat());
+      assertEquals(0, segmentInfos.getGlobalFieldMapVersion());
+      files = segmentInfos.files(dir, true);
+      for (String string : files) {
+        assertFalse(string.endsWith(".fnx"));
+      }
+      
+      Document d = new Document();
+      d.add(new Field("f1", "d1 first field", Store.YES, Index.ANALYZED,
+          TermVector.NO));
+      writer.addDocument(d);
+      writer.prepareCommit();
+      // the fnx file should still be under control of the SIS
+      assertTrue(DefaultSegmentInfosWriter.FORMAT_4_0 < segmentInfos.getFormat());
+      assertEquals(0, segmentInfos.getLastGlobalFieldMapVersion());
+      assertEquals(1, segmentInfos.getGlobalFieldMapVersion());
+      files = segmentInfos.files(dir, true);
+      for (String string : files) {
+        assertFalse(string.endsWith(".fnx"));
+      }
+      
+      writer.commit();
+      
+      // now we should see the fnx file even if this is a 3.x segment
+      assertTrue(DefaultSegmentInfosWriter.FORMAT_4_0 < segmentInfos.getFormat());
+      assertEquals(1, segmentInfos.getGlobalFieldMapVersion());
+      assertEquals(1, segmentInfos.getLastGlobalFieldMapVersion());
+      files = segmentInfos.files(dir, true);
+      assertTrue(files.remove("1.fnx"));
+      for (String string : files) {
+        assertFalse(string.endsWith(".fnx"));
+      }
+      writer.close();
+    } finally {
+      if (dir != null)
+        dir.close();
+      _TestUtil.rmDir(oldIndxeDir);
+    }
+  }
+
+  class KeepAllDeletionPolicy implements IndexDeletionPolicy {
+    public void onInit(List<? extends IndexCommit> commits) throws IOException {
+    }
+
+    public void onCommit(List<? extends IndexCommit> commits)
+        throws IOException {
+    }
+  }
+
+  public static void assertFNXFiles(Directory dir, String... expectedFnxFiles)
+      throws IOException {
+    String[] listAll = dir.listAll();
+    Set<String> fnxFiles = new HashSet<String>();
+    for (String string : listAll) {
+      if (string.endsWith(".fnx")) {
+        fnxFiles.add(string);
+      }
+    }
+    assertEquals("" + fnxFiles, expectedFnxFiles.length, fnxFiles.size());
+    for (String string : expectedFnxFiles) {
+      assertTrue(" missing fnx file: " + string, fnxFiles.contains(string));
+    }
+  }
+
+}

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java Tue Mar 22 11:12:09 2011
@@ -92,10 +92,9 @@ public class TestIndexFileDeleter extend
     CompoundFileReader cfsReader = new CompoundFileReader(dir, "_2.cfs");
     FieldInfos fieldInfos = new FieldInfos(cfsReader, "_2.fnm");
     int contentFieldIndex = -1;
-    for(i=0;i<fieldInfos.size();i++) {
-      FieldInfo fi = fieldInfos.fieldInfo(i);
+    for (FieldInfo fi : fieldInfos) {
       if (fi.name.equals("content")) {
-        contentFieldIndex = i;
+        contentFieldIndex = fi.number;
         break;
       }
     }

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Tue Mar 22 11:12:09 2011
@@ -33,7 +33,9 @@ import org.apache.lucene.search.DocIdSet
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.store.MockDirectoryWrapper.Failure;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.MockAnalyzer;
@@ -780,7 +782,6 @@ public class TestIndexWriterExceptions e
         }
       }
     }
-
     ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).sync();
     assertTrue(failure.didFail);
     failure.clearDoFail();
@@ -794,53 +795,79 @@ public class TestIndexWriterExceptions e
   
   private static class FailOnlyInCommit extends MockDirectoryWrapper.Failure {
 
-    boolean fail1, fail2;
+    boolean failOnCommit, failOnDeleteFile;
+    private final boolean dontFailDuringGlobalFieldMap;
+    private static final String PREPARE_STAGE = "prepareCommit";
+    private static final String FINISH_STAGE = "finishCommit";
+    private final String stage;
+    
+    public FailOnlyInCommit(boolean dontFailDuringGlobalFieldMap, String stage) {
+      this.dontFailDuringGlobalFieldMap = dontFailDuringGlobalFieldMap;
+      this.stage = stage;
+    }
 
     @Override
     public void eval(MockDirectoryWrapper dir)  throws IOException {
       StackTraceElement[] trace = new Exception().getStackTrace();
       boolean isCommit = false;
       boolean isDelete = false;
+      boolean isInGlobalFieldMap = false;
       for (int i = 0; i < trace.length; i++) {
-        if ("org.apache.lucene.index.SegmentInfos".equals(trace[i].getClassName()) && "prepareCommit".equals(trace[i].getMethodName()))
+        if ("org.apache.lucene.index.SegmentInfos".equals(trace[i].getClassName()) && stage.equals(trace[i].getMethodName()))
           isCommit = true;
         if ("org.apache.lucene.store.MockDirectoryWrapper".equals(trace[i].getClassName()) && "deleteFile".equals(trace[i].getMethodName()))
           isDelete = true;
+        if ("org.apache.lucene.index.SegmentInfos".equals(trace[i].getClassName()) && "writeGlobalFieldMap".equals(trace[i].getMethodName()))
+          isInGlobalFieldMap = true;
+          
+      }
+      if (isInGlobalFieldMap && dontFailDuringGlobalFieldMap) {
+        isCommit = false;
       }
-
       if (isCommit) {
         if (!isDelete) {
-          fail1 = true;
+          failOnCommit = true;
           throw new RuntimeException("now fail first");
         } else {
-          fail2 = true;
+          failOnDeleteFile = true;
           throw new IOException("now fail during delete");
         }
       }
     }
   }
   
-  // LUCENE-1214
   public void testExceptionsDuringCommit() throws Throwable {
-    MockDirectoryWrapper dir = newDirectory();
-    FailOnlyInCommit failure = new FailOnlyInCommit();
-    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()));
-    Document doc = new Document();
-    doc.add(newField("field", "a field", Field.Store.YES,
-                      Field.Index.ANALYZED));
-    w.addDocument(doc);
-    dir.failOn(failure);
-    try {
-      w.close();
-      fail();
-    } catch (IOException ioe) {
-      fail("expected only RuntimeException");
-    } catch (RuntimeException re) {
-      // Expected
+    FailOnlyInCommit[] failures = new FailOnlyInCommit[] {
+        // LUCENE-1214
+        new FailOnlyInCommit(false, FailOnlyInCommit.PREPARE_STAGE), // fail during global field map is written
+        new FailOnlyInCommit(true, FailOnlyInCommit.PREPARE_STAGE), // fail after global field map is written
+        new FailOnlyInCommit(false, FailOnlyInCommit.FINISH_STAGE)  // fail while running finishCommit    
+    };
+    
+    for (FailOnlyInCommit failure : failures) {
+      MockDirectoryWrapper dir = newDirectory();
+      IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
+          TEST_VERSION_CURRENT, new MockAnalyzer()));
+      Document doc = new Document();
+      doc.add(newField("field", "a field", Field.Store.YES,
+          Field.Index.ANALYZED));
+      w.addDocument(doc);
+      dir.failOn(failure);
+      try {
+        w.close();
+        fail();
+      } catch (IOException ioe) {
+        fail("expected only RuntimeException");
+      } catch (RuntimeException re) {
+        // Expected
+      }
+      assertTrue(dir.fileExists("1.fnx"));
+      assertTrue(failure.failOnCommit && failure.failOnDeleteFile);
+      w.rollback();
+      assertFalse(dir.fileExists("1.fnx"));
+      assertEquals(0, dir.listAll().length);
+      dir.close();
     }
-    assertTrue(failure.fail1 && failure.fail2);
-    w.rollback();
-    dir.close();
   }
   
   public void testOptimizeExceptions() throws IOException {

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java Tue Mar 22 11:12:09 2011
@@ -216,7 +216,7 @@ public class TestPerFieldCodecSupport ex
               IndexFileNames.FIELD_INFOS_EXTENSION));
       FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
       assertEquals("faild for segment index: " + i, codec[i],
-          codecInfo.codecs[fieldInfo.codecId]);
+          codecInfo.codecs[fieldInfo.getCodecId()]);
     }
   }
 

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java Tue Mar 22 11:12:09 2011
@@ -78,9 +78,10 @@ public class TestSegmentMerger extends L
     merger.add(reader2);
     int docsMerged = merger.merge();
     assertTrue(docsMerged == 2);
+    final FieldInfos fieldInfos = merger.fieldInfos();
     //Should be able to open a new SegmentReader against the new directory
-    SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, merger.fieldInfos().hasProx(),
-                                                                                     merger.getSegmentCodecs(), merger.fieldInfos().hasVectors()),
+    SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, fieldInfos.hasProx(),
+                                                                                     merger.getSegmentCodecs(), fieldInfos.hasVectors(), fieldInfos),
                                                    BufferedIndexInput.BUFFER_SIZE, true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
 
     assertTrue(mergedReader != null);