You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/03/22 12:12:10 UTC
svn commit: r1084134 [2/2] - in /lucene/dev/trunk/lucene: ./
src/java/org/apache/lucene/index/ src/java/org/apache/lucene/index/codecs/
src/java/org/apache/lucene/index/codecs/preflex/
src/test-framework/org/apache/lucene/util/ src/test/org/apache/luce...
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java Tue Mar 22 11:12:09 2011
@@ -25,10 +25,4 @@ abstract class TermsHashConsumer {
abstract TermsHashConsumerPerThread addThread(TermsHashPerThread perThread);
abstract void flush(Map<TermsHashConsumerPerThread,Collection<TermsHashConsumerPerField>> threadsAndFields, final SegmentWriteState state) throws IOException;
abstract void abort();
-
- FieldInfos fieldInfos;
-
- void setFieldInfos(FieldInfos fieldInfos) {
- this.fieldInfos = fieldInfos;
}
-}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java Tue Mar 22 11:12:09 2011
@@ -56,7 +56,9 @@ public class DefaultSegmentInfosReader e
infos.version = input.readLong(); // read version
infos.counter = input.readInt(); // read counter
-
+ if (infos.getFormat() <= DefaultSegmentInfosWriter.FORMAT_4_0) {
+ infos.setGlobalFieldMapVersion(input.readLong());
+ }
for (int i = input.readInt(); i > 0; i--) { // read segmentInfos
SegmentInfo si = new SegmentInfo(directory, format, input, codecs);
if (si.getVersion() == null) {
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java Tue Mar 22 11:12:09 2011
@@ -59,6 +59,7 @@ public class DefaultSegmentInfosWriter e
out.writeInt(FORMAT_CURRENT); // write FORMAT
out.writeLong(infos.version);
out.writeInt(infos.counter); // write counter
+ out.writeLong(infos.getGlobalFieldMapVersion());
out.writeInt(infos.size()); // write infos
for (SegmentInfo si : infos) {
si.write(out);
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java Tue Mar 22 11:12:09 2011
@@ -19,14 +19,15 @@ package org.apache.lucene.index.codecs.p
import java.io.IOException;
import java.util.Collection;
-import java.util.Iterator;
-import java.util.TreeMap;
+import java.util.Comparator;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
-import java.util.Comparator;
+import java.util.TreeMap;
-import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.CompoundFileReader;
import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FieldsEnum;
@@ -35,7 +36,6 @@ import org.apache.lucene.index.SegmentIn
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.CompoundFileReader;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
@@ -94,13 +94,11 @@ public class PreFlexFields extends Field
// so that if an index update removes them we'll still have them
freqStream = dir.openInput(info.name + ".frq", readBufferSize);
boolean anyProx = false;
- final int numFields = fieldInfos.size();
- for(int i=0;i<numFields;i++) {
- final FieldInfo fieldInfo = fieldInfos.fieldInfo(i);
- if (fieldInfo.isIndexed) {
- fields.put(fieldInfo.name, fieldInfo);
- preTerms.put(fieldInfo.name, new PreTerms(fieldInfo));
- if (!fieldInfo.omitTermFreqAndPositions) {
+ for (FieldInfo fi : fieldInfos) {
+ if (fi.isIndexed) {
+ fields.put(fi.name, fi);
+ preTerms.put(fi.name, new PreTerms(fi));
+ if (!fi.omitTermFreqAndPositions) {
anyProx = true;
}
}
Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java Tue Mar 22 11:12:09 2011
@@ -27,6 +27,7 @@ import java.io.OutputStream;
import java.io.PrintStream;
import java.lang.reflect.Method;
import java.util.Enumeration;
+import java.util.List;
import java.util.Random;
import java.util.Map;
import java.util.HashMap;
@@ -35,8 +36,11 @@ import java.util.zip.ZipFile;
import org.junit.Assert;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.index.ConcurrentMergeScheduler;
+import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.MergeScheduler;
@@ -325,4 +329,13 @@ public class _TestUtil {
throw new RuntimeException(e);
}
}
+
+ /** Adds field info for a Document. */
+ public static void add(Document doc, FieldInfos fieldInfos) {
+ List<Fieldable> fields = doc.getFields();
+ for (Fieldable field : fields) {
+ fieldInfos.add(field.name(), field.isIndexed(), field.isTermVectorStored(), field.isStorePositionWithTermVector(),
+ field.isStoreOffsetWithTermVector(), field.getOmitNorms(), false, field.getOmitTermFreqAndPositions());
+ }
+ }
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Tue Mar 22 11:12:09 2011
@@ -1045,8 +1045,8 @@ public class TestAddIndexes extends Luce
IndexWriter w3 = new IndexWriter(dir, conf);
w3.addIndexes(readers);
w3.close();
-
- assertEquals("Only one compound segment should exist", 3, dir.listAll().length);
+ // we should now see segments_X, segments.gen,_Y.cfs, _Z.fnx
+ assertEquals("Only one compound segment should exist", 4, dir.listAll().length);
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Tue Mar 22 11:12:09 2011
@@ -529,10 +529,9 @@ public class TestBackwardsCompatibility
CompoundFileReader cfsReader = new CompoundFileReader(dir, "_0.cfs");
FieldInfos fieldInfos = new FieldInfos(cfsReader, "_0.fnm");
int contentFieldIndex = -1;
- for(int i=0;i<fieldInfos.size();i++) {
- FieldInfo fi = fieldInfos.fieldInfo(i);
+ for (FieldInfo fi : fieldInfos) {
if (fi.name.equals("content")) {
- contentFieldIndex = i;
+ contentFieldIndex = fi.number;
break;
}
}
@@ -544,7 +543,8 @@ public class TestBackwardsCompatibility
"_0_1.del",
"_0_1.s" + contentFieldIndex,
"segments_2",
- "segments.gen"};
+ "segments.gen",
+ "1.fnx"};
String[] actual = dir.listAll();
Arrays.sort(expected);
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestCodecs.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestCodecs.java Tue Mar 22 11:12:09 2011
@@ -23,14 +23,14 @@ import java.util.HashSet;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.PostingsConsumer;
-import org.apache.lucene.index.codecs.TermsConsumer;
import org.apache.lucene.index.codecs.TermStats;
+import org.apache.lucene.index.codecs.TermsConsumer;
import org.apache.lucene.index.codecs.mocksep.MockSepCodec;
import org.apache.lucene.index.codecs.preflex.PreFlexCodec;
import org.apache.lucene.search.DocIdSetIterator;
@@ -238,8 +238,9 @@ public class TestCodecs extends LuceneTe
final FieldData[] fields = new FieldData[] {field};
final Directory dir = newDirectory();
+ FieldInfos clonedFieldInfos = (FieldInfos) fieldInfos.clone();
this.write(fieldInfos, dir, fields, true);
- final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()), fieldInfos.hasVectors());
+ final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecs.build(clonedFieldInfos, CodecProvider.getDefault()), clonedFieldInfos.hasVectors(), clonedFieldInfos);
si.setHasProx(false);
final FieldsProducer reader = si.getSegmentCodecs().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 64, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR));
@@ -289,8 +290,10 @@ public class TestCodecs extends LuceneTe
if (VERBOSE) {
System.out.println("TEST: now write postings");
}
+
+ FieldInfos clonedFieldInfos = (FieldInfos) fieldInfos.clone();
this.write(fieldInfos, dir, fields, false);
- final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()), fieldInfos.hasVectors());
+ final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecs.build(clonedFieldInfos, CodecProvider.getDefault()), clonedFieldInfos.hasVectors(), clonedFieldInfos);
if (VERBOSE) {
System.out.println("TEST: now read postings");
@@ -440,7 +443,7 @@ public class TestCodecs extends LuceneTe
final FieldData field = fields[TestCodecs.random.nextInt(fields.length)];
final TermsEnum termsEnum = termsDict.terms(field.fieldInfo.name).iterator();
- if (si.getSegmentCodecs().codecs[field.fieldInfo.codecId] instanceof PreFlexCodec) {
+ if (si.getSegmentCodecs().codecs[field.fieldInfo.getCodecId()] instanceof PreFlexCodec) {
// code below expects unicode sort order
continue;
}
@@ -594,7 +597,7 @@ public class TestCodecs extends LuceneTe
final FieldsConsumer consumer = state.segmentCodecs.codec().fieldsConsumer(state);
Arrays.sort(fields);
for (final FieldData field : fields) {
- if (!allowPreFlex && codecInfo.codecs[field.fieldInfo.codecId] instanceof PreFlexCodec) {
+ if (!allowPreFlex && codecInfo.codecs[field.fieldInfo.getCodecId()] instanceof PreFlexCodec) {
// code below expects unicode sort order
continue;
}
Added: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java?rev=1084134&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java (added)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java Tue Mar 22 11:12:09 2011
@@ -0,0 +1,338 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.Field.TermVector;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Test;
+
+public class TestConsistentFieldNumbers extends LuceneTestCase {
+
+ @Test
+ public void testSameFieldNumbersAcrossSegments() throws Exception {
+ for (int i = 0; i < 2; i++) {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(NoMergePolicy.COMPOUND_FILES));
+
+ Document d1 = new Document();
+ d1.add(new Field("f1", "first field", Store.YES, Index.ANALYZED, TermVector.NO));
+ d1.add(new Field("f2", "second field", Store.YES, Index.ANALYZED, TermVector.NO));
+ writer.addDocument(d1);
+
+ if (i == 1) {
+ writer.close();
+ writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(NoMergePolicy.COMPOUND_FILES));
+ } else {
+ writer.commit();
+ }
+
+ Document d2 = new Document();
+ d2.add(new Field("f2", "second field", Store.YES, Index.ANALYZED, TermVector.NO));
+ d2.add(new Field("f1", "first field", Store.YES, Index.ANALYZED, TermVector.YES));
+ d2.add(new Field("f3", "third field", Store.YES, Index.ANALYZED, TermVector.NO));
+ d2.add(new Field("f4", "fourth field", Store.YES, Index.ANALYZED, TermVector.NO));
+ writer.addDocument(d2);
+
+ writer.close();
+
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(dir);
+ assertEquals(2, sis.size());
+
+ FieldInfos fis1 = sis.info(0).getFieldInfos();
+ FieldInfos fis2 = sis.info(1).getFieldInfos();
+
+ assertEquals("f1", fis1.fieldInfo(0).name);
+ assertEquals("f2", fis1.fieldInfo(1).name);
+ assertEquals("f1", fis2.fieldInfo(0).name);
+ assertEquals("f2", fis2.fieldInfo(1).name);
+ assertEquals("f3", fis2.fieldInfo(2).name);
+ assertEquals("f4", fis2.fieldInfo(3).name);
+
+ writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()));
+ writer.optimize();
+ writer.close();
+
+ sis = new SegmentInfos();
+ sis.read(dir);
+ assertEquals(1, sis.size());
+
+ FieldInfos fis3 = sis.info(0).getFieldInfos();
+
+ assertEquals("f1", fis3.fieldInfo(0).name);
+ assertEquals("f2", fis3.fieldInfo(1).name);
+ assertEquals("f3", fis3.fieldInfo(2).name);
+ assertEquals("f4", fis3.fieldInfo(3).name);
+
+
+ dir.close();
+ }
+ }
+
+ @Test
+ public void testAddIndexes() throws Exception {
+ Directory dir1 = newDirectory();
+ Directory dir2 = newDirectory();
+ IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(NoMergePolicy.COMPOUND_FILES));
+
+ Document d1 = new Document();
+ d1.add(new Field("f1", "first field", Store.YES, Index.ANALYZED, TermVector.NO));
+ d1.add(new Field("f2", "second field", Store.YES, Index.ANALYZED, TermVector.NO));
+ writer.addDocument(d1);
+
+ writer.close();
+ writer = new IndexWriter(dir2, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(NoMergePolicy.COMPOUND_FILES));
+
+ Document d2 = new Document();
+ d2.add(new Field("f2", "second field", Store.YES, Index.ANALYZED, TermVector.NO));
+ d2.add(new Field("f1", "first field", Store.YES, Index.ANALYZED, TermVector.YES));
+ d2.add(new Field("f3", "third field", Store.YES, Index.ANALYZED, TermVector.NO));
+ d2.add(new Field("f4", "fourth field", Store.YES, Index.ANALYZED, TermVector.NO));
+ writer.addDocument(d2);
+
+ writer.close();
+
+ writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(NoMergePolicy.COMPOUND_FILES));
+ writer.addIndexes(dir2);
+ writer.close();
+
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(dir1);
+ assertEquals(2, sis.size());
+
+ FieldInfos fis1 = sis.info(0).getFieldInfos();
+ FieldInfos fis2 = sis.info(1).getFieldInfos();
+
+ assertEquals("f1", fis1.fieldInfo(0).name);
+ assertEquals("f2", fis1.fieldInfo(1).name);
+ // make sure the ordering of the "external" segment is preserved
+ assertEquals("f2", fis2.fieldInfo(0).name);
+ assertEquals("f1", fis2.fieldInfo(1).name);
+ assertEquals("f3", fis2.fieldInfo(2).name);
+ assertEquals("f4", fis2.fieldInfo(3).name);
+
+ writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()));
+ writer.optimize();
+ writer.close();
+
+ sis = new SegmentInfos();
+ sis.read(dir1);
+ assertEquals(1, sis.size());
+
+ FieldInfos fis3 = sis.info(0).getFieldInfos();
+
+ // after merging the ordering should be identical to the first segment
+ assertEquals("f1", fis3.fieldInfo(0).name);
+ assertEquals("f2", fis3.fieldInfo(1).name);
+ assertEquals("f3", fis3.fieldInfo(2).name);
+ assertEquals("f4", fis3.fieldInfo(3).name);
+
+ dir1.close();
+ dir2.close();
+ }
+
+ public void testFieldNumberGaps() throws IOException {
+ for (int i = 0; i < 39; i++) {
+ Directory dir = newDirectory();
+ {
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(
+ NoMergePolicy.NO_COMPOUND_FILES));
+ Document d = new Document();
+ d.add(new Field("f1", "d1 first field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ d.add(new Field("f2", "d1 second field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ writer.addDocument(d);
+ writer.close();
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(dir);
+ assertEquals(1, sis.size());
+ FieldInfos fis1 = sis.info(0).getFieldInfos();
+ assertEquals("f1", fis1.fieldInfo(0).name);
+ assertEquals("f2", fis1.fieldInfo(1).name);
+ assertTrue(dir.fileExists("1.fnx"));
+ }
+
+
+ {
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(
+ random.nextBoolean() ? NoMergePolicy.NO_COMPOUND_FILES
+ : NoMergePolicy.COMPOUND_FILES));
+ Document d = new Document();
+ d.add(new Field("f1", "d2 first field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ d.add(new Field("f3", new byte[] { 1, 2, 3 }));
+ writer.addDocument(d);
+ writer.close();
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(dir);
+ assertEquals(2, sis.size());
+ FieldInfos fis1 = sis.info(0).getFieldInfos();
+ FieldInfos fis2 = sis.info(1).getFieldInfos();
+ assertEquals("f1", fis1.fieldInfo(0).name);
+ assertEquals("f2", fis1.fieldInfo(1).name);
+ assertEquals("f1", fis2.fieldInfo(0).name);
+ assertNull(fis2.fieldInfo(1));
+ assertEquals("f3", fis2.fieldInfo(2).name);
+ assertFalse(dir.fileExists("1.fnx"));
+ assertTrue(dir.fileExists("2.fnx"));
+ }
+
+ {
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(
+ random.nextBoolean() ? NoMergePolicy.NO_COMPOUND_FILES
+ : NoMergePolicy.COMPOUND_FILES));
+ Document d = new Document();
+ d.add(new Field("f1", "d3 first field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ d.add(new Field("f2", "d3 second field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ d.add(new Field("f3", new byte[] { 1, 2, 3, 4, 5 }));
+ writer.addDocument(d);
+ writer.close();
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(dir);
+ assertEquals(3, sis.size());
+ FieldInfos fis1 = sis.info(0).getFieldInfos();
+ FieldInfos fis2 = sis.info(1).getFieldInfos();
+ FieldInfos fis3 = sis.info(2).getFieldInfos();
+ assertEquals("f1", fis1.fieldInfo(0).name);
+ assertEquals("f2", fis1.fieldInfo(1).name);
+ assertEquals("f1", fis2.fieldInfo(0).name);
+ assertNull(fis2.fieldInfo(1));
+ assertEquals("f3", fis2.fieldInfo(2).name);
+ assertEquals("f1", fis3.fieldInfo(0).name);
+ assertEquals("f2", fis3.fieldInfo(1).name);
+ assertEquals("f3", fis3.fieldInfo(2).name);
+ assertFalse(dir.fileExists("1.fnx"));
+ assertTrue(dir.fileExists("2.fnx"));
+ assertFalse(dir.fileExists("3.fnx"));
+ }
+
+ {
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(
+ random.nextBoolean() ? NoMergePolicy.NO_COMPOUND_FILES
+ : NoMergePolicy.COMPOUND_FILES));
+ writer.deleteDocuments(new Term("f1", "d1"));
+ // nuke the first segment entirely so that the segment with gaps is
+ // loaded first!
+ writer.expungeDeletes();
+ writer.close();
+ }
+
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(
+ new LogByteSizeMergePolicy()));
+ writer.optimize();
+ assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges);
+ writer.close();
+
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(dir);
+ assertEquals(1, sis.size());
+ FieldInfos fis1 = sis.info(0).getFieldInfos();
+ assertEquals("f1", fis1.fieldInfo(0).name);
+ assertEquals("f2", fis1.fieldInfo(1).name);
+ assertEquals("f3", fis1.fieldInfo(2).name);
+ assertFalse(dir.fileExists("1.fnx"));
+ assertTrue(dir.fileExists("2.fnx"));
+ assertFalse(dir.fileExists("3.fnx"));
+ dir.close();
+ }
+ }
+
+ @Test
+ public void testManyFields() throws Exception {
+ final int NUM_DOCS = 2000;
+ final int MAX_FIELDS = 50;
+
+ int[][] docs = new int[NUM_DOCS][4];
+ for (int i = 0; i < docs.length; i++) {
+ for (int j = 0; j < docs[i].length;j++) {
+ docs[i][j] = random.nextInt(MAX_FIELDS);
+ }
+ }
+
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()));
+
+ for (int i = 0; i < NUM_DOCS; i++) {
+ Document d = new Document();
+ for (int j = 0; j < docs[i].length; j++) {
+ d.add(getField(docs[i][j]));
+ }
+
+ writer.addDocument(d);
+ }
+
+ writer.optimize();
+ writer.close();
+
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(dir);
+ for (SegmentInfo si : sis) {
+ FieldInfos fis = si.getFieldInfos();
+
+ for (FieldInfo fi : fis) {
+ Field expected = getField(Integer.parseInt(fi.name));
+ assertEquals(expected.isIndexed(), fi.isIndexed);
+ assertEquals(expected.isTermVectorStored(), fi.storeTermVector);
+ assertEquals(expected.isStorePositionWithTermVector(), fi.storePositionWithTermVector);
+ assertEquals(expected.isStoreOffsetWithTermVector(), fi.storeOffsetWithTermVector);
+ }
+ }
+
+ dir.close();
+ }
+
+ private Field getField(int number) {
+ int mode = number % 16;
+ String fieldName = "" + number;
+ switch (mode) {
+ case 0: return new Field(fieldName, "some text", Store.YES, Index.ANALYZED, TermVector.NO);
+ case 1: return new Field(fieldName, "some text", Store.NO, Index.ANALYZED, TermVector.NO);
+ case 2: return new Field(fieldName, "some text", Store.YES, Index.NOT_ANALYZED, TermVector.NO);
+ case 3: return new Field(fieldName, "some text", Store.NO, Index.NOT_ANALYZED, TermVector.NO);
+ case 4: return new Field(fieldName, "some text", Store.YES, Index.ANALYZED, TermVector.WITH_OFFSETS);
+ case 5: return new Field(fieldName, "some text", Store.NO, Index.ANALYZED, TermVector.WITH_OFFSETS);
+ case 6: return new Field(fieldName, "some text", Store.YES, Index.NOT_ANALYZED, TermVector.WITH_OFFSETS);
+ case 7: return new Field(fieldName, "some text", Store.NO, Index.NOT_ANALYZED, TermVector.WITH_OFFSETS);
+ case 8: return new Field(fieldName, "some text", Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS);
+ case 9: return new Field(fieldName, "some text", Store.NO, Index.ANALYZED, TermVector.WITH_POSITIONS);
+ case 10: return new Field(fieldName, "some text", Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS);
+ case 11: return new Field(fieldName, "some text", Store.NO, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS);
+ case 12: return new Field(fieldName, "some text", Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS);
+ case 13: return new Field(fieldName, "some text", Store.NO, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS);
+ case 14: return new Field(fieldName, "some text", Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS);
+ case 15: return new Field(fieldName, "some text", Store.NO, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS);
+ default: return null;
+ }
+ }
+}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDoc.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDoc.java Tue Mar 22 11:12:09 2011
@@ -202,10 +202,10 @@ public class TestDoc extends LuceneTestC
merger.merge();
r1.close();
r2.close();
-
+ final FieldInfos fieldInfos = merger.fieldInfos();
final SegmentInfo info = new SegmentInfo(merged, si1.docCount + si2.docCount, si1.dir,
- false, merger.fieldInfos().hasProx(), merger.getSegmentCodecs(),
- merger.fieldInfos().hasVectors());
+ false, fieldInfos.hasProx(), merger.getSegmentCodecs(),
+ fieldInfos.hasVectors(), fieldInfos);
if (useCompoundFile) {
Collection<String> filesToDelete = merger.createCompoundFile(merged + ".cfs", info);
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java Tue Mar 22 11:12:09 2011
@@ -25,20 +25,20 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field.TermVector;
+import org.apache.lucene.document.Fieldable;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.util.BytesRef;
public class TestDocumentWriter extends LuceneTestCase {
private Directory dir;
@@ -98,8 +98,7 @@ public class TestDocumentWriter extends
// test that the norms are not present in the segment if
// omitNorms is true
- for (int i = 0; i < reader.core.fieldInfos.size(); i++) {
- FieldInfo fi = reader.core.fieldInfos.fieldInfo(i);
+ for (FieldInfo fi : reader.core.fieldInfos) {
if (fi.isIndexed) {
assertTrue(fi.omitNorms == !reader.hasNorms(fi.name));
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java Tue Mar 22 11:12:09 2011
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
import org.apache.lucene.document.Document;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexOutput;
@@ -40,7 +41,7 @@ public class TestFieldInfos extends Luce
//Positive test of FieldInfos
assertTrue(testDoc != null);
FieldInfos fieldInfos = new FieldInfos();
- fieldInfos.add(testDoc);
+ _TestUtil.add(testDoc, fieldInfos);
//Since the complement is stored as well in the fields map
assertTrue(fieldInfos.size() == DocHelper.all.size()); //this is all b/c we are using the no-arg constructor
Directory dir = newDirectory();
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java Tue Mar 22 11:12:09 2011
@@ -49,7 +49,7 @@ public class TestFieldsReader extends Lu
super.setUp();
fieldInfos = new FieldInfos();
DocHelper.setupDoc(testDoc);
- fieldInfos.add(testDoc);
+ _TestUtil.add(testDoc, fieldInfos);
dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newLogMergePolicy());
((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(false);
Added: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java?rev=1084134&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java (added)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java Tue Mar 22 11:12:09 2011
@@ -0,0 +1,537 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.Field.TermVector;
+import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
+import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+
+public class TestGlobalFieldNumbers extends LuceneTestCase {
+
+ public void testGlobalFieldNumberFiles() throws IOException {
+ for (int i = 0; i < 39; i++) {
+ Directory dir = newDirectory();
+ {
+ IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer());
+ IndexWriter writer = new IndexWriter(dir, config);
+ Document d = new Document();
+ d.add(new Field("f1", "d1 first field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ d.add(new Field("f2", "d1 second field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ writer.addDocument(d);
+ Collection<String> files = writer.segmentInfos.files(dir, true);
+ for (String string : files) {
+ assertFalse(string.endsWith(".fnx"));
+ }
+ writer.commit();
+ files = writer.segmentInfos.files(dir, true);
+ files.remove("1.fnx");
+ for (String string : files) {
+ assertFalse(string.endsWith(".fnx"));
+ }
+
+ assertFNXFiles(dir, "1.fnx");
+ d = new Document();
+ d.add(new Field("f1", "d2 first field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ d.add(new Field("f3", new byte[] { 1, 2, 3 }));
+ writer.addDocument(d);
+ writer.commit();
+ files = writer.segmentInfos.files(dir, true);
+ files.remove("2.fnx");
+ for (String string : files) {
+ assertFalse(string.endsWith(".fnx"));
+ }
+ assertFNXFiles(dir, "2.fnx");
+ writer.close();
+ assertFNXFiles(dir, "2.fnx");
+ }
+
+ {
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()));
+ Document d = new Document();
+ d.add(new Field("f1", "d3 first field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ d.add(new Field("f2", "d3 second field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ d.add(new Field("f3", new byte[] { 1, 2, 3, 4, 5 }));
+ writer.addDocument(d);
+ writer.close();
+ Collection<String> files = writer.segmentInfos.files(dir, true);
+ files.remove("2.fnx");
+ for (String string : files) {
+ assertFalse(string.endsWith(".fnx"));
+ }
+
+ assertFNXFiles(dir, "2.fnx");
+ }
+
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()));
+ writer.optimize();
+ assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges);
+ writer.close();
+ assertFNXFiles(dir, "2.fnx");
+
+ dir.close();
+ }
+ }
+
+ public void testIndexReaderCommit() throws IOException {
+ for (int i = 0; i < 39; i++) {
+ Directory dir = newDirectory();
+ {
+ IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer());
+ IndexWriter writer = new IndexWriter(dir, config);
+ Document d = new Document();
+ d.add(new Field("f1", "d1 first field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ d.add(new Field("f2", "d1 second field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ writer.addDocument(d);
+ writer.commit();
+ assertFNXFiles(dir, "1.fnx");
+ d = new Document();
+ d.add(new Field("f1", "d2 first field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ d.add(new Field("f3", new byte[] { 1, 2, 3 }));
+ writer.addDocument(d);
+ writer.commit();
+ assertFNXFiles(dir, "2.fnx");
+ writer.close();
+ assertFNXFiles(dir, "2.fnx");
+ }
+ IndexReader reader = IndexReader.open(dir, false);
+ reader.deleteDocument(0);
+ reader.commit();
+ reader.close();
+ // make sure this reader can not modify the field map
+ assertFNXFiles(dir, "2.fnx");
+
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()));
+ writer.optimize();
+ assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges);
+ writer.close();
+ assertFNXFiles(dir, "2.fnx");
+
+ dir.close();
+ }
+ }
+
+ public void testGlobalFieldNumberFilesAcrossCommits() throws IOException {
+ for (int i = 0; i < 39; i++) {
+ Directory dir = newDirectory();
+ {
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()).setIndexDeletionPolicy(
+ new KeepAllDeletionPolicy()));
+ Document d = new Document();
+ d.add(new Field("f1", "d1 first field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ d.add(new Field("f2", "d1 second field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ writer.addDocument(d);
+ writer.commit();
+ assertFNXFiles(dir, "1.fnx");
+ d = new Document();
+ d.add(new Field("f1", "d2 first field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ d.add(new Field("f3", new byte[] { 1, 2, 3 }));
+ writer.addDocument(d);
+ writer.commit();
+ writer.commit();
+ writer.commit();
+ assertFNXFiles(dir, "1.fnx", "2.fnx");
+ writer.close();
+ assertFNXFiles(dir, "1.fnx", "2.fnx");
+ }
+
+ {
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()));
+ Document d = new Document();
+ d.add(new Field("f1", "d3 first field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ d.add(new Field("f2", "d3 second field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ d.add(new Field("f3", new byte[] { 1, 2, 3, 4, 5 }));
+ writer.addDocument(d);
+ writer.close();
+ assertFNXFiles(dir, "2.fnx");
+ }
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()));
+ writer.optimize();
+ assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges);
+ writer.close();
+ assertFNXFiles(dir, "2.fnx");
+ dir.close();
+ }
+ }
+
+ public void testGlobalFieldNumberOnOldCommit() throws IOException {
+ for (int i = 0; i < 39; i++) {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()).setIndexDeletionPolicy(
+ new KeepAllDeletionPolicy()));
+ Document d = new Document();
+ d.add(new Field("f1", "d1 first field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ d.add(new Field("f2", "d1 second field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ writer.addDocument(d);
+ writer.commit();
+ assertFNXFiles(dir, "1.fnx");
+ d = new Document();
+ d.add(new Field("f1", "d2 first field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ d.add(new Field("f3", new byte[] { 1, 2, 3 }));
+ writer.addDocument(d);
+ assertFNXFiles(dir, "1.fnx");
+ writer.close();
+ assertFNXFiles(dir, "1.fnx", "2.fnx");
+ // open first commit
+ List<IndexCommit> listCommits = IndexReader.listCommits(dir);
+ assertEquals(2, listCommits.size());
+ writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer()).setIndexDeletionPolicy(
+ new KeepAllDeletionPolicy()).setIndexCommit(listCommits.get(0)));
+
+ d = new Document();
+ d.add(new Field("f1", "d2 first field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ d.add(new Field("f3", new byte[] { 1, 2, 3 }));
+ writer.addDocument(d);
+ writer.commit();
+ // now we have 3 files since f3 is not present in the first commit
+ assertFNXFiles(dir, "1.fnx", "2.fnx", "3.fnx");
+ writer.close();
+ assertFNXFiles(dir, "1.fnx", "2.fnx", "3.fnx");
+
+ writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer()));
+ writer.commit();
+ listCommits = IndexReader.listCommits(dir);
+ assertEquals(1, listCommits.size());
+ assertFNXFiles(dir, "3.fnx");
+ writer.close();
+ assertFNXFiles(dir, "3.fnx");
+ dir.close();
+ }
+ }
+
+ private final Directory buildRandomIndex(String[] fieldNames, int numDocs,
+ IndexWriterConfig conf) throws CorruptIndexException,
+ LockObtainFailedException, IOException {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, conf);
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ final int numFields = 1 + random.nextInt(fieldNames.length);
+ for (int j = 0; j < numFields; j++) {
+ doc.add(newField(fieldNames[random.nextInt(fieldNames.length)],
+ _TestUtil.randomRealisticUnicodeString(random),
+ Index.toIndex(true, random.nextBoolean(), random.nextBoolean())));
+
+ }
+ writer.addDocument(doc);
+ if (random.nextInt(20) == 0) {
+ writer.commit();
+ }
+ }
+ writer.close();
+ return dir;
+ }
+
+ public void testOptimize() throws IOException {
+ for (int i = 0; i < 2; i++) {
+ Set<String> fieldNames = new HashSet<String>();
+ final int numFields = 2 + random.nextInt(200);
+ for (int j = 0; j < numFields; j++) {
+ fieldNames.add("field_" + j);
+ }
+ Directory base = buildRandomIndex(fieldNames.toArray(new String[0]),
+ 20 + random.nextInt(100),
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ IndexWriter writer = new IndexWriter(base, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()));
+ FieldNumberBiMap globalFieldMap = writer.segmentInfos
+ .getOrLoadGlobalFieldNumberMap(base);
+ Set<Entry<String, Integer>> entries = globalFieldMap.entries();
+ writer.optimize();
+ writer.commit();
+ writer.close();
+ Set<Entry<String, Integer>> afterOptmize = globalFieldMap.entries();
+ assertEquals(entries, afterOptmize);
+ base.close();
+ }
+ }
+
+ public void testAddIndexesStableFieldNumbers() throws IOException {
+ for (int i = 0; i < 2; i++) {
+ Set<String> fieldNames = new HashSet<String>();
+ final int numFields = 2 + random.nextInt(50);
+ for (int j = 0; j < numFields; j++) {
+ fieldNames.add("field_" + j);
+ }
+
+ Directory base = newDirectory();
+ IndexWriter writer = new IndexWriter(base, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()));
+ Document doc = new Document();
+ for (String string : fieldNames) {
+ doc.add(newField(string,
+ _TestUtil.randomRealisticUnicodeString(random),
+ Index.toIndex(true, random.nextBoolean(), random.nextBoolean())));
+
+ }
+ writer.addDocument(doc);
+ writer.commit();
+ FieldNumberBiMap globalFieldMap = writer.segmentInfos
+ .getOrLoadGlobalFieldNumberMap(base);
+ final Set<Entry<String, Integer>> entries = globalFieldMap.entries();
+ assertEquals(entries.size(), fieldNames.size());
+ for (Entry<String, Integer> entry : entries) {
+ // all fields are in this fieldMap
+ assertTrue(fieldNames.contains(entry.getKey()));
+ }
+ writer.close();
+
+ int numIndexes = 1 + random.nextInt(10);
+ for (int j = 0; j < numIndexes; j++) {
+ Directory toAdd = buildRandomIndex(fieldNames.toArray(new String[0]),
+ 1 + random.nextInt(50),
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ IndexWriter w = new IndexWriter(base, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()));
+ if (random.nextBoolean()) {
+ IndexReader open = IndexReader.open(toAdd);
+ w.addIndexes(open);
+ open.close();
+ } else {
+ w.addIndexes(toAdd);
+ }
+
+ w.close();
+ FieldNumberBiMap map = w.segmentInfos
+ .getOrLoadGlobalFieldNumberMap(toAdd);
+ assertEquals(entries, map.entries());
+ toAdd.close();
+ }
+ IndexWriter w = new IndexWriter(base, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(
+ new LogByteSizeMergePolicy()));
+ w.optimize();
+ w.close();
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(base);
+ SegmentInfo segmentInfo = sis.get(sis.size() - 1);// last segment must
+ // have all fields with
+ // consistent numbers
+ FieldInfos fieldInfos = segmentInfo.getFieldInfos();
+ assertEquals(fieldInfos.size(), entries.size());
+ for (Entry<String, Integer> entry : entries) {
+ assertEquals(entry.getValue(),
+ Integer.valueOf(fieldInfos.fieldNumber(entry.getKey())));
+ assertEquals(entry.getKey(), fieldInfos.fieldName(entry.getValue()));
+ }
+ base.close();
+ }
+ }
+
+ final String[] oldNames = { "30.cfs", "30.nocfs", "31.cfs", "31.nocfs", };
+
+ public void testAddOldIndex() throws IOException {
+ int i = random.nextInt(oldNames.length);
+ File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
+ Directory dir = null;
+ try {
+ _TestUtil
+ .unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
+ dir = newFSDirectory(oldIndxeDir);
+ SegmentInfos infos = new SegmentInfos();
+ infos.read(dir);
+ SortedMap<Integer, String> sortedMap = new TreeMap<Integer, String>();
+
+ FieldNumberBiMap biMap = new FieldNumberBiMap();
+ int maxFieldNum = Integer.MIN_VALUE;
+ for (SegmentInfo segmentInfo : infos) {
+ for (FieldInfo fieldInfo : segmentInfo.getFieldInfos()) {
+ int globNumber = biMap.addOrGet(fieldInfo.name, fieldInfo.number);
+ maxFieldNum = Math.max(maxFieldNum, globNumber);
+ sortedMap.put(globNumber, fieldInfo.name);
+ }
+ }
+ Directory base = newDirectory();
+ IndexWriter writer = new IndexWriter(base, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(
+ NoMergePolicy.NO_COMPOUND_FILES));
+
+ SortedMap<Integer, String> copySortedMap = new TreeMap<Integer, String>(
+ sortedMap);
+ while (!sortedMap.isEmpty()) { // add every field at least once
+ Document doc = new Document();
+ int nextField = random.nextInt(maxFieldNum + 1);
+ sortedMap.remove(nextField);
+
+ String name = copySortedMap.get(nextField);
+ assertNotNull(name);
+
+ doc.add(newField(name, _TestUtil.randomRealisticUnicodeString(random),
+ Index.toIndex(true, random.nextBoolean(), random.nextBoolean())));
+ writer.addDocument(doc);
+ if (random.nextInt(10) == 0) {
+ writer.commit();
+ }
+ }
+ Set<Entry<String, Integer>> expectedEntries = writer.segmentInfos
+ .getOrLoadGlobalFieldNumberMap(base).entries();
+ writer.addIndexes(dir); // add the old index
+ writer.close();
+
+ writer = new IndexWriter(base, newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer()).setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES));
+ writer.commit(); // make sure the old index is the latest segment
+ writer.close();
+
+ // we don't merge here since we use NoMergePolicy
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(base);
+ // check that the latest global field numbers are consistent and carried
+ // over from the 4.0 index
+ FieldNumberBiMap actualGlobalMap = sis
+ .getOrLoadGlobalFieldNumberMap(base);
+ assertEquals(expectedEntries, actualGlobalMap.entries());
+ base.close();
+ } finally {
+ if (dir != null)
+ dir.close();
+ _TestUtil.rmDir(oldIndxeDir);
+ }
+ }
+
+ public void testFilesOnOldIndex() throws IOException {
+ int i = random.nextInt(oldNames.length);
+ File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
+ Directory dir = null;
+
+ MergePolicy policy = random.nextBoolean() ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES;
+ try {
+ _TestUtil
+ .unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
+ dir = newFSDirectory(oldIndxeDir);
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(policy));
+ SegmentInfos segmentInfos = writer.segmentInfos;
+ assertTrue(DefaultSegmentInfosWriter.FORMAT_4_0 < segmentInfos.getFormat());
+ assertEquals(0, segmentInfos.getGlobalFieldMapVersion());
+ Collection<String> files = segmentInfos.files(dir, true);
+ for (String string : files) {
+ assertFalse(string.endsWith(".fnx"));
+ }
+ writer.commit();
+
+ assertTrue(DefaultSegmentInfosWriter.FORMAT_4_0 < segmentInfos.getFormat());
+ assertEquals(0, segmentInfos.getGlobalFieldMapVersion());
+ files = segmentInfos.files(dir, true);
+ for (String string : files) {
+ assertFalse(string.endsWith(".fnx"));
+ }
+
+ Document d = new Document();
+ d.add(new Field("f1", "d1 first field", Store.YES, Index.ANALYZED,
+ TermVector.NO));
+ writer.addDocument(d);
+ writer.prepareCommit();
+ // the fnx file should still be under control of the SIS
+ assertTrue(DefaultSegmentInfosWriter.FORMAT_4_0 < segmentInfos.getFormat());
+ assertEquals(0, segmentInfos.getLastGlobalFieldMapVersion());
+ assertEquals(1, segmentInfos.getGlobalFieldMapVersion());
+ files = segmentInfos.files(dir, true);
+ for (String string : files) {
+ assertFalse(string.endsWith(".fnx"));
+ }
+
+ writer.commit();
+
+ // now we should see the fnx file even if this is a 3.x segment
+ assertTrue(DefaultSegmentInfosWriter.FORMAT_4_0 < segmentInfos.getFormat());
+ assertEquals(1, segmentInfos.getGlobalFieldMapVersion());
+ assertEquals(1, segmentInfos.getLastGlobalFieldMapVersion());
+ files = segmentInfos.files(dir, true);
+ assertTrue(files.remove("1.fnx"));
+ for (String string : files) {
+ assertFalse(string.endsWith(".fnx"));
+ }
+ writer.close();
+ } finally {
+ if (dir != null)
+ dir.close();
+ _TestUtil.rmDir(oldIndxeDir);
+ }
+ }
+
+ class KeepAllDeletionPolicy implements IndexDeletionPolicy {
+ public void onInit(List<? extends IndexCommit> commits) throws IOException {
+ }
+
+ public void onCommit(List<? extends IndexCommit> commits)
+ throws IOException {
+ }
+ }
+
+ public static void assertFNXFiles(Directory dir, String... expectedFnxFiles)
+ throws IOException {
+ String[] listAll = dir.listAll();
+ Set<String> fnxFiles = new HashSet<String>();
+ for (String string : listAll) {
+ if (string.endsWith(".fnx")) {
+ fnxFiles.add(string);
+ }
+ }
+ assertEquals("" + fnxFiles, expectedFnxFiles.length, fnxFiles.size());
+ for (String string : expectedFnxFiles) {
+ assertTrue(" missing fnx file: " + string, fnxFiles.contains(string));
+ }
+ }
+
+}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java Tue Mar 22 11:12:09 2011
@@ -92,10 +92,9 @@ public class TestIndexFileDeleter extend
CompoundFileReader cfsReader = new CompoundFileReader(dir, "_2.cfs");
FieldInfos fieldInfos = new FieldInfos(cfsReader, "_2.fnm");
int contentFieldIndex = -1;
- for(i=0;i<fieldInfos.size();i++) {
- FieldInfo fi = fieldInfos.fieldInfo(i);
+ for (FieldInfo fi : fieldInfos) {
if (fi.name.equals("content")) {
- contentFieldIndex = i;
+ contentFieldIndex = fi.number;
break;
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Tue Mar 22 11:12:09 2011
@@ -33,7 +33,9 @@ import org.apache.lucene.search.DocIdSet
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.store.MockDirectoryWrapper.Failure;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -780,7 +782,6 @@ public class TestIndexWriterExceptions e
}
}
}
-
((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).sync();
assertTrue(failure.didFail);
failure.clearDoFail();
@@ -794,53 +795,79 @@ public class TestIndexWriterExceptions e
private static class FailOnlyInCommit extends MockDirectoryWrapper.Failure {
- boolean fail1, fail2;
+ boolean failOnCommit, failOnDeleteFile;
+ private final boolean dontFailDuringGlobalFieldMap;
+ private static final String PREPARE_STAGE = "prepareCommit";
+ private static final String FINISH_STAGE = "finishCommit";
+ private final String stage;
+
+ public FailOnlyInCommit(boolean dontFailDuringGlobalFieldMap, String stage) {
+ this.dontFailDuringGlobalFieldMap = dontFailDuringGlobalFieldMap;
+ this.stage = stage;
+ }
@Override
public void eval(MockDirectoryWrapper dir) throws IOException {
StackTraceElement[] trace = new Exception().getStackTrace();
boolean isCommit = false;
boolean isDelete = false;
+ boolean isInGlobalFieldMap = false;
for (int i = 0; i < trace.length; i++) {
- if ("org.apache.lucene.index.SegmentInfos".equals(trace[i].getClassName()) && "prepareCommit".equals(trace[i].getMethodName()))
+ if ("org.apache.lucene.index.SegmentInfos".equals(trace[i].getClassName()) && stage.equals(trace[i].getMethodName()))
isCommit = true;
if ("org.apache.lucene.store.MockDirectoryWrapper".equals(trace[i].getClassName()) && "deleteFile".equals(trace[i].getMethodName()))
isDelete = true;
+ if ("org.apache.lucene.index.SegmentInfos".equals(trace[i].getClassName()) && "writeGlobalFieldMap".equals(trace[i].getMethodName()))
+ isInGlobalFieldMap = true;
+
+ }
+ if (isInGlobalFieldMap && dontFailDuringGlobalFieldMap) {
+ isCommit = false;
}
-
if (isCommit) {
if (!isDelete) {
- fail1 = true;
+ failOnCommit = true;
throw new RuntimeException("now fail first");
} else {
- fail2 = true;
+ failOnDeleteFile = true;
throw new IOException("now fail during delete");
}
}
}
}
- // LUCENE-1214
public void testExceptionsDuringCommit() throws Throwable {
- MockDirectoryWrapper dir = newDirectory();
- FailOnlyInCommit failure = new FailOnlyInCommit();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()));
- Document doc = new Document();
- doc.add(newField("field", "a field", Field.Store.YES,
- Field.Index.ANALYZED));
- w.addDocument(doc);
- dir.failOn(failure);
- try {
- w.close();
- fail();
- } catch (IOException ioe) {
- fail("expected only RuntimeException");
- } catch (RuntimeException re) {
- // Expected
+ FailOnlyInCommit[] failures = new FailOnlyInCommit[] {
+ // LUCENE-1214
+ new FailOnlyInCommit(false, FailOnlyInCommit.PREPARE_STAGE), // fail during global field map is written
+ new FailOnlyInCommit(true, FailOnlyInCommit.PREPARE_STAGE), // fail after global field map is written
+ new FailOnlyInCommit(false, FailOnlyInCommit.FINISH_STAGE) // fail while running finishCommit
+ };
+
+ for (FailOnlyInCommit failure : failures) {
+ MockDirectoryWrapper dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()));
+ Document doc = new Document();
+ doc.add(newField("field", "a field", Field.Store.YES,
+ Field.Index.ANALYZED));
+ w.addDocument(doc);
+ dir.failOn(failure);
+ try {
+ w.close();
+ fail();
+ } catch (IOException ioe) {
+ fail("expected only RuntimeException");
+ } catch (RuntimeException re) {
+ // Expected
+ }
+ assertTrue(dir.fileExists("1.fnx"));
+ assertTrue(failure.failOnCommit && failure.failOnDeleteFile);
+ w.rollback();
+ assertFalse(dir.fileExists("1.fnx"));
+ assertEquals(0, dir.listAll().length);
+ dir.close();
}
- assertTrue(failure.fail1 && failure.fail2);
- w.rollback();
- dir.close();
}
public void testOptimizeExceptions() throws IOException {
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java Tue Mar 22 11:12:09 2011
@@ -216,7 +216,7 @@ public class TestPerFieldCodecSupport ex
IndexFileNames.FIELD_INFOS_EXTENSION));
FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
assertEquals("faild for segment index: " + i, codec[i],
- codecInfo.codecs[fieldInfo.codecId]);
+ codecInfo.codecs[fieldInfo.getCodecId()]);
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1084134&r1=1084133&r2=1084134&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java Tue Mar 22 11:12:09 2011
@@ -78,9 +78,10 @@ public class TestSegmentMerger extends L
merger.add(reader2);
int docsMerged = merger.merge();
assertTrue(docsMerged == 2);
+ final FieldInfos fieldInfos = merger.fieldInfos();
//Should be able to open a new SegmentReader against the new directory
- SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, merger.fieldInfos().hasProx(),
- merger.getSegmentCodecs(), merger.fieldInfos().hasVectors()),
+ SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, fieldInfos.hasProx(),
+ merger.getSegmentCodecs(), fieldInfos.hasVectors(), fieldInfos),
BufferedIndexInput.BUFFER_SIZE, true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
assertTrue(mergedReader != null);