You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/10/05 16:43:51 UTC
svn commit: r1629501 [7/7] - in /lucene/dev/branches/branch_5x: ./
dev-tools/ lucene/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/
lucene/backward-codecs/src/java/...
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Sun Oct 5 14:43:47 2014
@@ -33,7 +33,6 @@ import org.apache.lucene.codecs.NormsPro
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Accountable;
@@ -92,13 +91,6 @@ public final class SegmentReader extends
// TODO: why is this public?
public SegmentReader(SegmentCommitInfo si, IOContext context) throws IOException {
this.si = si;
- // TODO if the segment uses CFS, we may open the CFS file twice: once for
- // reading the FieldInfos (if they are not gen'd) and second time by
- // SegmentCoreReaders. We can open the CFS here and pass to SCR, but then it
- // results in less readable code (resource not closed where it was opened).
- // Best if we could somehow read FieldInfos in SCR but not keep it there, but
- // constructors don't allow returning two things...
- fieldInfos = readFieldInfos(si);
core = new SegmentCoreReaders(this, si.info.dir, si, context);
segDocValues = new SegmentDocValues();
@@ -113,12 +105,9 @@ public final class SegmentReader extends
liveDocs = null;
}
numDocs = si.info.getDocCount() - si.getDelCount();
-
- if (fieldInfos.hasDocValues()) {
- docValuesProducer = initDocValuesProducer(codec);
- } else {
- docValuesProducer = null;
- }
+
+ fieldInfos = initFieldInfos();
+ docValuesProducer = initDocValuesProducer();
success = true;
} finally {
@@ -153,24 +142,11 @@ public final class SegmentReader extends
this.core = sr.core;
core.incRef();
this.segDocValues = sr.segDocValues;
-
-// System.out.println("[" + Thread.currentThread().getName() + "] SR.init: sharing reader: " + sr + " for gens=" + sr.genDVProducers.keySet());
-
- // increment refCount of DocValuesProducers that are used by this reader
+
boolean success = false;
try {
- final Codec codec = si.info.getCodec();
- if (si.getFieldInfosGen() == -1) {
- fieldInfos = sr.fieldInfos;
- } else {
- fieldInfos = readFieldInfos(si);
- }
-
- if (fieldInfos.hasDocValues()) {
- docValuesProducer = initDocValuesProducer(codec);
- } else {
- docValuesProducer = null;
- }
+ fieldInfos = initFieldInfos();
+ docValuesProducer = initDocValuesProducer();
success = true;
} finally {
if (!success) {
@@ -179,49 +155,34 @@ public final class SegmentReader extends
}
}
- // initialize the per-field DocValuesProducer
- private DocValuesProducer initDocValuesProducer(Codec codec) throws IOException {
+ /**
+ * init most recent DocValues for the current commit
+ */
+ private DocValuesProducer initDocValuesProducer() throws IOException {
final Directory dir = core.cfsReader != null ? core.cfsReader : si.info.dir;
- final DocValuesFormat dvFormat = codec.docValuesFormat();
+ final DocValuesFormat dvFormat = si.info.getCodec().docValuesFormat();
- if (!si.hasFieldUpdates()) {
+ if (!fieldInfos.hasDocValues()) {
+ return null;
+ } else if (si.hasFieldUpdates()) {
+ return new SegmentDocValuesProducer(si, dir, fieldInfos, segDocValues, dvFormat);
+ } else {
// simple case, no DocValues updates
return segDocValues.getDocValuesProducer(-1L, si, IOContext.READ, dir, dvFormat, fieldInfos);
- } else {
- return new SegmentDocValuesProducer(si, dir, fieldInfos, segDocValues, dvFormat);
}
}
/**
- * Reads the most recent {@link FieldInfos} of the given segment info.
- *
- * @lucene.internal
+ * init most recent FieldInfos for the current commit
*/
- static FieldInfos readFieldInfos(SegmentCommitInfo info) throws IOException {
- final Directory dir;
- final boolean closeDir;
- if (info.getFieldInfosGen() == -1 && info.info.getUseCompoundFile()) {
- // no fieldInfos gen and segment uses a compound file
- dir = new CompoundFileDirectory(info.info.dir,
- IndexFileNames.segmentFileName(info.info.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION),
- IOContext.READONCE,
- false);
- closeDir = true;
+ private FieldInfos initFieldInfos() throws IOException {
+ if (!si.hasFieldUpdates()) {
+ return core.coreFieldInfos;
} else {
- // gen'd FIS are read outside CFS, or the segment doesn't use a compound file
- dir = info.info.dir;
- closeDir = false;
- }
-
- try {
- final String segmentSuffix = info.getFieldInfosGen() == -1 ? "" : Long.toString(info.getFieldInfosGen(), Character.MAX_RADIX);
- Codec codec = info.info.getCodec();
- FieldInfosFormat fisFormat = codec.fieldInfosFormat();
- return fisFormat.getFieldInfosReader().read(dir, info.info, segmentSuffix, IOContext.READONCE);
- } finally {
- if (closeDir) {
- dir.close();
- }
+ // updates always outside of CFS
+ FieldInfosFormat fisFormat = si.info.getCodec().fieldInfosFormat();
+ final String segmentSuffix = Long.toString(si.getFieldInfosGen(), Character.MAX_RADIX);
+ return fisFormat.getFieldInfosReader().read(si.info.dir, si.info, segmentSuffix, IOContext.READONCE);
}
}
@@ -566,7 +527,7 @@ public final class SegmentReader extends
@Override
public void checkIntegrity() throws IOException {
ensureOpen();
-
+
// stored fields
getFieldsReader().checkIntegrity();
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/store/Directory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/store/Directory.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/store/Directory.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/store/Directory.java Sun Oct 5 14:43:47 2014
@@ -100,12 +100,7 @@ public abstract class Directory implemen
*/
public abstract void renameFile(String source, String dest) throws IOException;
- /** Returns a stream reading an existing file, with the
- * specified read buffer size. The particular Directory
- * implementation may ignore the buffer size. Currently
- * the only Directory implementations that respect this
- * parameter are {@link FSDirectory} and {@link
- * CompoundFileDirectory}.
+ /** Returns a stream reading an existing file.
* <p>Throws {@link FileNotFoundException} or {@link NoSuchFileException}
* if the file does not exist.
*/
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat.java Sun Oct 5 14:43:47 2014
@@ -17,8 +17,17 @@ package org.apache.lucene.codecs.lucene4
* limitations under the License.
*/
+import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.blocktree.FieldReader;
+import org.apache.lucene.codecs.blocktree.Stats;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
import org.apache.lucene.index.BasePostingsFormatTestCase;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.util.TestUtil;
/**
@@ -31,4 +40,28 @@ public class TestBlockPostingsFormat ext
protected Codec getCodec() {
return codec;
}
+
+ /** Make sure the final sub-block(s) are not skipped. */
+ public void testFinalBlock() throws Exception {
+ Directory d = newDirectory();
+ IndexWriter w = new IndexWriter(d, new IndexWriterConfig(new MockAnalyzer(random())));
+ for(int i=0;i<25;i++) {
+ Document doc = new Document();
+ doc.add(newStringField("field", Character.toString((char) (97+i)), Field.Store.NO));
+ doc.add(newStringField("field", "z" + Character.toString((char) (97+i)), Field.Store.NO));
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w, true);
+ assertEquals(1, r.leaves().size());
+ FieldReader field = (FieldReader) r.leaves().get(0).reader().fields().terms("field");
+ // We should see exactly two blocks: one root block (prefix empty string) and one block for z* terms (prefix z):
+ Stats stats = field.computeStats();
+ assertEquals(0, stats.floorBlockCount);
+ assertEquals(2, stats.nonFloorBlockCount);
+ r.close();
+ w.close();
+ d.close();
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java Sun Oct 5 14:43:47 2014
@@ -1116,7 +1116,10 @@ public class TestAddIndexes extends Luce
w3.close();
// we should now see segments_X,
// _Y.cfs,_Y.cfe, _Z.si
- assertEquals("Only one compound segment should exist, but got: " + Arrays.toString(dir.listAll()), 4, dir.listAll().length);
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(dir);
+ assertEquals("Only one compound segment should exist", 1, sis.size());
+ assertTrue(sis.info(0).info.getUseCompoundFile());
dir.close();
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java Sun Oct 5 14:43:47 2014
@@ -24,11 +24,9 @@ import org.apache.lucene.codecs.CodecUti
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.MockDirectoryWrapper;
-import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -65,33 +63,32 @@ public class TestAllFilesHaveChecksumFoo
}
}
riw.close();
- checkHeaders(dir);
+ checkFooters(dir);
dir.close();
}
- private void checkHeaders(Directory dir) throws IOException {
- for (String file : dir.listAll()) {
- if (file.equals(IndexWriter.WRITE_LOCK_NAME)) {
- continue; // write.lock has no footer, thats ok
+ private void checkFooters(Directory dir) throws IOException {
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(dir);
+ checkFooter(dir, sis.getSegmentsFileName());
+
+ for (SegmentCommitInfo si : sis) {
+ for (String file : si.files()) {
+ checkFooter(dir, file);
}
- if (file.endsWith(IndexFileNames.COMPOUND_FILE_EXTENSION)) {
- CompoundFileDirectory cfsDir = new CompoundFileDirectory(dir, file, newIOContext(random()), false);
- checkHeaders(cfsDir); // recurse into cfs
- cfsDir.close();
- }
- IndexInput in = null;
- boolean success = false;
- try {
- in = dir.openInput(file, newIOContext(random()));
- CodecUtil.checksumEntireFile(in);
- success = true;
- } finally {
- if (success) {
- IOUtils.close(in);
- } else {
- IOUtils.closeWhileHandlingException(in);
+ if (si.info.getUseCompoundFile()) {
+ try (Directory cfsDir = si.info.getCodec().compoundFormat().getCompoundReader(dir, si.info, newIOContext(random()))) {
+ for (String cfsFile : cfsDir.listAll()) {
+ checkFooter(cfsDir, cfsFile);
+ }
}
}
}
}
+
+ private void checkFooter(Directory dir, String file) throws IOException {
+ try (IndexInput in = dir.openInput(file, newIOContext(random()))) {
+ CodecUtil.checksumEntireFile(in);
+ }
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java Sun Oct 5 14:43:47 2014
@@ -28,11 +28,9 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.MockDirectoryWrapper;
-import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -83,41 +81,39 @@ public class TestAllFilesHaveCodecHeader
}
private void checkHeaders(Directory dir, Map<String,String> namesToExtensions) throws IOException {
- for (String file : dir.listAll()) {
- if (file.equals(IndexWriter.WRITE_LOCK_NAME)) {
- continue; // write.lock has no header, thats ok
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(dir);
+ checkHeader(dir, sis.getSegmentsFileName(), namesToExtensions);
+
+ for (SegmentCommitInfo si : sis) {
+ for (String file : si.files()) {
+ checkHeader(dir, file, namesToExtensions);
}
- if (file.endsWith(IndexFileNames.COMPOUND_FILE_EXTENSION)) {
- CompoundFileDirectory cfsDir = new CompoundFileDirectory(dir, file, newIOContext(random()), false);
- checkHeaders(cfsDir, namesToExtensions); // recurse into cfs
- cfsDir.close();
- }
- IndexInput in = null;
- boolean success = false;
- try {
- in = dir.openInput(file, newIOContext(random()));
- int val = in.readInt();
- assertEquals(file + " has no codec header, instead found: " + val, CodecUtil.CODEC_MAGIC, val);
- String codecName = in.readString();
- assertFalse(codecName.isEmpty());
- String extension = IndexFileNames.getExtension(file);
- if (extension == null) {
- assertTrue(file.startsWith(IndexFileNames.SEGMENTS));
- extension = "<segments> (not a real extension, designates segments file)";
- }
- String previous = namesToExtensions.put(codecName, extension);
- if (previous != null && !previous.equals(extension)) {
- //TODO: not yet
- // fail("extensions " + previous + " and " + extension + " share same codecName " + codecName);
- }
- success = true;
- } finally {
- if (success) {
- IOUtils.close(in);
- } else {
- IOUtils.closeWhileHandlingException(in);
+ if (si.info.getUseCompoundFile()) {
+ try (Directory cfsDir = si.info.getCodec().compoundFormat().getCompoundReader(dir, si.info, newIOContext(random()))) {
+ for (String cfsFile : cfsDir.listAll()) {
+ checkHeader(cfsDir, cfsFile, namesToExtensions);
+ }
}
}
}
}
+
+ private void checkHeader(Directory dir, String file, Map<String,String> namesToExtensions) throws IOException {
+ try (IndexInput in = dir.openInput(file, newIOContext(random()))) {
+ int val = in.readInt();
+ assertEquals(file + " has no codec header, instead found: " + val, CodecUtil.CODEC_MAGIC, val);
+ String codecName = in.readString();
+ assertFalse(codecName.isEmpty());
+ String extension = IndexFileNames.getExtension(file);
+ if (extension == null) {
+ assertTrue(file.startsWith(IndexFileNames.SEGMENTS));
+ extension = "<segments> (not a real extension, designates segments file)";
+ }
+ String previous = namesToExtensions.put(codecName, extension);
+ if (previous != null && !previous.equals(extension)) {
+ fail("extensions " + previous + " and " + extension + " share same codecName " + codecName);
+ }
+ }
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecUtil.java Sun Oct 5 14:43:47 2014
@@ -26,6 +26,7 @@ import org.apache.lucene.store.RAMFile;
import org.apache.lucene.store.RAMInputStream;
import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.StringHelper;
/** tests for codecutil methods */
public class TestCodecUtil extends LuceneTestCase {
@@ -194,4 +195,61 @@ public class TestCodecUtil extends Lucen
}
input.close();
}
+
+ public void testSegmentHeaderLength() throws Exception {
+ RAMFile file = new RAMFile();
+ IndexOutput output = new RAMOutputStream(file, true);
+ CodecUtil.writeSegmentHeader(output, "FooBar", 5, StringHelper.randomId(), "xyz");
+ output.writeString("this is the data");
+ output.close();
+
+ IndexInput input = new RAMInputStream("file", file);
+ input.seek(CodecUtil.segmentHeaderLength("FooBar", "xyz"));
+ assertEquals("this is the data", input.readString());
+ input.close();
+ }
+
+ public void testWriteTooLongSuffix() throws Exception {
+ StringBuilder tooLong = new StringBuilder();
+ for (int i = 0; i < 256; i++) {
+ tooLong.append('a');
+ }
+ RAMFile file = new RAMFile();
+ IndexOutput output = new RAMOutputStream(file, true);
+ try {
+ CodecUtil.writeSegmentHeader(output, "foobar", 5, StringHelper.randomId(), tooLong.toString());
+ fail("didn't get expected exception");
+ } catch (IllegalArgumentException expected) {
+ // expected
+ }
+ }
+
+ public void testWriteVeryLongSuffix() throws Exception {
+ StringBuilder justLongEnough = new StringBuilder();
+ for (int i = 0; i < 255; i++) {
+ justLongEnough.append('a');
+ }
+ RAMFile file = new RAMFile();
+ IndexOutput output = new RAMOutputStream(file, true);
+ byte[] id = StringHelper.randomId();
+ CodecUtil.writeSegmentHeader(output, "foobar", 5, id, justLongEnough.toString());
+ output.close();
+
+ IndexInput input = new RAMInputStream("file", file);
+ CodecUtil.checkSegmentHeader(input, "foobar", 5, 5, id, justLongEnough.toString());
+ assertEquals(input.getFilePointer(), input.length());
+ assertEquals(input.getFilePointer(), CodecUtil.segmentHeaderLength("foobar", justLongEnough.toString()));
+ input.close();
+ }
+
+ public void testWriteNonAsciiSuffix() throws Exception {
+ RAMFile file = new RAMFile();
+ IndexOutput output = new RAMOutputStream(file, true);
+ try {
+ CodecUtil.writeSegmentHeader(output, "foobar", 5, StringHelper.randomId(), "\u1234");
+ fail("didn't get expected exception");
+ } catch (IllegalArgumentException expected) {
+ // expected
+ }
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java Sun Oct 5 14:43:47 2014
@@ -247,10 +247,10 @@ public class TestCodecs extends LuceneTe
final FieldData[] fields = new FieldData[] {field};
final FieldInfos fieldInfos = builder.finish();
final Directory dir = newDirectory();
- this.write(fieldInfos, dir, fields);
Codec codec = Codec.getDefault();
final SegmentInfo si = new SegmentInfo(dir, Version.LATEST, SEGMENT, 10000, false, codec, null, StringHelper.randomId());
-
+
+ this.write(si, fieldInfos, dir, fields);
final FieldsProducer reader = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random())));
final Iterator<String> fieldsEnum = reader.iterator();
@@ -304,9 +304,9 @@ public class TestCodecs extends LuceneTe
System.out.println("TEST: now write postings");
}
- this.write(fieldInfos, dir, fields);
Codec codec = Codec.getDefault();
final SegmentInfo si = new SegmentInfo(dir, Version.LATEST, SEGMENT, 10000, false, codec, null, StringHelper.randomId());
+ this.write(si, fieldInfos, dir, fields);
if (VERBOSE) {
System.out.println("TEST: now read postings");
@@ -798,10 +798,9 @@ public class TestCodecs extends LuceneTe
}
}
- private void write(final FieldInfos fieldInfos, final Directory dir, final FieldData[] fields) throws Throwable {
+ private void write(SegmentInfo si, final FieldInfos fieldInfos, final Directory dir, final FieldData[] fields) throws Throwable {
- final Codec codec = Codec.getDefault();
- final SegmentInfo si = new SegmentInfo(dir, Version.LATEST, SEGMENT, 10000, false, codec, null, StringHelper.randomId());
+ final Codec codec = si.getCodec();
final SegmentWriteState state = new SegmentWriteState(InfoStream.getDefault(), dir, si, fieldInfos, null, newIOContext(random()));
Arrays.sort(fields);
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java Sun Oct 5 14:43:47 2014
@@ -68,8 +68,8 @@ public class TestConsistentFieldNumbers
sis.read(dir);
assertEquals(2, sis.size());
- FieldInfos fis1 = SegmentReader.readFieldInfos(sis.info(0));
- FieldInfos fis2 = SegmentReader.readFieldInfos(sis.info(1));
+ FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
+ FieldInfos fis2 = IndexWriter.readFieldInfos(sis.info(1));
assertEquals("f1", fis1.fieldInfo(0).name);
assertEquals("f2", fis1.fieldInfo(1).name);
@@ -86,7 +86,7 @@ public class TestConsistentFieldNumbers
sis.read(dir);
assertEquals(1, sis.size());
- FieldInfos fis3 = SegmentReader.readFieldInfos(sis.info(0));
+ FieldInfos fis3 = IndexWriter.readFieldInfos(sis.info(0));
assertEquals("f1", fis3.fieldInfo(0).name);
assertEquals("f2", fis3.fieldInfo(1).name);
@@ -134,8 +134,8 @@ public class TestConsistentFieldNumbers
sis.read(dir1);
assertEquals(2, sis.size());
- FieldInfos fis1 = SegmentReader.readFieldInfos(sis.info(0));
- FieldInfos fis2 = SegmentReader.readFieldInfos(sis.info(1));
+ FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
+ FieldInfos fis2 = IndexWriter.readFieldInfos(sis.info(1));
assertEquals("f1", fis1.fieldInfo(0).name);
assertEquals("f2", fis1.fieldInfo(1).name);
@@ -164,7 +164,7 @@ public class TestConsistentFieldNumbers
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
assertEquals(1, sis.size());
- FieldInfos fis1 = SegmentReader.readFieldInfos(sis.info(0));
+ FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
assertEquals("f1", fis1.fieldInfo(0).name);
assertEquals("f2", fis1.fieldInfo(1).name);
}
@@ -181,8 +181,8 @@ public class TestConsistentFieldNumbers
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
assertEquals(2, sis.size());
- FieldInfos fis1 = SegmentReader.readFieldInfos(sis.info(0));
- FieldInfos fis2 = SegmentReader.readFieldInfos(sis.info(1));
+ FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
+ FieldInfos fis2 = IndexWriter.readFieldInfos(sis.info(1));
assertEquals("f1", fis1.fieldInfo(0).name);
assertEquals("f2", fis1.fieldInfo(1).name);
assertEquals("f1", fis2.fieldInfo(0).name);
@@ -202,9 +202,9 @@ public class TestConsistentFieldNumbers
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
assertEquals(3, sis.size());
- FieldInfos fis1 = SegmentReader.readFieldInfos(sis.info(0));
- FieldInfos fis2 = SegmentReader.readFieldInfos(sis.info(1));
- FieldInfos fis3 = SegmentReader.readFieldInfos(sis.info(2));
+ FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
+ FieldInfos fis2 = IndexWriter.readFieldInfos(sis.info(1));
+ FieldInfos fis3 = IndexWriter.readFieldInfos(sis.info(2));
assertEquals("f1", fis1.fieldInfo(0).name);
assertEquals("f2", fis1.fieldInfo(1).name);
assertEquals("f1", fis2.fieldInfo(0).name);
@@ -234,7 +234,7 @@ public class TestConsistentFieldNumbers
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
assertEquals(1, sis.size());
- FieldInfos fis1 = SegmentReader.readFieldInfos(sis.info(0));
+ FieldInfos fis1 = IndexWriter.readFieldInfos(sis.info(0));
assertEquals("f1", fis1.fieldInfo(0).name);
assertEquals("f2", fis1.fieldInfo(1).name);
assertEquals("f3", fis1.fieldInfo(2).name);
@@ -272,7 +272,7 @@ public class TestConsistentFieldNumbers
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
for (SegmentCommitInfo si : sis) {
- FieldInfos fis = SegmentReader.readFieldInfos(si);
+ FieldInfos fis = IndexWriter.readFieldInfos(si);
for (FieldInfo fi : fis) {
Field expected = getField(Integer.parseInt(fi.name));
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java Sun Oct 5 14:43:47 2014
@@ -223,7 +223,7 @@ public class TestDoc extends LuceneTestC
SegmentMerger merger = new SegmentMerger(Arrays.<LeafReader>asList(r1, r2),
si, InfoStream.getDefault(), trackingDir,
- MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), context, true);
+ MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), context);
MergeState mergeState = merger.merge();
r1.close();
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java Sun Oct 5 14:43:47 2014
@@ -22,7 +22,6 @@ import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.codecs.Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@@ -84,6 +83,13 @@ public class TestIndexFileDeleter extend
Term searchTerm = new Term("id", "7");
writer.deleteDocuments(searchTerm);
writer.close();
+
+ // read in index to try to not depend on codec-specific filenames so much
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(dir);
+ SegmentInfo si0 = sis.info(0).info;
+ SegmentInfo si1 = sis.info(1).info;
+ SegmentInfo si3 = sis.info(3).info;
// Now, artificially create an extra .del file & extra
// .s0 file:
@@ -110,11 +116,13 @@ public class TestIndexFileDeleter extend
// non-existent segment:
copyFile(dir, "_0_1" + ext, "_188_1" + ext);
+ String cfsFiles0[] = si0.getCodec().compoundFormat().files(si0);
+
// Create a bogus segment file:
- copyFile(dir, "_0.cfs", "_188.cfs");
+ copyFile(dir, cfsFiles0[0], "_188.cfs");
// Create a bogus fnm file when the CFS already exists:
- copyFile(dir, "_0.cfs", "_0.fnm");
+ copyFile(dir, cfsFiles0[0], "_0.fnm");
// Create some old segments file:
copyFile(dir, "segments_2", "segments");
@@ -124,8 +132,14 @@ public class TestIndexFileDeleter extend
// TODO: assert is bogus (relies upon codec-specific filenames)
assertTrue(slowFileExists(dir, "_3.fdt") || slowFileExists(dir, "_3.fld"));
- assertTrue(!slowFileExists(dir, "_3.cfs"));
- copyFile(dir, "_1.cfs", "_3.cfs");
+
+ String cfsFiles3[] = si3.getCodec().compoundFormat().files(si3);
+ for (String f : cfsFiles3) {
+ assertTrue(!slowFileExists(dir, f));
+ }
+
+ String cfsFiles1[] = si1.getCodec().compoundFormat().files(si1);
+ copyFile(dir, cfsFiles1[0], "_3.cfs");
String[] filesPre = dir.listAll();
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Sun Oct 5 14:43:47 2014
@@ -1326,6 +1326,7 @@ public class TestIndexWriter extends Luc
public void testDeleteUnusedFiles() throws Exception {
+ assumeFalse("test relies on exact filenames", Codec.getDefault() instanceof SimpleTextCodec);
for(int iter=0;iter<2;iter++) {
MockDirectoryWrapper dir = newMockDirectory(); // relies on windows semantics
dir.setEnableVirusScanner(false); // but ensures files are actually deleted
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Sun Oct 5 14:43:47 2014
@@ -1232,17 +1232,19 @@ public class TestIndexWriterExceptions e
long gen = SegmentInfos.getLastCommitGeneration(dir);
assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
-
- String[] files = dir.listAll();
+
boolean corrupted = false;
- for(int i=0;i<files.length;i++) {
- if (files[i].endsWith(".cfs")) {
- dir.deleteFile(files[i]);
- corrupted = true;
- break;
- }
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(dir);
+ for (SegmentCommitInfo si : sis) {
+ assertTrue(si.info.getUseCompoundFile());
+ String cfsFiles[] = si.info.getCodec().compoundFormat().files(si.info);
+ dir.deleteFile(cfsFiles[0]);
+ corrupted = true;
+ break;
}
- assertTrue("failed to find cfs file to remove", corrupted);
+
+ assertTrue("failed to find cfs file to remove: ", corrupted);
IndexReader reader = null;
try {
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java Sun Oct 5 14:43:47 2014
@@ -83,7 +83,7 @@ public class TestSegmentMerger extends L
SegmentMerger merger = new SegmentMerger(Arrays.<LeafReader>asList(reader1, reader2),
si, InfoStream.getDefault(), mergedDir,
- MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), newIOContext(random()), true);
+ MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), newIOContext(random()));
MergeState mergeState = merger.merge();
int docsMerged = mergeState.segmentInfo.getDocCount();
assertTrue(docsMerged == 2);
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java Sun Oct 5 14:43:47 2014
@@ -129,7 +129,7 @@ public class TestTermVectorsReader exten
seg = writer.newestSegment();
writer.close();
- fieldInfos = SegmentReader.readFieldInfos(seg);
+ fieldInfos = IndexWriter.readFieldInfos(seg);
}
@Override
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java Sun Oct 5 14:43:47 2014
@@ -122,21 +122,4 @@ public class TestNRTCachingDirectory ext
writer.close();
cachedFSDir.close();
}
-
- // LUCENE-5724
- public void testLargeCFS() throws IOException {
- Directory dir = new NRTCachingDirectory(newFSDirectory(createTempDir()), 2.0, 25.0);
- IOContext context = new IOContext(new FlushInfo(0, 512*1024*1024));
- IndexOutput out = dir.createOutput("big.bin", context);
- byte[] bytes = new byte[512];
- for(int i=0;i<1024*1024;i++) {
- out.writeBytes(bytes, 0, bytes.length);
- }
- out.close();
-
- Directory cfsDir = new CompoundFileDirectory(dir, "big.cfs", context, true);
- dir.copy(cfsDir, "big.bin", "big.bin", context);
- cfsDir.close();
- dir.close();
- }
}
Modified: lucene/dev/branches/branch_5x/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java Sun Oct 5 14:43:47 2014
@@ -78,7 +78,9 @@ public class TestIndexSplitter extends L
Path destDir2 = createTempDir(LuceneTestCase.getTestClass().getSimpleName());
IndexSplitter.main(new String[] {dir.toAbsolutePath().toString(), destDir2.toAbsolutePath().toString(), splitSegName});
Directory fsDirDest2 = newFSDirectory(destDir2);
- assertEquals(4, fsDirDest2.listAll().length);
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(fsDirDest2);
+ assertEquals(1, sis.size());
r = DirectoryReader.open(fsDirDest2);
assertEquals(50, r.maxDoc());
r.close();
Modified: lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsFormat.java Sun Oct 5 14:43:47 2014
@@ -99,15 +99,10 @@ public class IDVersionPostingsFormat ext
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new IDVersionPostingsReader();
+ PostingsReaderBase postingsReader = new IDVersionPostingsReader(state);
boolean success = false;
try {
- FieldsProducer ret = new VersionBlockTreeTermsReader(state.directory,
- state.fieldInfos,
- state.segmentInfo,
- postingsReader,
- state.context,
- state.segmentSuffix);
+ FieldsProducer ret = new VersionBlockTreeTermsReader(postingsReader, state);
success = true;
return ret;
} finally {
Modified: lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java Sun Oct 5 14:43:47 2014
@@ -26,21 +26,27 @@ import org.apache.lucene.codecs.Postings
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
-import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.Bits;
final class IDVersionPostingsReader extends PostingsReaderBase {
+ final SegmentReadState state;
+
+ public IDVersionPostingsReader(SegmentReadState state) {
+ this.state = state;
+ }
@Override
public void init(IndexInput termsIn) throws IOException {
// Make sure we are talking to the matching postings writer
- CodecUtil.checkHeader(termsIn,
- IDVersionPostingsWriter.TERMS_CODEC,
- IDVersionPostingsWriter.VERSION_START,
- IDVersionPostingsWriter.VERSION_CURRENT);
+ CodecUtil.checkSegmentHeader(termsIn,
+ IDVersionPostingsWriter.TERMS_CODEC,
+ IDVersionPostingsWriter.VERSION_START,
+ IDVersionPostingsWriter.VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
}
@Override
Modified: lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java Sun Oct 5 14:43:47 2014
@@ -26,7 +26,6 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
final class IDVersionPostingsWriter extends PushPostingsWriterBase {
@@ -34,7 +33,7 @@ final class IDVersionPostingsWriter exte
final static String TERMS_CODEC = "IDVersionPostingsWriterTerms";
// Increment version to change it
- final static int VERSION_START = 0;
+ final static int VERSION_START = 1;
final static int VERSION_CURRENT = VERSION_START;
final static IDVersionTermState emptyState = new IDVersionTermState();
@@ -57,7 +56,7 @@ final class IDVersionPostingsWriter exte
@Override
public void init(IndexOutput termsOut) throws IOException {
- CodecUtil.writeHeader(termsOut, TERMS_CODEC, VERSION_CURRENT);
+ CodecUtil.writeSegmentHeader(termsOut, TERMS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
}
@Override
Modified: lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java (original)
+++ lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java Sun Oct 5 14:43:47 2014
@@ -283,7 +283,7 @@ public final class IDVersionSegmentTerms
targetUpto = 0;
IDVersionSegmentTermsEnumFrame lastFrame = stack[0];
- assert validIndexPrefix <= term.length(): "validIndexPrefix=" + validIndexPrefix + " term.length=" + term.length() + " seg=" + fr.parent.segment;
+ assert validIndexPrefix <= term.length(): "validIndexPrefix=" + validIndexPrefix + " term.length=" + term.length() + " seg=" + fr.parent;
final int targetLimit = Math.min(target.length, validIndexPrefix);
@@ -1063,6 +1063,6 @@ public final class IDVersionSegmentTerms
@Override
public String toString() {
- return "IDVersionSegmentTermsEnum(seg=" + fr.parent.segment + ")";
+ return "IDVersionSegmentTermsEnum(seg=" + fr.parent + ")";
}
}
Modified: lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsReader.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsReader.java Sun Oct 5 14:43:47 2014
@@ -29,12 +29,9 @@ import org.apache.lucene.codecs.FieldsPr
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.Terms;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
@@ -61,38 +58,36 @@ public final class VersionBlockTreeTerms
private final TreeMap<String,VersionFieldReader> fields = new TreeMap<>();
- /** File offset where the directory starts in the terms file. */
- private long dirOffset;
-
- /** File offset where the directory starts in the index file. */
- private long indexDirOffset;
-
- final String segment;
-
- private final int version;
-
/** Sole constructor. */
- public VersionBlockTreeTermsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo info,
- PostingsReaderBase postingsReader, IOContext ioContext,
- String segmentSuffix)
- throws IOException {
+ public VersionBlockTreeTermsReader(PostingsReaderBase postingsReader, SegmentReadState state) throws IOException {
this.postingsReader = postingsReader;
- this.segment = info.name;
- in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, VersionBlockTreeTermsWriter.TERMS_EXTENSION),
- ioContext);
+ String termsFile = IndexFileNames.segmentFileName(state.segmentInfo.name,
+ state.segmentSuffix,
+ VersionBlockTreeTermsWriter.TERMS_EXTENSION);
+ in = state.directory.openInput(termsFile, state.context);
boolean success = false;
IndexInput indexIn = null;
try {
- version = readHeader(in);
- indexIn = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, VersionBlockTreeTermsWriter.TERMS_INDEX_EXTENSION),
- ioContext);
- int indexVersion = readIndexHeader(indexIn);
- if (indexVersion != version) {
- throw new CorruptIndexException("mixmatched version files: " + in + "=" + version + "," + indexIn + "=" + indexVersion, indexIn);
+ int termsVersion = CodecUtil.checkSegmentHeader(in, VersionBlockTreeTermsWriter.TERMS_CODEC_NAME,
+ VersionBlockTreeTermsWriter.VERSION_START,
+ VersionBlockTreeTermsWriter.VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
+
+ String indexFile = IndexFileNames.segmentFileName(state.segmentInfo.name,
+ state.segmentSuffix,
+ VersionBlockTreeTermsWriter.TERMS_INDEX_EXTENSION);
+ indexIn = state.directory.openInput(indexFile, state.context);
+ int indexVersion = CodecUtil.checkSegmentHeader(indexIn, VersionBlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME,
+ VersionBlockTreeTermsWriter.VERSION_START,
+ VersionBlockTreeTermsWriter.VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
+
+ if (indexVersion != termsVersion) {
+ throw new CorruptIndexException("mixmatched version files: " + in + "=" + termsVersion + "," + indexIn + "=" + indexVersion, indexIn);
}
// verify
@@ -108,8 +103,8 @@ public final class VersionBlockTreeTerms
CodecUtil.retrieveChecksum(in);
// Read per-field details
- seekDir(in, dirOffset);
- seekDir(indexIn, indexDirOffset);
+ seekDir(in);
+ seekDir(indexIn);
final int numFields = in.readVInt();
if (numFields < 0) {
@@ -126,7 +121,7 @@ public final class VersionBlockTreeTerms
code.length = numBytes;
final long version = in.readVLong();
final Pair<BytesRef,Long> rootCode = VersionBlockTreeTermsWriter.FST_OUTPUTS.newPair(code, version);
- final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+ final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
assert fieldInfo != null: "field=" + field;
final long sumTotalTermFreq = numTerms;
final long sumDocFreq = numTerms;
@@ -136,8 +131,8 @@ public final class VersionBlockTreeTerms
BytesRef minTerm = readBytesRef(in);
BytesRef maxTerm = readBytesRef(in);
- if (docCount < 0 || docCount > info.getDocCount()) { // #docs with field must be <= #docs
- throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + info.getDocCount(), in);
+ if (docCount < 0 || docCount > state.segmentInfo.getDocCount()) { // #docs with field must be <= #docs
+ throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + state.segmentInfo.getDocCount(), in);
}
if (sumDocFreq < docCount) { // #postings must be >= #docs with field
throw new CorruptIndexException("invalid sumDocFreq: " + sumDocFreq + " docCount: " + docCount, in);
@@ -172,27 +167,10 @@ public final class VersionBlockTreeTerms
return bytes;
}
- /** Reads terms file header. */
- private int readHeader(IndexInput input) throws IOException {
- int version = CodecUtil.checkHeader(input, VersionBlockTreeTermsWriter.TERMS_CODEC_NAME,
- VersionBlockTreeTermsWriter.VERSION_START,
- VersionBlockTreeTermsWriter.VERSION_CURRENT);
- return version;
- }
-
- /** Reads index file header. */
- private int readIndexHeader(IndexInput input) throws IOException {
- int version = CodecUtil.checkHeader(input, VersionBlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME,
- VersionBlockTreeTermsWriter.VERSION_START,
- VersionBlockTreeTermsWriter.VERSION_CURRENT);
- return version;
- }
-
/** Seek {@code input} to the directory offset. */
- private void seekDir(IndexInput input, long dirOffset)
- throws IOException {
+ private void seekDir(IndexInput input) throws IOException {
input.seek(input.length() - CodecUtil.footerLength() - 8);
- dirOffset = input.readLong();
+ long dirOffset = input.readLong();
input.seek(dirOffset);
}
Modified: lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java Sun Oct 5 14:43:47 2014
@@ -121,7 +121,7 @@ public final class VersionBlockTreeTerms
final static String TERMS_CODEC_NAME = "VERSION_BLOCK_TREE_TERMS_DICT";
/** Initial terms format. */
- public static final int VERSION_START = 0;
+ public static final int VERSION_START = 1;
/** Current terms format. */
public static final int VERSION_CURRENT = VERSION_START;
@@ -199,13 +199,13 @@ public final class VersionBlockTreeTerms
fieldInfos = state.fieldInfos;
this.minItemsInBlock = minItemsInBlock;
this.maxItemsInBlock = maxItemsInBlock;
- CodecUtil.writeHeader(out, TERMS_CODEC_NAME, VERSION_CURRENT);
+ CodecUtil.writeSegmentHeader(out, TERMS_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
//DEBUG = state.segmentName.equals("_4a");
final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION);
indexOut = state.directory.createOutput(termsIndexFileName, state.context);
- CodecUtil.writeHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT);
+ CodecUtil.writeSegmentHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
this.postingsWriter = postingsWriter;
// segment = state.segmentInfo.name;
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java Sun Oct 5 14:43:47 2014
@@ -301,6 +301,11 @@ public class AssertingDocValuesFormat ex
}
@Override
+ public DocValuesProducer getMergeInstance() throws IOException {
+ return new AssertingDocValuesProducer(in.getMergeInstance(), maxDoc);
+ }
+
+ @Override
public String toString() {
return getClass().getSimpleName() + "(" + in.toString() + ")";
}
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java Sun Oct 5 14:43:47 2014
@@ -124,6 +124,11 @@ public class AssertingNormsFormat extend
}
@Override
+ public NormsProducer getMergeInstance() throws IOException {
+ return new AssertingNormsProducer(in.getMergeInstance(), maxDoc);
+ }
+
+ @Override
public String toString() {
return getClass().getSimpleName() + "(" + in.toString() + ")";
}
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java Sun Oct 5 14:43:47 2014
@@ -111,6 +111,11 @@ public final class AssertingPostingsForm
}
@Override
+ public FieldsProducer getMergeInstance() throws IOException {
+ return new AssertingFieldsProducer(in.getMergeInstance());
+ }
+
+ @Override
public String toString() {
return getClass().getSimpleName() + "(" + in.toString() + ")";
}
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java Sun Oct 5 14:43:47 2014
@@ -97,6 +97,11 @@ public class AssertingStoredFieldsFormat
}
@Override
+ public StoredFieldsReader getMergeInstance() throws IOException {
+ return new AssertingStoredFieldsReader(in.getMergeInstance(), maxDoc);
+ }
+
+ @Override
public String toString() {
return getClass().getSimpleName() + "(" + in.toString() + ")";
}
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java Sun Oct 5 14:43:47 2014
@@ -96,6 +96,11 @@ public class AssertingTermVectorsFormat
}
@Override
+ public TermVectorsReader getMergeInstance() throws IOException {
+ return new AssertingTermVectorsReader(in.getMergeInstance());
+ }
+
+ @Override
public String toString() {
return getClass().getSimpleName() + "(" + in.toString() + ")";
}
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyCodec.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyCodec.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyCodec.java Sun Oct 5 14:43:47 2014
@@ -20,6 +20,7 @@ package org.apache.lucene.codecs.cranky;
import java.util.Random;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.CompoundFormat;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.FilterCodec;
@@ -86,6 +87,11 @@ public class CrankyCodec extends FilterC
}
@Override
+ public CompoundFormat compoundFormat() {
+ return new CrankyCompoundFormat(delegate.compoundFormat(), random);
+ }
+
+ @Override
public String toString() {
return "Cranky(" + delegate + ")";
}
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java Sun Oct 5 14:43:47 2014
@@ -35,7 +35,6 @@ import org.apache.lucene.codecs.lucene41
import org.apache.lucene.codecs.lucene41.Lucene41PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.BytesRef;
// TODO: we could make separate base class that can wrapp
// any PostingsBaseFormat and make it ord-able...
@@ -100,11 +99,7 @@ public final class Lucene41WithOrds exte
boolean success = false;
try {
- indexReader = new FixedGapTermsIndexReader(state.directory,
- state.fieldInfos,
- state.segmentInfo.name,
- BytesRef.getUTF8SortedAsUnicodeComparator(),
- state.segmentSuffix, state.context);
+ indexReader = new FixedGapTermsIndexReader(state);
success = true;
} finally {
if (!success) {
@@ -114,13 +109,7 @@ public final class Lucene41WithOrds exte
success = false;
try {
- FieldsProducer ret = new BlockTermsReader(indexReader,
- state.directory,
- state.fieldInfos,
- state.segmentInfo,
- postings,
- state.context,
- state.segmentSuffix);
+ FieldsProducer ret = new BlockTermsReader(indexReader, postings, state);
success = true;
return ret;
} finally {
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapDocFreqInterval.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapDocFreqInterval.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapDocFreqInterval.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapDocFreqInterval.java Sun Oct 5 14:43:47 2014
@@ -103,10 +103,7 @@ public final class Lucene41VarGapDocFreq
boolean success = false;
try {
- indexReader = new VariableGapTermsIndexReader(state.directory,
- state.fieldInfos,
- state.segmentInfo.name,
- state.segmentSuffix, state.context);
+ indexReader = new VariableGapTermsIndexReader(state);
success = true;
} finally {
if (!success) {
@@ -116,13 +113,7 @@ public final class Lucene41VarGapDocFreq
success = false;
try {
- FieldsProducer ret = new BlockTermsReader(indexReader,
- state.directory,
- state.fieldInfos,
- state.segmentInfo,
- postings,
- state.context,
- state.segmentSuffix);
+ FieldsProducer ret = new BlockTermsReader(indexReader, postings, state);
success = true;
return ret;
} finally {
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapFixedInterval.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapFixedInterval.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapFixedInterval.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapFixedInterval.java Sun Oct 5 14:43:47 2014
@@ -100,10 +100,7 @@ public final class Lucene41VarGapFixedIn
boolean success = false;
try {
- indexReader = new VariableGapTermsIndexReader(state.directory,
- state.fieldInfos,
- state.segmentInfo.name,
- state.segmentSuffix, state.context);
+ indexReader = new VariableGapTermsIndexReader(state);
success = true;
} finally {
if (!success) {
@@ -113,13 +110,7 @@ public final class Lucene41VarGapFixedIn
success = false;
try {
- FieldsProducer ret = new BlockTermsReader(indexReader,
- state.directory,
- state.fieldInfos,
- state.segmentInfo,
- postings,
- state.context,
- state.segmentSuffix);
+ FieldsProducer ret = new BlockTermsReader(indexReader, postings, state);
success = true;
return ret;
} finally {
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java Sun Oct 5 14:43:47 2014
@@ -340,11 +340,7 @@ public final class MockRandomPostingsFor
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: fixed-gap terms index");
}
- indexReader = new FixedGapTermsIndexReader(state.directory,
- state.fieldInfos,
- state.segmentInfo.name,
- BytesRef.getUTF8SortedAsUnicodeComparator(),
- state.segmentSuffix, state.context);
+ indexReader = new FixedGapTermsIndexReader(state);
} else {
final int n2 = random.nextInt(3);
if (n2 == 1) {
@@ -355,10 +351,7 @@ public final class MockRandomPostingsFor
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: variable-gap terms index");
}
- indexReader = new VariableGapTermsIndexReader(state.directory,
- state.fieldInfos,
- state.segmentInfo.name,
- state.segmentSuffix, state.context);
+ indexReader = new VariableGapTermsIndexReader(state);
}
@@ -371,13 +364,7 @@ public final class MockRandomPostingsFor
success = false;
try {
- fields = new BlockTermsReader(indexReader,
- state.directory,
- state.fieldInfos,
- state.segmentInfo,
- postingsReader,
- state.context,
- state.segmentSuffix);
+ fields = new BlockTermsReader(indexReader, postingsReader, state);
success = true;
} finally {
if (!success) {
@@ -396,12 +383,7 @@ public final class MockRandomPostingsFor
boolean success = false;
try {
- fields = new OrdsBlockTreeTermsReader(state.directory,
- state.fieldInfos,
- state.segmentInfo,
- postingsReader,
- state.context,
- state.segmentSuffix);
+ fields = new OrdsBlockTreeTermsReader(postingsReader, state);
success = true;
} finally {
if (!success) {
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java Sun Oct 5 14:43:47 2014
@@ -139,6 +139,19 @@ public abstract class BaseNormsFormatTes
}
}
+ public void testSparse() throws Exception {
+ int iterations = atLeast(1);
+ final Random r = random();
+ for (int i = 0; i < iterations; i++) {
+ doTestNormsVersusStoredFields(new LongProducer() {
+ @Override
+ long next() {
+ return r.nextInt(100) == 0 ? TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE) : 0;
+ }
+ });
+ }
+ }
+
private void doTestNormsVersusStoredFields(LongProducer longs) throws Exception {
int numDocs = atLeast(500);
long norms[] = new long[numDocs];
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java Sun Oct 5 14:43:47 2014
@@ -18,7 +18,6 @@ package org.apache.lucene.store;
*/
import java.io.EOFException;
-import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
@@ -584,39 +583,6 @@ public abstract class BaseDirectoryTestC
dir.close();
}
- // LUCENE-3382 test that delegate compound files correctly.
- public void testCompoundFileAppendTwice() throws IOException {
- Directory newDir = getDirectory(createTempDir("testCompoundFileAppendTwice"));
- CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
- createSequenceFile(newDir, "d1", (byte) 0, 15);
- IndexOutput out = csw.createOutput("d.xyz", newIOContext(random()));
- out.writeInt(0);
- out.close();
- assertEquals(1, csw.listAll().length);
- assertEquals("d.xyz", csw.listAll()[0]);
-
- csw.close();
-
- CompoundFileDirectory cfr = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
- assertEquals(1, cfr.listAll().length);
- assertEquals("d.xyz", cfr.listAll()[0]);
- cfr.close();
- newDir.close();
- }
-
- /** Creates a file of the specified size with sequential data. The first
- * byte is written as the start byte provided. All subsequent bytes are
- * computed as start + offset where offset is the number of the byte.
- */
- private void createSequenceFile(Directory dir, String name, byte start, int size) throws IOException {
- IndexOutput os = dir.createOutput(name, newIOContext(random()));
- for (int i=0; i < size; i++) {
- os.writeByte(start);
- start ++;
- }
- os.close();
- }
-
public void testCopyBytes() throws Exception {
testCopyBytes(getDirectory(createTempDir("testCopyBytes")));
}
@@ -1041,5 +1007,29 @@ public abstract class BaseDirectoryTestC
input.close();
dir.close();
}
+
+ /**
+ * This test that writes larger than the size of the buffer output
+ * will correctly increment the file pointer.
+ */
+ public void testLargeWrites() throws IOException {
+ Directory dir = getDirectory(createTempDir("largeWrites"));
+ IndexOutput os = dir.createOutput("testBufferStart.txt", newIOContext(random()));
+
+ byte[] largeBuf = new byte[2048];
+ for (int i=0; i<largeBuf.length; i++) {
+ largeBuf[i] = (byte) (Math.random() * 256);
+ }
+
+ long currentPos = os.getFilePointer();
+ os.writeBytes(largeBuf, largeBuf.length);
+
+ try {
+ assertEquals(currentPos + largeBuf.length, os.getFilePointer());
+ } finally {
+ os.close();
+ }
+ dir.close();
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java Sun Oct 5 14:43:47 2014
@@ -466,6 +466,11 @@ public class MockDirectoryWrapper extend
}
}
}
+
+ /** returns current open file handle count */
+ public synchronized long getFileHandleCount() {
+ return openFileHandles.size();
+ }
@Override
public synchronized void deleteFile(String name) throws IOException {
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Sun Oct 5 14:43:47 2014
@@ -868,7 +868,6 @@ public abstract class LuceneTestCase ext
}
c.setUseCompoundFile(r.nextBoolean());
c.setReaderPooling(r.nextBoolean());
- c.setCheckIntegrityAtMerge(r.nextBoolean());
return c;
}
@@ -1042,12 +1041,6 @@ public abstract class LuceneTestCase ext
}
if (rarely(r)) {
- // change merge integrity check parameters
- c.setCheckIntegrityAtMerge(r.nextBoolean());
- didChange = true;
- }
-
- if (rarely(r)) {
// change CMS merge parameters
MergeScheduler ms = c.getMergeScheduler();
if (ms instanceof ConcurrentMergeScheduler) {
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java Sun Oct 5 14:43:47 2014
@@ -232,8 +232,6 @@ public class SolrIndexConfig {
iwc.setMergedSegmentWarmer(warmer);
}
- iwc.setCheckIntegrityAtMerge(checkIntegrityAtMerge);
-
return iwc;
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java Sun Oct 5 14:43:47 2014
@@ -78,18 +78,6 @@ public class SolrIndexConfigTest extends
}
- @Test
- public void testCheckIntegrityAtMerge() throws Exception {
- SolrConfig solrConfig = new SolrConfig("solr" + File.separator
- + "collection1", "solrconfig-indexconfig.xml", null);
- SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null, null);
- assertNotNull(solrIndexConfig.checkIntegrityAtMerge);
- assertTrue(solrIndexConfig.checkIntegrityAtMerge);
- IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema("schema.xml", solrConfig);
- IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(indexSchema);
- assertTrue(iwc.getCheckIntegrityAtMerge());
- }
-
public void testMergedSegmentWarmerIndexConfigCreation() throws Exception {
SolrConfig solrConfig = new SolrConfig("solr" + File.separator
+ "collection1", "solrconfig-warmer.xml", null);