You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/11/02 04:45:01 UTC
svn commit: r1196443 - in /lucene/dev/branches/lucene2621/lucene:
contrib/misc/src/java/org/apache/lucene/index/codecs/appending/
src/java/org/apache/lucene/index/ src/java/org/apache/lucene/index/codecs/
Author: rmuir
Date: Wed Nov 2 03:45:01 2011
New Revision: 1196443
URL: http://svn.apache.org/viewvc?rev=1196443&view=rev
Log:
LUCENE-3490: dispatch SIReader, fix AppendingCodec
Removed:
lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosReader.java
Modified:
lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosFormat.java
lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java
Modified: lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosFormat.java?rev=1196443&r1=1196442&r2=1196443&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosFormat.java (original)
+++ lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosFormat.java Wed Nov 2 03:45:01 2011
@@ -17,20 +17,13 @@ package org.apache.lucene.index.codecs.a
* limitations under the License.
*/
-import org.apache.lucene.index.codecs.SegmentInfosFormat;
-import org.apache.lucene.index.codecs.SegmentInfosReader;
+import org.apache.lucene.index.codecs.DefaultSegmentInfosFormat;
import org.apache.lucene.index.codecs.SegmentInfosWriter;
-public class AppendingSegmentInfosFormat extends SegmentInfosFormat {
- private final SegmentInfosReader reader = new AppendingSegmentInfosReader();
+public class AppendingSegmentInfosFormat extends DefaultSegmentInfosFormat {
private final SegmentInfosWriter writer = new AppendingSegmentInfosWriter();
@Override
- public SegmentInfosReader getSegmentInfosReader() {
- return reader;
- }
-
- @Override
public SegmentInfosWriter getSegmentInfosWriter() {
return writer;
}
Modified: lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java?rev=1196443&r1=1196442&r2=1196443&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java (original)
+++ lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java Wed Nov 2 03:45:01 2011
@@ -20,24 +20,11 @@ package org.apache.lucene.index.codecs.a
import java.io.IOException;
import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
public class AppendingSegmentInfosWriter extends DefaultSegmentInfosWriter {
@Override
- protected IndexOutput createOutput(Directory dir, String segmentsFileName, IOContext context)
- throws IOException {
- return dir.createOutput(segmentsFileName, context);
- }
-
- @Override
- public void finishCommit(IndexOutput out) throws IOException {
- out.close();
- }
-
- @Override
public void prepareCommit(IndexOutput segmentOutput) throws IOException {
// noop
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1196443&r1=1196442&r2=1196443&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfos.java Wed Nov 2 03:45:01 2011
@@ -36,6 +36,7 @@ import org.apache.lucene.index.codecs.Co
import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter;
import org.apache.lucene.index.codecs.SegmentInfosReader;
import org.apache.lucene.index.codecs.SegmentInfosWriter;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -243,14 +244,40 @@ public final class SegmentInfos implemen
lastGeneration = generation;
+ // nocommit: scary to have default impl reopen the file... but to make it a bit more flexible,
+ // maybe we could use a plain indexinput here... could default impl rewind/wrap with checksumII,
+ // and any checksumming is then up to implementation?
+ ChecksumIndexInput input = null;
try {
- // nocommit: read preamble here then forward to codec
- codecFormat = Codec.forName("Lucene40");
+ input = new ChecksumIndexInput(directory.openInput(segmentFileName, IOContext.READ));
+ final int format = input.readInt();
+ setFormat(format);
+
+ // check that it is a format we can understand
+ if (format > DefaultSegmentInfosWriter.FORMAT_MINIMUM)
+ throw new IndexFormatTooOldException(segmentFileName, format,
+ DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT);
+ if (format < DefaultSegmentInfosWriter.FORMAT_CURRENT)
+ throw new IndexFormatTooNewException(segmentFileName, format,
+ DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT);
+
+ if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) {
+ codecFormat = Codec.forName(input.readString());
+ } else {
+ codecFormat = Codec.forName("Lucene3x");
+ }
SegmentInfosReader infosReader = codecFormat.segmentInfosFormat().getSegmentInfosReader();
- infosReader.read(directory, segmentFileName, this, IOContext.READ);
+ infosReader.read(directory, segmentFileName, input, this, IOContext.READ);
+ final long checksumNow = input.getChecksum();
+ final long checksumThen = input.readLong();
+ if (checksumNow != checksumThen)
+ throw new CorruptIndexException("checksum mismatch in segments file");
success = true;
}
finally {
+ if (input != null) {
+ input.close();
+ }
if (!success) {
// Clear any segment infos we had loaded so we
// have a clean slate on retry:
@@ -304,7 +331,7 @@ public final class SegmentInfos implemen
try {
SegmentInfosWriter infosWriter = codec.segmentInfosFormat().getSegmentInfosWriter();
- segnOutput = infosWriter.writeInfos(directory, segmentFileName, this, IOContext.DEFAULT);
+ segnOutput = infosWriter.writeInfos(directory, segmentFileName, codec.getName(), this, IOContext.DEFAULT);
infosWriter.prepareCommit(segnOutput);
pendingSegnOutput = segnOutput;
success = true;
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java?rev=1196443&r1=1196442&r2=1196443&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java Wed Nov 2 03:45:01 2011
@@ -19,17 +19,14 @@ package org.apache.lucene.index.codecs;
import java.io.IOException;
-import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexFormatTooOldException;
-import org.apache.lucene.index.IndexFormatTooNewException;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
/**
* Default implementation of {@link SegmentInfosReader}.
@@ -37,89 +34,53 @@ import org.apache.lucene.store.IndexInpu
*/
public class DefaultSegmentInfosReader extends SegmentInfosReader {
+ // nocommit: shove all backwards code to preflex!
@Override
- public void read(Directory directory, String segmentsFileName, SegmentInfos infos, IOContext context) throws IOException {
- IndexInput input = null;
- try {
- input = openInput(directory, segmentsFileName, context);
- final int format = input.readInt();
- infos.setFormat(format);
-
- // check that it is a format we can understand
- if (format > DefaultSegmentInfosWriter.FORMAT_MINIMUM)
- throw new IndexFormatTooOldException(segmentsFileName, format,
- DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT);
- if (format < DefaultSegmentInfosWriter.FORMAT_CURRENT)
- throw new IndexFormatTooNewException(segmentsFileName, format,
- DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT);
-
- infos.version = input.readLong(); // read version
- infos.counter = input.readInt(); // read counter
- if (infos.getFormat() <= DefaultSegmentInfosWriter.FORMAT_4_0) {
- infos.setGlobalFieldMapVersion(input.readLong());
- }
- for (int i = input.readInt(); i > 0; i--) { // read segmentInfos
- SegmentInfo si = new SegmentInfo(directory, format, input);
- if (si.getVersion() == null) {
- // Could be a 3.0 - try to open the doc stores - if it fails, it's a
- // 2.x segment, and an IndexFormatTooOldException will be thrown,
- // which is what we want.
- Directory dir = directory;
- if (si.getDocStoreOffset() != -1) {
- if (si.getDocStoreIsCompoundFile()) {
- dir = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(
- si.getDocStoreSegment(), "",
- IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), context, false);
- }
- } else if (si.getUseCompoundFile()) {
- dir = new CompoundFileDirectory(dir,IndexFileNames.segmentFileName(
- si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false);
+ public void read(Directory directory, String segmentsFileName, ChecksumIndexInput input, SegmentInfos infos, IOContext context) throws IOException {
+ infos.version = input.readLong(); // read version
+ infos.counter = input.readInt(); // read counter
+ final int format = infos.getFormat();
+ if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) {
+ infos.setGlobalFieldMapVersion(input.readLong());
+ }
+ for (int i = input.readInt(); i > 0; i--) { // read segmentInfos
+ SegmentInfo si = new SegmentInfo(directory, format, input);
+ if (si.getVersion() == null) {
+ // Could be a 3.0 - try to open the doc stores - if it fails, it's a
+ // 2.x segment, and an IndexFormatTooOldException will be thrown,
+ // which is what we want.
+ Directory dir = directory;
+ if (si.getDocStoreOffset() != -1) {
+ if (si.getDocStoreIsCompoundFile()) {
+ dir = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(
+ si.getDocStoreSegment(), "",
+ IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), context, false);
}
+ } else if (si.getUseCompoundFile()) {
+ dir = new CompoundFileDirectory(dir,IndexFileNames.segmentFileName(
+ si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false);
+ }
- try {
- DefaultFieldsReader.checkCodeVersion(dir, si.getDocStoreSegment());
- } finally {
- // If we opened the directory, close it
- if (dir != directory) dir.close();
- }
-
- // Above call succeeded, so it's a 3.0 segment. Upgrade it so the next
- // time the segment is read, its version won't be null and we won't
- // need to open FieldsReader every time for each such segment.
- si.setVersion("3.0");
- } else if (si.getVersion().equals("2.x")) {
- // If it's a 3x index touched by 3.1+ code, then segments record their
- // version, whether they are 2.x ones or not. We detect that and throw
- // appropriate exception.
- throw new IndexFormatTooOldException(si.name, si.getVersion());
+ try {
+ DefaultFieldsReader.checkCodeVersion(dir, si.getDocStoreSegment());
+ } finally {
+ // If we opened the directory, close it
+ if (dir != directory) dir.close();
}
- infos.add(si);
- }
-
- infos.userData = input.readStringStringMap();
- finalizeInput(input);
-
- } finally {
- if (input != null) {
- input.close();
+
+ // Above call succeeded, so it's a 3.0 segment. Upgrade it so the next
+ // time the segment is read, its version won't be null and we won't
+ // need to open FieldsReader every time for each such segment.
+ si.setVersion("3.0");
+ } else if (si.getVersion().equals("2.x")) {
+ // If it's a 3x index touched by 3.1+ code, then segments record their
+ // version, whether they are 2.x ones or not. We detect that and throw
+ // appropriate exception.
+ throw new IndexFormatTooOldException(si.name, si.getVersion());
}
+ infos.add(si);
}
-
- }
-
- public IndexInput openInput(Directory dir, String segmentsFileName, IOContext context) throws IOException {
- IndexInput in = dir.openInput(segmentsFileName, context);
- return new ChecksumIndexInput(in);
-
- }
-
- public void finalizeInput(IndexInput input) throws IOException, CorruptIndexException {
- ChecksumIndexInput cksumInput = (ChecksumIndexInput)input;
- final long checksumNow = cksumInput.getChecksum();
- final long checksumThen = cksumInput.readLong();
- if (checksumNow != checksumThen)
- throw new CorruptIndexException("checksum mismatch in segments file");
-
+
+ infos.userData = input.readStringStringMap();
}
-
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java?rev=1196443&r1=1196442&r2=1196443&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java Wed Nov 2 03:45:01 2011
@@ -56,12 +56,13 @@ public class DefaultSegmentInfosWriter e
public static final int FORMAT_MINIMUM = FORMAT_DIAGNOSTICS;
@Override
- public IndexOutput writeInfos(Directory dir, String segmentFileName, SegmentInfos infos, IOContext context)
+ public IndexOutput writeInfos(Directory dir, String segmentFileName, String codecID, SegmentInfos infos, IOContext context)
throws IOException {
IndexOutput out = createOutput(dir, segmentFileName, new IOContext(new FlushInfo(infos.size(), infos.totalDocCount())));
boolean success = false;
try {
out.writeInt(FORMAT_CURRENT); // write FORMAT
+ out.writeString(codecID); // write codecID
out.writeLong(infos.version);
out.writeInt(infos.counter); // write counter
out.writeLong(infos.getGlobalFieldMapVersion());
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java?rev=1196443&r1=1196442&r2=1196443&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java Wed Nov 2 03:45:01 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs;
import java.io.IOException;
import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -37,5 +38,5 @@ public abstract class SegmentInfosReader
* @param infos empty instance to be populated with data
* @throws IOException
*/
- public abstract void read(Directory directory, String segmentsFileName, SegmentInfos infos, IOContext context) throws IOException;
+ public abstract void read(Directory directory, String segmentsFileName, ChecksumIndexInput header, SegmentInfos infos, IOContext context) throws IOException;
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java?rev=1196443&r1=1196442&r2=1196443&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java Wed Nov 2 03:45:01 2011
@@ -42,7 +42,7 @@ public abstract class SegmentInfosWriter
* phase commit" operations as described above.
* @throws IOException
*/
- public abstract IndexOutput writeInfos(Directory dir, String segmentsFileName, SegmentInfos infos, IOContext context) throws IOException;
+ public abstract IndexOutput writeInfos(Directory dir, String segmentsFileName, String codecID, SegmentInfos infos, IOContext context) throws IOException;
/**
* First phase of the two-phase commit - ensure that all output can be