You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2011/01/23 06:07:47 UTC
svn commit: r1062324 - in /lucene/dev/branches/branch_3x/lucene: ./
src/java/org/apache/lucene/index/ src/java/org/apache/lucene/util/
Author: shaie
Date: Sun Jan 23 05:07:47 2011
New Revision: 1062324
URL: http://svn.apache.org/viewvc?rev=1062324&view=rev
Log:
LUCENE-2720: IndexWriter should throw IndexFormatTooOldExc on open, not later during optimize/getReader/close
Modified:
lucene/dev/branches/branch_3x/lucene/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldsReader.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/Constants.java
Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1062324&r1=1062323&r2=1062324&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Sun Jan 23 05:07:47 2011
@@ -469,6 +469,9 @@ New features
* LUCENE-2864: Add getMaxTermFrequency (maximum within-document TF) to
FieldInvertState so that it can be used in Similarity.computeNorm.
(Robert Muir)
+
+* LUCENE-2720: Segments now record the code version which created them.
+ (Shai Erera, Mike McCandless, Uwe Schindler)
Optimizations
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldsReader.java?rev=1062324&r1=1062323&r2=1062324&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldsReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldsReader.java Sun Jan 23 05:07:47 2011
@@ -75,6 +75,26 @@ final class FieldsReader implements Clon
ensureOpen();
return new FieldsReader(fieldInfos, numTotalDocs, size, format, formatSize, docStoreOffset, cloneableFieldsStream, cloneableIndexStream);
}
+
+ /**
+ * Detects the code version this segment was written with. Returns either
+ * "2.x" for all pre-3.0 segments, or "3.0" for 3.0 segments. This method
+ * should not be called for 3.1+ segments since they already record their code
+ * version.
+ */
+ static String detectCodeVersion(Directory dir, String segment) throws IOException {
+ IndexInput idxStream = dir.openInput(IndexFileNames.segmentFileName(segment, IndexFileNames.FIELDS_INDEX_EXTENSION), 1024);
+ try {
+ int format = idxStream.readInt();
+ if (format < FieldsWriter.FORMAT_LUCENE_3_0_NO_COMPRESSED_FIELDS) {
+ return "2.x";
+ } else {
+ return "3.0";
+ }
+ } finally {
+ idxStream.close();
+ }
+ }
// Used only by clone
private FieldsReader(FieldInfos fieldInfos, int numTotalDocs, int size, int format, int formatSize,
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1062324&r1=1062323&r2=1062324&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Sun Jan 23 05:07:47 2011
@@ -21,6 +21,8 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.BitVector;
+import org.apache.lucene.util.Constants;
+
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
@@ -95,6 +97,12 @@ public final class SegmentInfo {
private Map<String,String> diagnostics;
+ // Tracks the Lucene version this segment was created with, since 3.1. The
+ // format expected is "x.y" - "2.x" for pre-3.0 indexes, and specific versions
+ // afterwards ("3.0", "3.1" etc.).
+ // see Constants.LUCENE_MAIN_VERSION.
+ private String version;
+
public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, boolean hasSingleNormFile,
boolean hasProx, boolean hasVectors) {
this.name = name;
@@ -108,6 +116,7 @@ public final class SegmentInfo {
delCount = 0;
this.hasProx = hasProx;
this.hasVectors = hasVectors;
+ this.version = Constants.LUCENE_MAIN_VERSION;
}
/**
@@ -115,6 +124,7 @@ public final class SegmentInfo {
*/
void reset(SegmentInfo src) {
clearFiles();
+ version = src.version;
name = src.name;
docCount = src.docCount;
dir = src.dir;
@@ -153,6 +163,9 @@ public final class SegmentInfo {
*/
SegmentInfo(Directory dir, int format, IndexInput input) throws IOException {
this.dir = dir;
+ if (format <= SegmentInfos.FORMAT_3_1) {
+ version = input.readString();
+ }
name = input.readString();
docCount = input.readInt();
if (format <= SegmentInfos.FORMAT_LOCKLESS) {
@@ -361,6 +374,7 @@ public final class SegmentInfo {
if (normGen != null) {
si.normGen = normGen.clone();
}
+ si.version = version;
return si;
}
@@ -572,6 +586,8 @@ public final class SegmentInfo {
void write(IndexOutput output)
throws IOException {
assert delCount <= docCount: "delCount=" + delCount + " docCount=" + docCount + " segment=" + name;
+ // Write the Lucene version that created this segment, since 3.1
+ output.writeString(version);
output.writeString(name);
output.writeInt(docCount);
output.writeLong(delGen);
@@ -732,8 +748,9 @@ public final class SegmentInfo {
/** Used for debugging. Format may suddenly change.
*
* <p>Current format looks like
- * <code>_a:c45/4->_1</code>, which means the segment's
- * name is <code>_a</code>; it's using compound file
+ * <code>_a(3.1):c45/4->_1</code>, which means the segment's
+ * name is <code>_a</code>; it was created with Lucene 3.1 (or
+ * '?' if it's unkown); it's using compound file
* format (would be <code>C</code> if not compound); it
* has 45 documents; it has 4 deletions (this part is
* left off when there are no deletions); it's using the
@@ -743,7 +760,7 @@ public final class SegmentInfo {
public String toString(Directory dir, int pendingDelCount) {
StringBuilder s = new StringBuilder();
- s.append(name).append(':');
+ s.append(name).append('(').append(version == null ? "?" : version).append(')').append(':');
char cfs;
try {
@@ -813,4 +830,24 @@ public final class SegmentInfo {
public int hashCode() {
return dir.hashCode() + name.hashCode();
}
+
+ /**
+ * Used by SegmentInfos to upgrade segments that do not record their code
+ * version (either "2.x" or "3.0").
+ * <p>
+ * <b>NOTE:</b> this method is used for internal purposes only - you should
+ * not modify the version of a SegmentInfo, or it may result in unexpected
+ * exceptions thrown when you attempt to open the index.
+ *
+ * @lucene.internal
+ */
+ void setVersion(String version) {
+ this.version = version;
+ }
+
+ /** Returns the version of the code which wrote the segment. */
+ public String getVersion() {
+ return version;
+ }
+
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1062324&r1=1062323&r2=1062324&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java Sun Jan 23 05:07:47 2011
@@ -92,8 +92,11 @@ public final class SegmentInfos extends
/** Each segment records whether it has term vectors */
public static final int FORMAT_HAS_VECTORS = -10;
+ /** Each segment records the Lucene version that created it. */
+ public static final int FORMAT_3_1 = -11;
+
/* This must always point to the most recent file format. */
- public static final int CURRENT_FORMAT = FORMAT_HAS_VECTORS;
+ public static final int CURRENT_FORMAT = FORMAT_3_1;
public int counter = 0; // used to name new segments
/**
@@ -267,7 +270,30 @@ public final class SegmentInfos extends
}
for (int i = input.readInt(); i > 0; i--) { // read segmentInfos
- add(new SegmentInfo(directory, format, input));
+ SegmentInfo si = new SegmentInfo(directory, format, input);
+ if (si.getVersion() == null) {
+ // It's a pre-3.1 segment, upgrade its version to either 3.0 or 2.x
+ Directory dir = directory;
+ if (si.getDocStoreOffset() != -1) {
+ if (si.getDocStoreIsCompoundFile()) {
+ dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName(
+ si.getDocStoreSegment(),
+ IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), 1024);
+ }
+ } else if (si.getUseCompoundFile()) {
+ dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName(
+ si.name, IndexFileNames.COMPOUND_FILE_EXTENSION), 1024);
+ }
+
+ try {
+ String store = si.getDocStoreOffset() != -1 ? si.getDocStoreSegment() : si.name;
+ si.setVersion(FieldsReader.detectCodeVersion(dir, store));
+ } finally {
+ // If we opened the directory, close it
+ if (dir != directory) dir.close();
+ }
+ }
+ add(si);
}
if(format >= 0){ // in old format the version number may be at the end of the file
@@ -351,8 +377,8 @@ public final class SegmentInfos extends
segnOutput.writeLong(version);
segnOutput.writeInt(counter); // write counter
segnOutput.writeInt(size()); // write infos
- for (int i = 0; i < size(); i++) {
- info(i).write(segnOutput);
+ for (SegmentInfo si : this) {
+ si.write(segnOutput);
}
segnOutput.writeStringStringMap(userData);
segnOutput.prepareCommit();
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/Constants.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/Constants.java?rev=1062324&r1=1062323&r2=1062324&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/Constants.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/Constants.java Sun Jan 23 05:07:47 2011
@@ -70,6 +70,9 @@ public final class Constants {
return s.toString();
}
+ // NOTE: we track per-segment version as a String with the "X.Y" format, e.g.
+ // "4.0", "3.1", "3.0". Therefore when we change this constant, we should keep
+ // the format.
public static final String LUCENE_MAIN_VERSION = ident("3.1");
public static final String LUCENE_VERSION;