You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/11/14 15:52:22 UTC
svn commit: r1201732 - in
/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index:
SegmentInfo.java codecs/DefaultSegmentInfosReader.java
codecs/DefaultSegmentInfosWriter.java
Author: rmuir
Date: Mon Nov 14 14:52:22 2011
New Revision: 1201732
URL: http://svn.apache.org/viewvc?rev=1201732&view=rev
Log:
LUCENE-2621: move SI i/o into codec
Modified:
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1201732&r1=1201731&r2=1201732&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Mon Nov 14 14:52:22 2011
@@ -28,13 +28,9 @@ import java.util.Map.Entry;
import java.util.Set;
import org.apache.lucene.index.codecs.Codec;
-import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter;
-import org.apache.lucene.index.codecs.DefaultTermVectorsReader;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.StringHelper;
@@ -46,11 +42,11 @@ import org.apache.lucene.util.StringHelp
*/
public final class SegmentInfo implements Cloneable {
// TODO: remove with hasVector and hasProx
- private static final int CHECK_FIELDINFO = -2;
+ public static final int CHECK_FIELDINFO = -2;
- // TODO: remove these from this class
- public static final int NO = DefaultSegmentInfosWriter.NO; // e.g. no norms; no deletes;
- public static final int YES = DefaultSegmentInfosWriter.YES; // e.g. have norms; have deletes;
+ // TODO: remove these from this class, for now this is the representation
+ public static final int NO = -1; // e.g. no norms; no deletes;
+ public static final int YES = 1; // e.g. have norms; have deletes;
static final int WITHOUT_GEN = 0; // a file name that has no GEN in it.
public String name; // unique name in dir
@@ -170,102 +166,30 @@ public final class SegmentInfo implement
}
/**
- * Construct a new SegmentInfo instance by reading a
- * previously saved SegmentInfo from input.
+ * Construct a new complete SegmentInfo instance from input.
* <p>Note: this is public only to allow access from
* the codecs package.</p>
- *
- * @param dir directory to load from
- * @param format format of the segments info file
- * @param input input handle to read segment info from
*/
- // TODO pull this i/o out of this class, maybe codec reader should just create SIs?
- public SegmentInfo(Directory dir, int format, IndexInput input) throws IOException {
+ public SegmentInfo(Directory dir, String version, String name, int docCount, long delGen, int docStoreOffset,
+ String docStoreSegment, boolean docStoreIsCompoundFile, Map<Integer,Long> normGen, boolean isCompoundFile,
+ int delCount, int hasProx, Codec codec, Map<String,String> diagnostics, int hasVectors) {
this.dir = dir;
- if (format <= DefaultSegmentInfosWriter.FORMAT_3_1) {
- version = input.readString();
- }
- name = input.readString();
- docCount = input.readInt();
- delGen = input.readLong();
- docStoreOffset = input.readInt();
- if (docStoreOffset != -1) {
- docStoreSegment = input.readString();
- docStoreIsCompoundFile = input.readByte() == YES;
- } else {
- docStoreSegment = name;
- docStoreIsCompoundFile = false;
- }
-
- if (format > DefaultSegmentInfosWriter.FORMAT_4_0) {
- // pre-4.0 indexes write a byte if there is a single norms file
- byte b = input.readByte();
- assert 1 == b;
- }
-
- int numNormGen = input.readInt();
- if (numNormGen == NO) {
- normGen = null;
- } else {
- normGen = new HashMap<Integer, Long>();
- for(int j=0;j<numNormGen;j++) {
- int fieldNumber = j;
- if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) {
- fieldNumber = input.readInt();
- }
-
- normGen.put(fieldNumber, input.readLong());
- }
- }
- isCompoundFile = input.readByte() == YES;
-
- delCount = input.readInt();
- assert delCount <= docCount;
-
- hasProx = input.readByte();
-
-
- // System.out.println(Thread.currentThread().getName() + ": si.read hasProx=" + hasProx + " seg=" + name);
- // note: if the codec is not available: Codec.forName will throw an exception.
- if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) {
- codec = Codec.forName(input.readString());
- } else {
- codec = Codec.forName("Lucene3x");
- }
- diagnostics = input.readStringStringMap();
-
- if (format <= DefaultSegmentInfosWriter.FORMAT_HAS_VECTORS) {
- hasVectors = input.readByte();
- } else {
- final String storesSegment;
- final String ext;
- final boolean isCompoundFile;
- if (docStoreOffset != -1) {
- storesSegment = docStoreSegment;
- isCompoundFile = docStoreIsCompoundFile;
- ext = IndexFileNames.COMPOUND_FILE_STORE_EXTENSION;
- } else {
- storesSegment = name;
- isCompoundFile = getUseCompoundFile();
- ext = IndexFileNames.COMPOUND_FILE_EXTENSION;
- }
- final Directory dirToTest;
- if (isCompoundFile) {
- dirToTest = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(storesSegment, "", ext), IOContext.READONCE, false);
- } else {
- dirToTest = dir;
- }
- try {
- // TODO: remove this manual file check or push to preflex codec
- hasVectors = dirToTest.fileExists(IndexFileNames.segmentFileName(storesSegment, "", DefaultTermVectorsReader.VECTORS_INDEX_EXTENSION)) ? YES : NO;
- } finally {
- if (isCompoundFile) {
- dirToTest.close();
- }
- }
- }
+ this.version = version;
+ this.name = name;
+ this.docCount = docCount;
+ this.delGen = delGen;
+ this.docStoreOffset = docStoreOffset;
+ this.docStoreSegment = docStoreSegment;
+ this.docStoreIsCompoundFile = docStoreIsCompoundFile;
+ this.normGen = normGen;
+ this.isCompoundFile = isCompoundFile;
+ this.delCount = delCount;
+ this.hasProx = hasProx;
+ this.codec = codec;
+ this.diagnostics = diagnostics;
+ this.hasVectors = hasVectors;
}
-
+
synchronized void loadFieldInfos(Directory dir, boolean checkCompoundFile) throws IOException {
if (fieldInfos == null) {
Directory dir0 = dir;
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java?rev=1201732&r1=1201731&r2=1201732&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java Mon Nov 14 14:52:22 2011
@@ -18,6 +18,8 @@ package org.apache.lucene.index.codecs;
*/
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexFormatTooOldException;
@@ -51,7 +53,7 @@ public class DefaultSegmentInfosReader e
infos.setGlobalFieldMapVersion(input.readLong());
}
for (int i = input.readInt(); i > 0; i--) { // read segmentInfos
- SegmentInfo si = new SegmentInfo(directory, format, input);
+ SegmentInfo si = readSegmentInfo(directory, format, input);
if (si.getVersion() == null) {
// Could be a 3.0 - try to open the doc stores - if it fails, it's a
// 2.x segment, and an IndexFormatTooOldException will be thrown,
@@ -90,4 +92,100 @@ public class DefaultSegmentInfosReader e
infos.userData = input.readStringStringMap();
}
+
+ // if we make a preflex impl we can remove a lot of this hair...
+ public SegmentInfo readSegmentInfo(Directory dir, int format, ChecksumIndexInput input) throws IOException {
+ final String version;
+ if (format <= DefaultSegmentInfosWriter.FORMAT_3_1) {
+ version = input.readString();
+ } else {
+ version = null;
+ }
+ final String name = input.readString();
+ final int docCount = input.readInt();
+ final long delGen = input.readLong();
+ final int docStoreOffset = input.readInt();
+ final String docStoreSegment;
+ final boolean docStoreIsCompoundFile;
+ if (docStoreOffset != -1) {
+ docStoreSegment = input.readString();
+ docStoreIsCompoundFile = input.readByte() == SegmentInfo.YES;
+ } else {
+ docStoreSegment = name;
+ docStoreIsCompoundFile = false;
+ }
+
+ if (format > DefaultSegmentInfosWriter.FORMAT_4_0) {
+ // pre-4.0 indexes write a byte if there is a single norms file
+ byte b = input.readByte();
+ assert 1 == b;
+ }
+
+ final int numNormGen = input.readInt();
+ final Map<Integer,Long> normGen;
+ if (numNormGen == SegmentInfo.NO) {
+ normGen = null;
+ } else {
+ normGen = new HashMap<Integer, Long>();
+ for(int j=0;j<numNormGen;j++) {
+ int fieldNumber = j;
+ if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) {
+ fieldNumber = input.readInt();
+ }
+
+ normGen.put(fieldNumber, input.readLong());
+ }
+ }
+ final boolean isCompoundFile = input.readByte() == SegmentInfo.YES;
+
+ final int delCount = input.readInt();
+ assert delCount <= docCount;
+
+ final int hasProx = input.readByte();
+
+ final Codec codec;
+ // note: if the codec is not available: Codec.forName will throw an exception.
+ if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) {
+ codec = Codec.forName(input.readString());
+ } else {
+ codec = Codec.forName("Lucene3x");
+ }
+ final Map<String,String> diagnostics = input.readStringStringMap();
+
+ final int hasVectors;
+ if (format <= DefaultSegmentInfosWriter.FORMAT_HAS_VECTORS) {
+ hasVectors = input.readByte();
+ } else {
+ final String storesSegment;
+ final String ext;
+ final boolean storeIsCompoundFile;
+ if (docStoreOffset != -1) {
+ storesSegment = docStoreSegment;
+ storeIsCompoundFile = docStoreIsCompoundFile;
+ ext = IndexFileNames.COMPOUND_FILE_STORE_EXTENSION;
+ } else {
+ storesSegment = name;
+ storeIsCompoundFile = isCompoundFile;
+ ext = IndexFileNames.COMPOUND_FILE_EXTENSION;
+ }
+ final Directory dirToTest;
+ if (storeIsCompoundFile) {
+ dirToTest = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(storesSegment, "", ext), IOContext.READONCE, false);
+ } else {
+ dirToTest = dir;
+ }
+ try {
+ // TODO: remove this manual file check or push to preflex codec
+ hasVectors = dirToTest.fileExists(IndexFileNames.segmentFileName(storesSegment, "", DefaultTermVectorsReader.VECTORS_INDEX_EXTENSION)) ? SegmentInfo.YES : SegmentInfo.NO;
+ } finally {
+ if (isCompoundFile) {
+ dirToTest.close();
+ }
+ }
+ }
+
+ return new SegmentInfo(dir, version, name, docCount, delGen, docStoreOffset,
+ docStoreSegment, docStoreIsCompoundFile, normGen, isCompoundFile,
+ delCount, hasProx, codec, diagnostics, hasVectors);
+ }
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java?rev=1201732&r1=1201731&r2=1201732&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java Mon Nov 14 14:52:22 2011
@@ -83,9 +83,6 @@ public class DefaultSegmentInfosWriter e
}
}
- public static final int NO = -1; // e.g. no norms; no deletes;
- public static final int YES = 1; // e.g. have norms; have deletes;
-
/** Save a single segment's info. */
private void writeInfo(IndexOutput output, SegmentInfo si) throws IOException {
assert si.getDelCount() <= si.docCount: "delCount=" + si.getDelCount() + " docCount=" + si.docCount + " segment=" + si.name;
@@ -103,7 +100,7 @@ public class DefaultSegmentInfosWriter e
Map<Integer,Long> normGen = si.getNormGen();
if (normGen == null) {
- output.writeInt(NO);
+ output.writeInt(SegmentInfo.NO);
} else {
output.writeInt(normGen.size());
for (Entry<Integer,Long> entry : normGen.entrySet()) {
@@ -112,7 +109,7 @@ public class DefaultSegmentInfosWriter e
}
}
- output.writeByte((byte) (si.getUseCompoundFile() ? YES : NO));
+ output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));
output.writeInt(si.getDelCount());
output.writeByte((byte) (si.getHasProxInternal()));
output.writeString(si.getCodec().getName());