You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/05/28 21:28:18 UTC
svn commit: r1343365 [4/8] - in /lucene/dev/trunk: ./ lucene/
lucene/analysis/ lucene/analysis/common/ lucene/core/
lucene/core/src/java/org/apache/lucene/codecs/
lucene/core/src/java/org/apache/lucene/codecs/appending/
lucene/core/src/java/org/apache/...
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java Mon May 28 19:28:10 2012
@@ -121,9 +121,9 @@ class BufferedDeletesStream {
public final long gen;
// If non-null, contains segments that are 100% deleted
- public final List<SegmentInfo> allDeleted;
+ public final List<SegmentInfoPerCommit> allDeleted;
- ApplyDeletesResult(boolean anyDeletes, long gen, List<SegmentInfo> allDeleted) {
+ ApplyDeletesResult(boolean anyDeletes, long gen, List<SegmentInfoPerCommit> allDeleted) {
this.anyDeletes = anyDeletes;
this.gen = gen;
this.allDeleted = allDeleted;
@@ -131,9 +131,9 @@ class BufferedDeletesStream {
}
// Sorts SegmentInfos from smallest to biggest bufferedDelGen:
- private static final Comparator<SegmentInfo> sortSegInfoByDelGen = new Comparator<SegmentInfo>() {
+ private static final Comparator<SegmentInfoPerCommit> sortSegInfoByDelGen = new Comparator<SegmentInfoPerCommit>() {
@Override
- public int compare(SegmentInfo si1, SegmentInfo si2) {
+ public int compare(SegmentInfoPerCommit si1, SegmentInfoPerCommit si2) {
final long cmp = si1.getBufferedDeletesGen() - si2.getBufferedDeletesGen();
if (cmp > 0) {
return 1;
@@ -148,7 +148,7 @@ class BufferedDeletesStream {
/** Resolves the buffered deleted Term/Query/docIDs, into
* actual deleted docIDs in the liveDocs MutableBits for
* each SegmentReader. */
- public synchronized ApplyDeletesResult applyDeletes(IndexWriter.ReaderPool readerPool, List<SegmentInfo> infos) throws IOException {
+ public synchronized ApplyDeletesResult applyDeletes(IndexWriter.ReaderPool readerPool, List<SegmentInfoPerCommit> infos) throws IOException {
final long t0 = System.currentTimeMillis();
if (infos.size() == 0) {
@@ -168,7 +168,7 @@ class BufferedDeletesStream {
infoStream.message("BD", "applyDeletes: infos=" + infos + " packetCount=" + deletes.size());
}
- List<SegmentInfo> infos2 = new ArrayList<SegmentInfo>();
+ List<SegmentInfoPerCommit> infos2 = new ArrayList<SegmentInfoPerCommit>();
infos2.addAll(infos);
Collections.sort(infos2, sortSegInfoByDelGen);
@@ -178,13 +178,13 @@ class BufferedDeletesStream {
int infosIDX = infos2.size()-1;
int delIDX = deletes.size()-1;
- List<SegmentInfo> allDeleted = null;
+ List<SegmentInfoPerCommit> allDeleted = null;
while (infosIDX >= 0) {
//System.out.println("BD: cycle delIDX=" + delIDX + " infoIDX=" + infosIDX);
final FrozenBufferedDeletes packet = delIDX >= 0 ? deletes.get(delIDX) : null;
- final SegmentInfo info = infos2.get(infosIDX);
+ final SegmentInfoPerCommit info = infos2.get(infosIDX);
final long segGen = info.getBufferedDeletesGen();
if (packet != null && segGen < packet.delGen()) {
@@ -225,8 +225,8 @@ class BufferedDeletesStream {
// already did that on flush:
delCount += applyQueryDeletes(packet.queriesIterable(), rld, reader);
final int fullDelCount = rld.info.getDelCount() + rld.getPendingDeleteCount();
- assert fullDelCount <= rld.info.docCount;
- segAllDeletes = fullDelCount == rld.info.docCount;
+ assert fullDelCount <= rld.info.info.getDocCount();
+ segAllDeletes = fullDelCount == rld.info.info.getDocCount();
} finally {
rld.release(reader);
readerPool.release(rld);
@@ -235,7 +235,7 @@ class BufferedDeletesStream {
if (segAllDeletes) {
if (allDeleted == null) {
- allDeleted = new ArrayList<SegmentInfo>();
+ allDeleted = new ArrayList<SegmentInfoPerCommit>();
}
allDeleted.add(info);
}
@@ -271,8 +271,8 @@ class BufferedDeletesStream {
delCount += applyTermDeletes(coalescedDeletes.termsIterable(), rld, reader);
delCount += applyQueryDeletes(coalescedDeletes.queriesIterable(), rld, reader);
final int fullDelCount = rld.info.getDelCount() + rld.getPendingDeleteCount();
- assert fullDelCount <= rld.info.docCount;
- segAllDeletes = fullDelCount == rld.info.docCount;
+ assert fullDelCount <= rld.info.info.getDocCount();
+ segAllDeletes = fullDelCount == rld.info.info.getDocCount();
} finally {
rld.release(reader);
readerPool.release(rld);
@@ -281,7 +281,7 @@ class BufferedDeletesStream {
if (segAllDeletes) {
if (allDeleted == null) {
- allDeleted = new ArrayList<SegmentInfo>();
+ allDeleted = new ArrayList<SegmentInfoPerCommit>();
}
allDeleted.add(info);
}
@@ -316,7 +316,7 @@ class BufferedDeletesStream {
public synchronized void prune(SegmentInfos segmentInfos) {
assert checkDeleteStats();
long minGen = Long.MAX_VALUE;
- for(SegmentInfo info : segmentInfos) {
+ for(SegmentInfoPerCommit info : segmentInfos) {
minGen = Math.min(info.getBufferedDeletesGen(), minGen);
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Mon May 28 19:28:10 2012
@@ -88,9 +88,6 @@ public class CheckIndex {
/** Number of segments in the index. */
public int numSegments;
- /** String description of the version of the index. */
- public String segmentFormat;
-
/** Empty unless you passed specific segments list to check as optional 3rd argument.
* @see CheckIndex#checkIndex(List) */
public List<String> segmentsChecked = new ArrayList<String>();
@@ -185,11 +182,6 @@ public class CheckIndex {
/** Number of fields in this segment. */
int numFields;
- /** True if at least one of the fields in this segment
- * has position data
- * @see FieldType#setIndexOptions(org.apache.lucene.index.FieldInfo.IndexOptions) */
- public boolean hasProx;
-
/** Map that includes certain
* debugging details that IndexWriter records into
* each segment it creates */
@@ -369,8 +361,8 @@ public class CheckIndex {
String oldSegs = null;
boolean foundNonNullVersion = false;
Comparator<String> versionComparator = StringHelper.getVersionComparator();
- for (SegmentInfo si : sis) {
- String version = si.getVersion();
+ for (SegmentInfoPerCommit si : sis) {
+ String version = si.info.getVersion();
if (version == null) {
// pre-3.1 segment
oldSegs = "pre-3.1";
@@ -415,27 +407,8 @@ public class CheckIndex {
String sFormat = "";
boolean skip = false;
- if (format == SegmentInfos.FORMAT_DIAGNOSTICS) {
- sFormat = "FORMAT_DIAGNOSTICS [Lucene 2.9]";
- } else if (format == SegmentInfos.FORMAT_HAS_VECTORS) {
- sFormat = "FORMAT_HAS_VECTORS [Lucene 3.1]";
- } else if (format == SegmentInfos.FORMAT_3_1) {
- sFormat = "FORMAT_3_1 [Lucene 3.1+]";
- } else if (format == SegmentInfos.FORMAT_4_0) {
- sFormat = "FORMAT_4_0 [Lucene 4.0]";
- } else if (format == SegmentInfos.FORMAT_CURRENT) {
- throw new RuntimeException("BUG: You should update this tool!");
- } else if (format < SegmentInfos.FORMAT_CURRENT) {
- sFormat = "int=" + format + " [newer version of Lucene than this tool supports]";
- skip = true;
- } else if (format > SegmentInfos.FORMAT_MINIMUM) {
- sFormat = "int=" + format + " [older version of Lucene than this tool supports]";
- skip = true;
- }
-
result.segmentsFileName = segmentsFileName;
result.numSegments = numSegments;
- result.segmentFormat = sFormat;
result.userData = sis.getUserData();
String userDataString;
if (sis.getUserData().size() > 0) {
@@ -482,50 +455,41 @@ public class CheckIndex {
result.maxSegmentName = -1;
for(int i=0;i<numSegments;i++) {
- final SegmentInfo info = sis.info(i);
- int segmentName = Integer.parseInt(info.name.substring(1), Character.MAX_RADIX);
+ final SegmentInfoPerCommit info = sis.info(i);
+ int segmentName = Integer.parseInt(info.info.name.substring(1), Character.MAX_RADIX);
if (segmentName > result.maxSegmentName) {
result.maxSegmentName = segmentName;
}
- if (onlySegments != null && !onlySegments.contains(info.name))
+ if (onlySegments != null && !onlySegments.contains(info.info.name)) {
continue;
+ }
Status.SegmentInfoStatus segInfoStat = new Status.SegmentInfoStatus();
result.segmentInfos.add(segInfoStat);
- msg(" " + (1+i) + " of " + numSegments + ": name=" + info.name + " docCount=" + info.docCount);
- segInfoStat.name = info.name;
- segInfoStat.docCount = info.docCount;
+ msg(" " + (1+i) + " of " + numSegments + ": name=" + info.info.name + " docCount=" + info.info.getDocCount());
+ segInfoStat.name = info.info.name;
+ segInfoStat.docCount = info.info.getDocCount();
- int toLoseDocCount = info.docCount;
+ int toLoseDocCount = info.info.getDocCount();
SegmentReader reader = null;
try {
- final Codec codec = info.getCodec();
+ final Codec codec = info.info.getCodec();
msg(" codec=" + codec);
segInfoStat.codec = codec;
- msg(" compound=" + info.getUseCompoundFile());
- segInfoStat.compound = info.getUseCompoundFile();
- msg(" hasProx=" + info.getHasProx());
- segInfoStat.hasProx = info.getHasProx();
+ msg(" compound=" + info.info.getUseCompoundFile());
+ segInfoStat.compound = info.info.getUseCompoundFile();
msg(" numFiles=" + info.files().size());
segInfoStat.numFiles = info.files().size();
segInfoStat.sizeMB = info.sizeInBytes()/(1024.*1024.);
msg(" size (MB)=" + nf.format(segInfoStat.sizeMB));
- Map<String,String> diagnostics = info.getDiagnostics();
+ Map<String,String> diagnostics = info.info.getDiagnostics();
segInfoStat.diagnostics = diagnostics;
if (diagnostics.size() > 0) {
msg(" diagnostics = " + diagnostics);
}
- final int docStoreOffset = info.getDocStoreOffset();
- if (docStoreOffset != -1) {
- msg(" docStoreOffset=" + docStoreOffset);
- segInfoStat.docStoreOffset = docStoreOffset;
- msg(" docStoreSegment=" + info.getDocStoreSegment());
- segInfoStat.docStoreSegment = info.getDocStoreSegment();
- msg(" docStoreIsCompoundFile=" + info.getDocStoreIsCompoundFile());
- segInfoStat.docStoreCompoundFile = info.getDocStoreIsCompoundFile();
- }
+ // TODO: we could append the info attributes() to the msg?
if (info.hasDeletions()) {
msg(" no deletions");
@@ -545,14 +509,14 @@ public class CheckIndex {
final int numDocs = reader.numDocs();
toLoseDocCount = numDocs;
if (reader.hasDeletions()) {
- if (reader.numDocs() != info.docCount - info.getDelCount()) {
- throw new RuntimeException("delete count mismatch: info=" + (info.docCount - info.getDelCount()) + " vs reader=" + reader.numDocs());
+ if (reader.numDocs() != info.info.getDocCount() - info.getDelCount()) {
+ throw new RuntimeException("delete count mismatch: info=" + (info.info.getDocCount() - info.getDelCount()) + " vs reader=" + reader.numDocs());
}
- if ((info.docCount-reader.numDocs()) > reader.maxDoc()) {
- throw new RuntimeException("too many deleted docs: maxDoc()=" + reader.maxDoc() + " vs del count=" + (info.docCount-reader.numDocs()));
+ if ((info.info.getDocCount()-reader.numDocs()) > reader.maxDoc()) {
+ throw new RuntimeException("too many deleted docs: maxDoc()=" + reader.maxDoc() + " vs del count=" + (info.info.getDocCount()-reader.numDocs()));
}
- if (info.docCount - numDocs != info.getDelCount()) {
- throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs reader=" + (info.docCount - numDocs));
+ if (info.info.getDocCount() - numDocs != info.getDelCount()) {
+ throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs reader=" + (info.info.getDocCount() - numDocs));
}
Bits liveDocs = reader.getLiveDocs();
if (liveDocs == null) {
@@ -569,11 +533,11 @@ public class CheckIndex {
}
}
- segInfoStat.numDeleted = info.docCount - numDocs;
+ segInfoStat.numDeleted = info.info.getDocCount() - numDocs;
msg("OK [" + (segInfoStat.numDeleted) + " deleted docs]");
} else {
if (info.getDelCount() != 0) {
- throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs reader=" + (info.docCount - numDocs));
+ throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs reader=" + (info.info.getDocCount() - numDocs));
}
Bits liveDocs = reader.getLiveDocs();
if (liveDocs != null) {
@@ -586,8 +550,9 @@ public class CheckIndex {
}
msg("OK");
}
- if (reader.maxDoc() != info.docCount)
- throw new RuntimeException("SegmentReader.maxDoc() " + reader.maxDoc() + " != SegmentInfos.docCount " + info.docCount);
+ if (reader.maxDoc() != info.info.getDocCount()) {
+ throw new RuntimeException("SegmentReader.maxDoc() " + reader.maxDoc() + " != SegmentInfos.docCount " + info.info.getDocCount());
+ }
// Test getFieldInfos()
if (infoStream != null) {
@@ -609,7 +574,7 @@ public class CheckIndex {
// Test Term Vectors
segInfoStat.termVectorStatus = testTermVectors(fieldInfos, info, reader, nf);
- segInfoStat.docValuesStatus = testDocValues(info, reader);
+ segInfoStat.docValuesStatus = testDocValues(info, fieldInfos, reader);
// Rethrow the first exception we encountered
// This will cause stats for failed segments to be incremented properly
@@ -706,8 +671,7 @@ public class CheckIndex {
* checks Fields api is consistent with itself.
* searcher is optional, to verify with queries. Can be null.
*/
- // TODO: cutover term vectors to this!
- private Status.TermIndexStatus checkFields(Fields fields, Bits liveDocs, int maxDoc, FieldInfos fieldInfos, IndexSearcher searcher) throws IOException {
+ private Status.TermIndexStatus checkFields(Fields fields, Bits liveDocs, int maxDoc, FieldInfos fieldInfos, IndexSearcher searcher, boolean doPrint) throws IOException {
// TODO: we should probably return our own stats thing...?!
final Status.TermIndexStatus status = new Status.TermIndexStatus();
@@ -741,7 +705,7 @@ public class CheckIndex {
if (fi == null) {
throw new RuntimeException("fieldsEnum inconsistent with fieldInfos, no fieldInfos for: " + field);
}
- if (!fi.isIndexed) {
+ if (!fi.isIndexed()) {
throw new RuntimeException("fieldsEnum inconsistent with fieldInfos, isIndexed == false for: " + field);
}
@@ -1131,8 +1095,10 @@ public class CheckIndex {
if (status.termCount != uniqueTermCountAllFields) {
throw new RuntimeException("termCount mismatch " + uniqueTermCountAllFields + " vs " + (status.termCount));
}
-
- msg("OK [" + status.termCount + " terms; " + status.totFreq + " terms/docs pairs; " + status.totPos + " tokens]");
+
+ if (doPrint) {
+ msg("OK [" + status.termCount + " terms; " + status.totFreq + " terms/docs pairs; " + status.totPos + " tokens]");
+ }
if (verbose && status.blockTreeStats != null && infoStream != null && status.termCount > 0) {
for(Map.Entry<String,BlockTreeTermsReader.Stats> ent : status.blockTreeStats.entrySet()) {
@@ -1163,13 +1129,13 @@ public class CheckIndex {
}
final Fields fields = reader.fields();
- status = checkFields(fields, liveDocs, maxDoc, fieldInfos, is);
+ status = checkFields(fields, liveDocs, maxDoc, fieldInfos, is, true);
if (liveDocs != null) {
if (infoStream != null) {
infoStream.print(" test (ignoring deletes): terms, freq, prox...");
}
// TODO: can we make a IS that ignores all deletes?
- checkFields(fields, null, maxDoc, fieldInfos, null);
+ checkFields(fields, null, maxDoc, fieldInfos, null, true);
}
} catch (Throwable e) {
msg("ERROR: " + e);
@@ -1186,7 +1152,7 @@ public class CheckIndex {
/**
* Test stored fields for a segment.
*/
- private Status.StoredFieldStatus testStoredFields(SegmentInfo info, SegmentReader reader, NumberFormat format) {
+ private Status.StoredFieldStatus testStoredFields(SegmentInfoPerCommit info, SegmentReader reader, NumberFormat format) {
final Status.StoredFieldStatus status = new Status.StoredFieldStatus();
try {
@@ -1196,7 +1162,7 @@ public class CheckIndex {
// Scan stored fields for all documents
final Bits liveDocs = reader.getLiveDocs();
- for (int j = 0; j < info.docCount; ++j) {
+ for (int j = 0; j < info.info.getDocCount(); ++j) {
// Intentionally pull even deleted documents to
// make sure they too are not corrupt:
Document doc = reader.document(j);
@@ -1310,14 +1276,14 @@ public class CheckIndex {
}
}
- private Status.DocValuesStatus testDocValues(SegmentInfo info,
- SegmentReader reader) {
+ private Status.DocValuesStatus testDocValues(SegmentInfoPerCommit info,
+ FieldInfos fieldInfos,
+ SegmentReader reader) {
final Status.DocValuesStatus status = new Status.DocValuesStatus();
try {
if (infoStream != null) {
infoStream.print(" test: DocValues........");
}
- final FieldInfos fieldInfos = info.getFieldInfos();
for (FieldInfo fieldInfo : fieldInfos) {
if (fieldInfo.hasDocValues()) {
status.totalValueFields++;
@@ -1345,7 +1311,7 @@ public class CheckIndex {
/**
* Test term vectors for a segment.
*/
- private Status.TermVectorStatus testTermVectors(FieldInfos fieldInfos, SegmentInfo info, SegmentReader reader, NumberFormat format) {
+ private Status.TermVectorStatus testTermVectors(FieldInfos fieldInfos, SegmentInfoPerCommit info, SegmentReader reader, NumberFormat format) {
final Status.TermVectorStatus status = new Status.TermVectorStatus();
final Bits onlyDocIsDeleted = new FixedBitSet(1);
@@ -1375,7 +1341,7 @@ public class CheckIndex {
TermsEnum termsEnum = null;
TermsEnum postingsTermsEnum = null;
- for (int j = 0; j < info.docCount; ++j) {
+ for (int j = 0; j < info.info.getDocCount(); ++j) {
// Intentionally pull/visit (but don't count in
// stats) deleted documents to make sure they too
// are not corrupt:
@@ -1386,10 +1352,10 @@ public class CheckIndex {
if (tfv != null) {
// First run with no deletions:
- checkFields(tfv, null, 1, fieldInfos, null);
+ checkFields(tfv, null, 1, fieldInfos, null, false);
// Again, with the one doc deleted:
- checkFields(tfv, onlyDocIsDeleted, 1, fieldInfos, null);
+ checkFields(tfv, onlyDocIsDeleted, 1, fieldInfos, null, false);
// Only agg stats if the doc is live:
final boolean doStats = liveDocs == null || liveDocs.get(j);
@@ -1406,7 +1372,7 @@ public class CheckIndex {
// Make sure FieldInfo thinks this field is vector'd:
final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
- if (!fieldInfo.storeTermVector) {
+ if (!fieldInfo.hasVectors()) {
throw new RuntimeException("docID=" + j + " has term vectors for field=" + field + " but FieldInfo has storeTermVector=false");
}
@@ -1594,7 +1560,7 @@ public class CheckIndex {
if (result.partial)
throw new IllegalArgumentException("can only fix an index that was fully checked (this status checked a subset of segments)");
result.newSegments.changed();
- result.newSegments.commit(result.dir, codec);
+ result.newSegments.commit(result.dir);
}
private static boolean assertsOn;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java Mon May 28 19:28:10 2012
@@ -20,7 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
abstract class DocConsumer {
- abstract void processDocument(FieldInfos fieldInfos) throws IOException;
+ abstract void processDocument(FieldInfos.Builder fieldInfos) throws IOException;
abstract void finishDocument() throws IOException;
abstract void flush(final SegmentWriteState state) throws IOException;
abstract void abort();
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java Mon May 28 19:28:10 2012
@@ -23,7 +23,7 @@ import java.util.Map;
abstract class DocFieldConsumer {
/** Called when DocumentsWriterPerThread decides to create a new
* segment */
- abstract void flush(Map<FieldInfo, DocFieldConsumerPerField> fieldsToFlush, SegmentWriteState state) throws IOException;
+ abstract void flush(Map<String, DocFieldConsumerPerField> fieldsToFlush, SegmentWriteState state) throws IOException;
/** Called when an aborting exception is hit */
abstract void abort();
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java Mon May 28 19:28:10 2012
@@ -71,28 +71,28 @@ final class DocFieldProcessor extends Do
@Override
public void flush(SegmentWriteState state) throws IOException {
- Map<FieldInfo, DocFieldConsumerPerField> childFields = new HashMap<FieldInfo, DocFieldConsumerPerField>();
+ Map<String,DocFieldConsumerPerField> childFields = new HashMap<String,DocFieldConsumerPerField>();
Collection<DocFieldConsumerPerField> fields = fields();
for (DocFieldConsumerPerField f : fields) {
- childFields.put(f.getFieldInfo(), f);
+ childFields.put(f.getFieldInfo().name, f);
}
fieldsWriter.flush(state);
consumer.flush(childFields, state);
for (DocValuesConsumerAndDocID consumer : docValues.values()) {
- consumer.docValuesConsumer.finish(state.numDocs);
+ consumer.docValuesConsumer.finish(state.segmentInfo.getDocCount());
}
+
+ // close perDocConsumer during flush to ensure all files are flushed due to PerCodec CFS
+ IOUtils.close(perDocConsumer);
// Important to save after asking consumer to flush so
// consumer can alter the FieldInfo* if necessary. EG,
// FreqProxTermsWriter does this with
// FieldInfo.storePayload.
FieldInfosWriter infosWriter = codec.fieldInfosFormat().getFieldInfosWriter();
- infosWriter.write(state.directory, state.segmentName, state.fieldInfos, IOContext.DEFAULT);
-
- // close perDocConsumer during flush to ensure all files are flushed due to PerCodec CFS
- IOUtils.close(perDocConsumer);
+ infosWriter.write(state.directory, state.segmentInfo.name, state.fieldInfos, IOContext.DEFAULT);
}
@Override
@@ -203,7 +203,7 @@ final class DocFieldProcessor extends Do
}
@Override
- public void processDocument(FieldInfos fieldInfos) throws IOException {
+ public void processDocument(FieldInfos.Builder fieldInfos) throws IOException {
consumer.startDocument();
fieldsWriter.startDocument();
@@ -342,7 +342,8 @@ final class DocFieldProcessor extends Do
}
}
DocValuesConsumer docValuesConsumer = perDocConsumer.addValuesField(valueType, fieldInfo);
- fieldInfo.setDocValuesType(valueType, false);
+ assert fieldInfo.getDocValuesType() == null || fieldInfo.getDocValuesType() == valueType;
+ fieldInfo.setDocValuesType(valueType);
docValuesConsumerAndDocID = new DocValuesConsumerAndDocID(docValuesConsumer);
docValuesConsumerAndDocID.docID = docState.docID;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverter.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverter.java Mon May 28 19:28:10 2012
@@ -39,12 +39,12 @@ final class DocInverter extends DocField
}
@Override
- void flush(Map<FieldInfo, DocFieldConsumerPerField> fieldsToFlush, SegmentWriteState state) throws IOException {
+ void flush(Map<String, DocFieldConsumerPerField> fieldsToFlush, SegmentWriteState state) throws IOException {
- Map<FieldInfo, InvertedDocConsumerPerField> childFieldsToFlush = new HashMap<FieldInfo, InvertedDocConsumerPerField>();
- Map<FieldInfo, InvertedDocEndConsumerPerField> endChildFieldsToFlush = new HashMap<FieldInfo, InvertedDocEndConsumerPerField>();
+ Map<String, InvertedDocConsumerPerField> childFieldsToFlush = new HashMap<String, InvertedDocConsumerPerField>();
+ Map<String, InvertedDocEndConsumerPerField> endChildFieldsToFlush = new HashMap<String, InvertedDocEndConsumerPerField>();
- for (Map.Entry<FieldInfo, DocFieldConsumerPerField> fieldToFlush : fieldsToFlush.entrySet()) {
+ for (Map.Entry<String, DocFieldConsumerPerField> fieldToFlush : fieldsToFlush.entrySet()) {
DocInverterPerField perField = (DocInverterPerField) fieldToFlush.getValue();
childFieldsToFlush.put(fieldToFlush.getKey(), perField.consumer);
endChildFieldsToFlush.put(fieldToFlush.getKey(), perField.endConsumer);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java Mon May 28 19:28:10 2012
@@ -28,7 +28,7 @@ import org.apache.lucene.index.Documents
import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment;
import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain;
import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState;
-import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
+import org.apache.lucene.index.FieldInfos.FieldNumbers;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.AlreadyClosedException;
@@ -133,7 +133,7 @@ final class DocumentsWriter {
final DocumentsWriterFlushControl flushControl;
final Codec codec;
- DocumentsWriter(Codec codec, IndexWriterConfig config, Directory directory, IndexWriter writer, FieldNumberBiMap globalFieldNumbers,
+ DocumentsWriter(Codec codec, IndexWriterConfig config, Directory directory, IndexWriter writer, FieldNumbers globalFieldNumbers,
BufferedDeletesStream bufferedDeletesStream) throws IOException {
this.codec = codec;
this.directory = directory;
@@ -494,7 +494,8 @@ final class DocumentsWriter {
private void publishFlushedSegment(FlushedSegment newSegment, FrozenBufferedDeletes globalPacket)
throws IOException {
assert newSegment != null;
- final SegmentInfo segInfo = indexWriter.prepareFlushedSegment(newSegment);
+ assert newSegment.segmentInfo != null;
+ final SegmentInfoPerCommit segInfo = indexWriter.prepareFlushedSegment(newSegment);
final BufferedDeletes deletes = newSegment.segmentDeletes;
if (infoStream.isEnabled("DW")) {
infoStream.message("DW", Thread.currentThread().getName() + ": publishFlushedSegment seg-private deletes=" + deletes);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java Mon May 28 19:28:10 2012
@@ -170,7 +170,7 @@ class DocumentsWriterFlushQueue {
protected abstract boolean canPublish();
}
- static final class GlobalDeletesTicket extends FlushTicket{
+ static final class GlobalDeletesTicket extends FlushTicket {
protected GlobalDeletesTicket(FrozenBufferedDeletes frozenDeletes) {
super(frozenDeletes);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Mon May 28 19:28:10 2012
@@ -17,11 +17,9 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import static org.apache.lucene.util.ByteBlockPool.BYTE_BLOCK_MASK;
-import static org.apache.lucene.util.ByteBlockPool.BYTE_BLOCK_SIZE;
-
import java.io.IOException;
import java.text.NumberFormat;
+import java.util.HashSet;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.codecs.Codec;
@@ -30,12 +28,17 @@ import org.apache.lucene.search.similari
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.Counter;
+import org.apache.lucene.store.TrackingDirectoryWrapper;
import org.apache.lucene.util.ByteBlockPool.Allocator;
import org.apache.lucene.util.ByteBlockPool.DirectTrackingAllocator;
+import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.Counter;
import org.apache.lucene.util.InfoStream;
-import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.MutableBits;
+import org.apache.lucene.util.RamUsageEstimator;
+
+import static org.apache.lucene.util.ByteBlockPool.BYTE_BLOCK_MASK;
+import static org.apache.lucene.util.ByteBlockPool.BYTE_BLOCK_SIZE;
class DocumentsWriterPerThread {
@@ -112,14 +115,16 @@ class DocumentsWriterPerThread {
}
static class FlushedSegment {
- final SegmentInfo segmentInfo;
+ final SegmentInfoPerCommit segmentInfo;
+ final FieldInfos fieldInfos;
final BufferedDeletes segmentDeletes;
final MutableBits liveDocs;
final int delCount;
- private FlushedSegment(SegmentInfo segmentInfo,
+ private FlushedSegment(SegmentInfoPerCommit segmentInfo, FieldInfos fieldInfos,
BufferedDeletes segmentDeletes, MutableBits liveDocs, int delCount) {
this.segmentInfo = segmentInfo;
+ this.fieldInfos = fieldInfos;
this.segmentDeletes = segmentDeletes;
this.liveDocs = liveDocs;
this.delCount = delCount;
@@ -157,7 +162,8 @@ class DocumentsWriterPerThread {
final DocumentsWriter parent;
final Codec codec;
final IndexWriter writer;
- final Directory directory;
+ final TrackingDirectoryWrapper directory;
+ final Directory directoryOrig;
final DocState docState;
final DocConsumer consumer;
final Counter bytesUsed;
@@ -165,11 +171,11 @@ class DocumentsWriterPerThread {
SegmentWriteState flushState;
//Deletes for our still-in-RAM (to be flushed next) segment
BufferedDeletes pendingDeletes;
- String segment; // Current segment we are working on
+ SegmentInfo segmentInfo; // Current segment we are working on
boolean aborting = false; // True if an abort is pending
boolean hasAborted = false; // True if the last exception throws by #updateDocument was aborting
- private FieldInfos fieldInfos;
+ private FieldInfos.Builder fieldInfos;
private final InfoStream infoStream;
private int numDocsInRAM;
private int flushedDocCount;
@@ -180,8 +186,9 @@ class DocumentsWriterPerThread {
public DocumentsWriterPerThread(Directory directory, DocumentsWriter parent,
- FieldInfos fieldInfos, IndexingChain indexingChain) {
- this.directory = directory;
+ FieldInfos.Builder fieldInfos, IndexingChain indexingChain) {
+ this.directoryOrig = directory;
+ this.directory = new TrackingDirectoryWrapper(directory);
this.parent = parent;
this.fieldInfos = fieldInfos;
this.writer = parent.indexWriter;
@@ -196,8 +203,8 @@ class DocumentsWriterPerThread {
initialize();
}
- public DocumentsWriterPerThread(DocumentsWriterPerThread other, FieldInfos fieldInfos) {
- this(other.directory, other.parent, fieldInfos, other.parent.chain);
+ public DocumentsWriterPerThread(DocumentsWriterPerThread other, FieldInfos.Builder fieldInfos) {
+ this(other.directoryOrig, other.parent, fieldInfos, other.parent.chain);
}
void initialize() {
@@ -223,17 +230,11 @@ class DocumentsWriterPerThread {
docState.doc = doc;
docState.analyzer = analyzer;
docState.docID = numDocsInRAM;
- if (segment == null) {
- // this call is synchronized on IndexWriter.segmentInfos
- segment = writer.newSegmentName();
- assert numDocsInRAM == 0;
- if (INFO_VERBOSE && infoStream.isEnabled("DWPT")) {
- infoStream.message("DWPT", Thread.currentThread().getName() + " init seg=" + segment + " delQueue=" + deleteQueue);
- }
-
+ if (segmentInfo == null) {
+ initSegmentInfo();
}
if (INFO_VERBOSE && infoStream.isEnabled("DWPT")) {
- infoStream.message("DWPT", Thread.currentThread().getName() + " update delTerm=" + delTerm + " docID=" + docState.docID + " seg=" + segment);
+ infoStream.message("DWPT", Thread.currentThread().getName() + " update delTerm=" + delTerm + " docID=" + docState.docID + " seg=" + segmentInfo.name);
}
boolean success = false;
try {
@@ -265,21 +266,26 @@ class DocumentsWriterPerThread {
}
finishDocument(delTerm);
}
+
+ private void initSegmentInfo() {
+ String segment = writer.newSegmentName();
+ segmentInfo = new SegmentInfo(directoryOrig, Constants.LUCENE_MAIN_VERSION, segment, -1,
+ false, codec, null, null);
+ assert numDocsInRAM == 0;
+ if (INFO_VERBOSE && infoStream.isEnabled("DWPT")) {
+ infoStream.message("DWPT", Thread.currentThread().getName() + " init seg=" + segment + " delQueue=" + deleteQueue);
+ }
+ }
public int updateDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs, Analyzer analyzer, Term delTerm) throws IOException {
assert writer.testPoint("DocumentsWriterPerThread addDocuments start");
assert deleteQueue != null;
docState.analyzer = analyzer;
- if (segment == null) {
- // this call is synchronized on IndexWriter.segmentInfos
- segment = writer.newSegmentName();
- assert numDocsInRAM == 0;
- if (INFO_VERBOSE && infoStream.isEnabled("DWPT")) {
- infoStream.message("DWPT", Thread.currentThread().getName() + " init seg=" + segment + " delQueue=" + deleteQueue);
- }
+ if (segmentInfo == null) {
+ initSegmentInfo();
}
if (INFO_VERBOSE && infoStream.isEnabled("DWPT")) {
- infoStream.message("DWPT", Thread.currentThread().getName() + " update delTerm=" + delTerm + " docID=" + docState.docID + " seg=" + segment);
+ infoStream.message("DWPT", Thread.currentThread().getName() + " update delTerm=" + delTerm + " docID=" + docState.docID + " seg=" + segmentInfo.name);
}
int docCount = 0;
try {
@@ -405,15 +411,12 @@ class DocumentsWriterPerThread {
return numDocsInRAM;
}
- Codec getCodec() {
- return flushState.codec;
- }
-
/** Reset after a flush */
private void doAfterFlush() throws IOException {
- segment = null;
+ segmentInfo = null;
consumer.doAfterFlush();
- fieldInfos = FieldInfos.from(fieldInfos);
+ directory.getCreatedFiles().clear();
+ fieldInfos = new FieldInfos.Builder(fieldInfos.globalFieldNumbers);
parent.subtractFlushedNumDocs(numDocsInRAM);
numDocsInRAM = 0;
}
@@ -441,10 +444,12 @@ class DocumentsWriterPerThread {
FlushedSegment flush() throws IOException {
assert numDocsInRAM > 0;
assert deleteSlice == null : "all deletes must be applied in prepareFlush";
- flushState = new SegmentWriteState(infoStream, directory, segment, fieldInfos,
- numDocsInRAM, writer.getConfig().getTermIndexInterval(),
- codec, pendingDeletes, new IOContext(new FlushInfo(numDocsInRAM, bytesUsed())));
+ segmentInfo.setDocCount(numDocsInRAM);
+ flushState = new SegmentWriteState(infoStream, directory, segmentInfo, fieldInfos.finish(),
+ writer.getConfig().getTermIndexInterval(),
+ pendingDeletes, new IOContext(new FlushInfo(numDocsInRAM, bytesUsed())));
final double startMBUsed = parent.flushControl.netBytes() / 1024. / 1024.;
+
// Apply delete-by-docID now (delete-byDocID only
// happens when an exception is hit processing that
// doc, eg if analyzer has some problem w/ the text):
@@ -459,7 +464,7 @@ class DocumentsWriterPerThread {
}
if (infoStream.isEnabled("DWPT")) {
- infoStream.message("DWPT", "flush postings as segment " + flushState.segmentName + " numDocs=" + numDocsInRAM);
+ infoStream.message("DWPT", "flush postings as segment " + flushState.segmentInfo.name + " numDocs=" + numDocsInRAM);
}
if (aborting) {
@@ -474,14 +479,22 @@ class DocumentsWriterPerThread {
try {
consumer.flush(flushState);
pendingDeletes.terms.clear();
- final SegmentInfo newSegment = new SegmentInfo(segment, flushState.numDocs, directory, false, flushState.codec, fieldInfos.asReadOnly());
+ segmentInfo.setFiles(new HashSet<String>(directory.getCreatedFiles()));
+
+ final SegmentInfoPerCommit segmentInfoPerCommit = new SegmentInfoPerCommit(segmentInfo, 0, -1L);
if (infoStream.isEnabled("DWPT")) {
- infoStream.message("DWPT", "new segment has " + (flushState.liveDocs == null ? 0 : (flushState.numDocs - flushState.delCountOnFlush)) + " deleted docs");
- infoStream.message("DWPT", "new segment has " + (newSegment.getHasVectors() ? "vectors" : "no vectors"));
- infoStream.message("DWPT", "flushedFiles=" + newSegment.files());
- infoStream.message("DWPT", "flushed codec=" + newSegment.getCodec());
+ infoStream.message("DWPT", "new segment has " + (flushState.liveDocs == null ? 0 : (flushState.segmentInfo.getDocCount() - flushState.delCountOnFlush)) + " deleted docs");
+ infoStream.message("DWPT", "new segment has " +
+ (flushState.fieldInfos.hasVectors() ? "vectors" : "no vectors") + "; " +
+ (flushState.fieldInfos.hasNorms() ? "norms" : "no norms") + "; " +
+ (flushState.fieldInfos.hasDocValues() ? "docValues" : "no docValues") + "; " +
+ (flushState.fieldInfos.hasProx() ? "prox" : "no prox") + "; " +
+ (flushState.fieldInfos.hasFreq() ? "freqs" : "no freqs"));
+ infoStream.message("DWPT", "flushedFiles=" + segmentInfoPerCommit.files());
+ infoStream.message("DWPT", "flushed codec=" + codec);
}
- flushedDocCount += flushState.numDocs;
+
+ flushedDocCount += flushState.segmentInfo.getDocCount();
final BufferedDeletes segmentDeletes;
if (pendingDeletes.queries.isEmpty()) {
@@ -493,21 +506,26 @@ class DocumentsWriterPerThread {
}
if (infoStream.isEnabled("DWPT")) {
- final double newSegmentSize = newSegment.sizeInBytes()/1024./1024.;
- infoStream.message("DWPT", "flushed: segment=" + newSegment +
+ final double newSegmentSize = segmentInfo.sizeInBytes()/1024./1024.;
+ infoStream.message("DWPT", "flushed: segment=" + segmentInfo.name +
" ramUsed=" + nf.format(startMBUsed) + " MB" +
" newFlushedSize(includes docstores)=" + nf.format(newSegmentSize) + " MB" +
" docs/MB=" + nf.format(flushedDocCount / newSegmentSize));
}
+
+ assert segmentInfo != null;
+
+ FlushedSegment fs = new FlushedSegment(segmentInfoPerCommit, flushState.fieldInfos,
+ segmentDeletes, flushState.liveDocs, flushState.delCountOnFlush);
doAfterFlush();
success = true;
- return new FlushedSegment(newSegment, segmentDeletes, flushState.liveDocs, flushState.delCountOnFlush);
+ return fs;
} finally {
if (!success) {
- if (segment != null) {
+ if (segmentInfo != null) {
synchronized(parent.indexWriter) {
- parent.indexWriter.deleter.refresh(segment);
+ parent.indexWriter.deleter.refresh(segmentInfo.name);
}
}
abort();
@@ -515,9 +533,9 @@ class DocumentsWriterPerThread {
}
}
- /** Get current segment name we are writing. */
- String getSegment() {
- return segment;
+ /** Get current segment info we are writing. */
+ SegmentInfo getSegmentInfo() {
+ return segmentInfo;
}
long bytesUsed() {
@@ -550,14 +568,14 @@ class DocumentsWriterPerThread {
}
PerDocWriteState newPerDocWriteState(String segmentSuffix) {
- assert segment != null;
- return new PerDocWriteState(infoStream, directory, segment, fieldInfos, bytesUsed, segmentSuffix, IOContext.DEFAULT);
+ assert segmentInfo != null;
+ return new PerDocWriteState(infoStream, directory, segmentInfo, bytesUsed, segmentSuffix, IOContext.DEFAULT);
}
@Override
public String toString() {
return "DocumentsWriterPerThread [pendingDeletes=" + pendingDeletes
- + ", segment=" + segment + ", aborting=" + aborting + ", numDocsInRAM="
+ + ", segment=" + (segmentInfo != null ? segmentInfo.name : "null") + ", aborting=" + aborting + ", numDocsInRAM="
+ numDocsInRAM + ", deleteQueue=" + deleteQueue + "]";
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java Mon May 28 19:28:10 2012
@@ -18,7 +18,7 @@ package org.apache.lucene.index;
import java.util.concurrent.locks.ReentrantLock;
-import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
+import org.apache.lucene.index.FieldInfos.FieldNumbers;
import org.apache.lucene.util.SetOnce;
/**
@@ -121,7 +121,7 @@ abstract class DocumentsWriterPerThreadP
private final ThreadState[] threadStates;
private volatile int numThreadStatesActive;
- private final SetOnce<FieldNumberBiMap> globalFieldMap = new SetOnce<FieldNumberBiMap>();
+ private final SetOnce<FieldNumbers> globalFieldMap = new SetOnce<FieldNumbers>();
private final SetOnce<DocumentsWriter> documentsWriter = new SetOnce<DocumentsWriter>();
/**
@@ -135,11 +135,11 @@ abstract class DocumentsWriterPerThreadP
numThreadStatesActive = 0;
}
- void initialize(DocumentsWriter documentsWriter, FieldNumberBiMap globalFieldMap, IndexWriterConfig config) {
+ void initialize(DocumentsWriter documentsWriter, FieldNumbers globalFieldMap, IndexWriterConfig config) {
this.documentsWriter.set(documentsWriter); // thread pool is bound to DW
this.globalFieldMap.set(globalFieldMap);
for (int i = 0; i < threadStates.length; i++) {
- final FieldInfos infos = new FieldInfos(globalFieldMap);
+ final FieldInfos.Builder infos = new FieldInfos.Builder(globalFieldMap);
threadStates[i] = new ThreadState(new DocumentsWriterPerThread(documentsWriter.directory, documentsWriter, infos, documentsWriter.chain));
}
}
@@ -228,7 +228,7 @@ abstract class DocumentsWriterPerThreadP
assert globalFieldMap.get() != null;
final DocumentsWriterPerThread dwpt = threadState.dwpt;
if (!closed) {
- final FieldInfos infos = new FieldInfos(globalFieldMap.get());
+ final FieldInfos.Builder infos = new FieldInfos.Builder(globalFieldMap.get());
final DocumentsWriterPerThread newDwpt = new DocumentsWriterPerThread(dwpt, infos);
newDwpt.initialize();
threadState.resetWriter(newDwpt);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java Mon May 28 19:28:10 2012
@@ -1,8 +1,5 @@
package org.apache.lucene.index;
-import org.apache.lucene.index.DocValues.Type;
-
-
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -20,6 +17,11 @@ import org.apache.lucene.index.DocValues
* limitations under the License.
*/
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.index.DocValues.Type;
+
/**
* Access to the Fieldable Info file that describes document fields and whether or
* not they are indexed. Each segment has a separate Fieldable Info file. Objects
@@ -27,20 +29,23 @@ import org.apache.lucene.index.DocValues
* be adding documents at a time, with no other reader or writer threads
* accessing this object.
**/
+
public final class FieldInfo {
public final String name;
public final int number;
- public boolean isIndexed;
+ private boolean indexed;
private DocValues.Type docValueType;
// True if any document indexed term vectors
- public boolean storeTermVector;
+ private boolean storeTermVector;
private DocValues.Type normType;
- public boolean omitNorms; // omit norms associated with indexed fields
- public IndexOptions indexOptions;
- public boolean storePayloads; // whether this field stores payloads together with term positions
+ private boolean omitNorms; // omit norms associated with indexed fields
+ private IndexOptions indexOptions;
+ private boolean storePayloads; // whether this field stores payloads together with term positions
+
+ private Map<String,String> attributes;
/**
* Controls how much information is stored in the postings lists.
@@ -64,13 +69,13 @@ public final class FieldInfo {
/**
* @lucene.experimental
*/
- public FieldInfo(String name, boolean isIndexed, int number, boolean storeTermVector,
- boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues, DocValues.Type normsType) {
+ public FieldInfo(String name, boolean indexed, int number, boolean storeTermVector,
+ boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues, DocValues.Type normsType, Map<String,String> attributes) {
this.name = name;
- this.isIndexed = isIndexed;
+ this.indexed = indexed;
this.number = number;
this.docValueType = docValues;
- if (isIndexed) {
+ if (indexed) {
this.storeTermVector = storeTermVector;
this.storePayloads = storePayloads;
this.omitNorms = omitNorms;
@@ -83,22 +88,38 @@ public final class FieldInfo {
this.indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
this.normType = null;
}
+ this.attributes = attributes;
+ assert checkConsistency();
assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !storePayloads;
}
-
- @Override
- public FieldInfo clone() {
- return new FieldInfo(name, isIndexed, number, storeTermVector,
- omitNorms, storePayloads, indexOptions, docValueType, normType);
+
+ private boolean checkConsistency() {
+ if (!indexed) {
+ assert !storeTermVector;
+ assert !storePayloads;
+ assert !omitNorms;
+ assert normType == null;
+ assert indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
+ } else {
+ assert indexOptions != null;
+ if (omitNorms) {
+ assert normType == null;
+ }
+ }
+
+ // Cannot store payloads unless positions are indexed:
+ assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !this.storePayloads;
+
+ return true;
}
// should only be called by FieldInfos#addOrUpdate
- void update(boolean isIndexed, boolean storeTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions) {
+ void update(boolean indexed, boolean storeTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions) {
- if (this.isIndexed != isIndexed) {
- this.isIndexed = true; // once indexed, always index
+ if (this.indexed != indexed) {
+ this.indexed = true; // once indexed, always index
}
- if (isIndexed) { // if updated field data is not for indexing, leave the updates out
+ if (indexed) { // if updated field data is not for indexing, leave the updates out
if (this.storeTermVector != storeTermVector) {
this.storeTermVector = true; // once vector, always vector
}
@@ -107,6 +128,7 @@ public final class FieldInfo {
}
if (this.omitNorms != omitNorms) {
this.omitNorms = true; // if one require omitNorms at least once, it remains off for life
+ this.normType = null;
}
if (this.indexOptions != indexOptions) {
// downgrade
@@ -118,14 +140,17 @@ public final class FieldInfo {
}
}
assert this.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !this.storePayloads;
+ assert checkConsistency();
}
- void setDocValuesType(DocValues.Type type, boolean force) {
- if (docValueType == null || force) {
- docValueType = type;
- } else if (type != docValueType) {
- throw new IllegalArgumentException("DocValues type already set to " + docValueType + " but was: " + type);
- }
+ void setDocValuesType(DocValues.Type type) {
+ docValueType = type;
+ assert checkConsistency();
+ }
+
+ /** @return IndexOptions for the field */
+ public IndexOptions getIndexOptions() {
+ return indexOptions;
}
/**
@@ -149,22 +174,27 @@ public final class FieldInfo {
return normType;
}
- public void setStoreTermVectors() {
+ void setStoreTermVectors() {
storeTermVector = true;
+ assert checkConsistency();
}
-
- public void setNormValueType(Type type, boolean force) {
- if (normType == null || force) {
- normType = type;
- } else if (type != normType) {
- throw new IllegalArgumentException("Norm type already set to " + normType);
+
+ void setStorePayloads() {
+ if (indexed && indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) {
+ storePayloads = true;
}
+ assert checkConsistency();
+ }
+
+ void setNormValueType(Type type) {
+ normType = type;
+ assert checkConsistency();
}
/**
* @return true if norms are explicitly omitted for this field
*/
- public boolean omitNorms() {
+ public boolean omitsNorms() {
return omitNorms;
}
@@ -172,7 +202,62 @@ public final class FieldInfo {
* @return true if this field actually has any norms.
*/
public boolean hasNorms() {
- return isIndexed && !omitNorms && normType != null;
+ return normType != null;
+ }
+
+ /**
+ * @return true if this field is indexed.
+ */
+ public boolean isIndexed() {
+ return indexed;
+ }
+
+ /**
+ * @return true if any payloads exist for this field.
+ */
+ public boolean hasPayloads() {
+ return storePayloads;
}
+ /**
+ * @return true if any term vectors exist for this field.
+ */
+ public boolean hasVectors() {
+ return storeTermVector;
+ }
+
+ /**
+ * Get a codec attribute value, or null if it does not exist
+ */
+ public String getAttribute(String key) {
+ if (attributes == null) {
+ return null;
+ } else {
+ return attributes.get(key);
+ }
+ }
+
+ /**
+ * Puts a codec attribute value.
+ * <p>
+ * This is a key-value mapping for the field that the codec can use
+ * to store additional metadata, and will be available to the codec
+ * when reading the segment via {@link #getAttribute(String)}
+ * <p>
+ * If a value already exists for the field, it will be replaced with
+ * the new value.
+ */
+ public String putAttribute(String key, String value) {
+ if (attributes == null) {
+ attributes = new HashMap<String,String>();
+ }
+ return attributes.put(key, value);
+ }
+
+ /**
+ * @return internal codec attributes map. May be null if no mappings exist.
+ */
+ public Map<String,String> attributes() {
+ return attributes;
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java Mon May 28 19:28:10 2012
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@@ -30,14 +31,119 @@ import org.apache.lucene.index.FieldInfo
* Collection of {@link FieldInfo}s (accessible by number or by name).
* @lucene.experimental
*/
-public final class FieldInfos implements Iterable<FieldInfo> {
- static final class FieldNumberBiMap {
+public class FieldInfos implements Iterable<FieldInfo> {
+ private final boolean hasFreq;
+ private final boolean hasProx;
+ private final boolean hasVectors;
+ private final boolean hasNorms;
+ private final boolean hasDocValues;
+
+ private final SortedMap<Integer,FieldInfo> byNumber = new TreeMap<Integer,FieldInfo>();
+ private final HashMap<String,FieldInfo> byName = new HashMap<String,FieldInfo>();
+ private final Collection<FieldInfo> values; // for an unmodifiable iterator
+
+ public FieldInfos(FieldInfo[] infos) {
+ boolean hasVectors = false;
+ boolean hasProx = false;
+ boolean hasFreq = false;
+ boolean hasNorms = false;
+ boolean hasDocValues = false;
+
+ for (FieldInfo info : infos) {
+ assert !byNumber.containsKey(info.number);
+ byNumber.put(info.number, info);
+ assert !byName.containsKey(info.name);
+ byName.put(info.name, info);
+
+ hasVectors |= info.hasVectors();
+ hasProx |= info.isIndexed() && info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ hasFreq |= info.isIndexed() && info.getIndexOptions() != IndexOptions.DOCS_ONLY;
+ hasNorms |= info.hasNorms();
+ hasDocValues |= info.hasDocValues();
+ }
+
+ this.hasVectors = hasVectors;
+ this.hasProx = hasProx;
+ this.hasFreq = hasFreq;
+ this.hasNorms = hasNorms;
+ this.hasDocValues = hasDocValues;
+ this.values = Collections.unmodifiableCollection(byNumber.values());
+ }
+
+ /** Returns true if any fields have freqs */
+ public boolean hasFreq() {
+ return hasFreq;
+ }
+
+ /** Returns true if any fields have positions */
+ public boolean hasProx() {
+ return hasProx;
+ }
+
+ /**
+ * @return true if at least one field has any vectors
+ */
+ public boolean hasVectors() {
+ return hasVectors;
+ }
+
+ /**
+ * @return true if at least one field has any norms
+ */
+ public boolean hasNorms() {
+ return hasNorms;
+ }
+
+ /**
+ * @return true if at least one field has doc values
+ */
+ public boolean hasDocValues() {
+ return hasDocValues;
+ }
+
+ /**
+ * @return number of fields
+ */
+ public int size() {
+ assert byNumber.size() == byName.size();
+ return byNumber.size();
+ }
+
+ /**
+ * Returns an iterator over all the fieldinfo objects present,
+ * ordered by ascending field number
+ */
+ // TODO: what happens if in fact a different order is used?
+ public Iterator<FieldInfo> iterator() {
+ return values.iterator();
+ }
+
+ /**
+ * Return the fieldinfo object referenced by the field name
+ * @return the FieldInfo object or null when the given fieldName
+ * doesn't exist.
+ */
+ public FieldInfo fieldInfo(String fieldName) {
+ return byName.get(fieldName);
+ }
+
+ /**
+ * Return the fieldinfo object referenced by the fieldNumber.
+ * @param fieldNumber
+ * @return the FieldInfo object or null when the given fieldNumber
+ * doesn't exist.
+ */
+ public FieldInfo fieldInfo(int fieldNumber) {
+ return (fieldNumber >= 0) ? byNumber.get(fieldNumber) : null;
+ }
+
+ static final class FieldNumbers {
private final Map<Integer,String> numberToName;
private final Map<String,Integer> nameToNumber;
private int lowestUnassignedFieldNumber = -1;
- FieldNumberBiMap() {
+ FieldNumbers() {
this.nameToNumber = new HashMap<String, Integer>();
this.numberToName = new HashMap<Integer, String>();
}
@@ -54,8 +160,8 @@ public final class FieldInfos implements
final Integer preferredBoxed = Integer.valueOf(preferredFieldNumber);
if (preferredFieldNumber != -1 && !numberToName.containsKey(preferredBoxed)) {
- // cool - we can use this number globally
- fieldNumber = preferredBoxed;
+ // cool - we can use this number globally
+ fieldNumber = preferredBoxed;
} else {
// find a new FieldNumber
while (numberToName.containsKey(++lowestUnassignedFieldNumber)) {
@@ -66,7 +172,6 @@ public final class FieldInfos implements
numberToName.put(fieldNumber, fieldName);
nameToNumber.put(fieldName, fieldNumber);
-
}
return fieldNumber.intValue();
@@ -93,386 +198,115 @@ public final class FieldInfos implements
}
}
- private final SortedMap<Integer,FieldInfo> byNumber = new TreeMap<Integer,FieldInfo>();
- private final HashMap<String,FieldInfo> byName = new HashMap<String,FieldInfo>();
- private final FieldNumberBiMap globalFieldNumbers;
-
- private boolean hasFreq; // only set if readonly
- private boolean hasProx; // only set if readonly
- private boolean hasVectors; // only set if readonly
- private long version; // internal use to track changes
-
- /**
- * Creates a new read-only FieldInfos: only public to be accessible
- * from the codecs package
- *
- * @lucene.internal
- */
- public FieldInfos(FieldInfo[] infos, boolean hasFreq, boolean hasProx, boolean hasVectors) {
- this(null);
- this.hasFreq = hasFreq;
- this.hasProx = hasProx;
- this.hasVectors = hasVectors;
- for (FieldInfo info : infos) {
- putInternal(info);
- }
- }
-
- public FieldInfos() {
- this(new FieldNumberBiMap());
- }
+ static final class Builder {
+ private final HashMap<String,FieldInfo> byName = new HashMap<String,FieldInfo>();
+ final FieldNumbers globalFieldNumbers;
- public void add(FieldInfos other) {
- for(FieldInfo fieldInfo : other){
- add(fieldInfo);
+ Builder() {
+ this(new FieldNumbers());
}
- }
-
- /**
- * Creates a new FieldInfos instance with the given {@link FieldNumberBiMap}.
- * If the {@link FieldNumberBiMap} is <code>null</code> this instance will be read-only.
- * @see #isReadOnly()
- */
- FieldInfos(FieldNumberBiMap globalFieldNumbers) {
- this.globalFieldNumbers = globalFieldNumbers;
- }
-
- /**
- * adds the given field to this FieldInfos name / number mapping. The given FI
- * must be present in the global field number mapping before this method it
- * called
- */
- private void putInternal(FieldInfo fi) {
- assert !byNumber.containsKey(fi.number);
- assert !byName.containsKey(fi.name);
- assert globalFieldNumbers == null || globalFieldNumbers.containsConsistent(Integer.valueOf(fi.number), fi.name);
- byNumber.put(fi.number, fi);
- byName.put(fi.name, fi);
- }
-
- private int nextFieldNumber(String name, int preferredFieldNumber) {
- // get a global number for this field
- final int fieldNumber = globalFieldNumbers.addOrGet(name,
- preferredFieldNumber);
- assert byNumber.get(fieldNumber) == null : "field number " + fieldNumber
- + " already taken";
- return fieldNumber;
- }
-
- /**
- * Returns a deep clone of this FieldInfos instance.
- */
- @Override
- synchronized public FieldInfos clone() {
- FieldInfos fis = new FieldInfos(globalFieldNumbers);
- fis.hasFreq = hasFreq;
- fis.hasProx = hasProx;
- fis.hasVectors = hasVectors;
- for (FieldInfo fi : this) {
- FieldInfo clone = fi.clone();
- fis.putInternal(clone);
+
+ /**
+ * Creates a new instance with the given {@link FieldNumbers}.
+ */
+ Builder(FieldNumbers globalFieldNumbers) {
+ assert globalFieldNumbers != null;
+ this.globalFieldNumbers = globalFieldNumbers;
}
- return fis;
- }
- /** Returns true if any fields have positions */
- public boolean hasProx() {
- if (isReadOnly()) {
- return hasProx;
- }
- // mutable FIs must check!
- for (FieldInfo fi : this) {
- if (fi.isIndexed && fi.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) {
- return true;
+ public void add(FieldInfos other) {
+ for(FieldInfo fieldInfo : other){
+ add(fieldInfo);
}
}
- return false;
- }
-
- /** Returns true if any fields have freqs */
- public boolean hasFreq() {
- if (isReadOnly()) {
- return hasFreq;
- }
- // mutable FIs must check!
- for (FieldInfo fi : this) {
- if (fi.isIndexed && fi.indexOptions != IndexOptions.DOCS_ONLY) {
- return true;
- }
- }
- return false;
- }
-
- /**
- * Adds or updates fields that are indexed. Whether they have termvectors has to be specified.
- *
- * @param names The names of the fields
- * @param storeTermVectors Whether the fields store term vectors or not
- */
- synchronized public void addOrUpdateIndexed(Collection<String> names, boolean storeTermVectors) {
- for (String name : names) {
- addOrUpdate(name, true, storeTermVectors);
- }
- }
-
- /**
- * Assumes the fields are not storing term vectors.
- *
- * @param names The names of the fields
- * @param isIndexed Whether the fields are indexed or not
- *
- * @see #addOrUpdate(String, boolean)
- */
- synchronized public void addOrUpdate(Collection<String> names, boolean isIndexed) {
- for (String name : names) {
- addOrUpdate(name, isIndexed);
+
+ /**
+ * adds the given field to this FieldInfos name / number mapping. The given FI
+ * must be present in the global field number mapping before this method it
+ * called
+ */
+ private void putInternal(FieldInfo fi) {
+ assert !byName.containsKey(fi.name);
+ assert globalFieldNumbers.containsConsistent(Integer.valueOf(fi.number), fi.name);
+ byName.put(fi.name, fi);
}
- }
-
- /**
- * Calls 5 parameter add with false for all TermVector parameters.
- *
- * @param name The name of the IndexableField
- * @param isIndexed true if the field is indexed
- * @see #addOrUpdate(String, boolean, boolean)
- */
- synchronized public void addOrUpdate(String name, boolean isIndexed) {
- addOrUpdate(name, isIndexed, false, false);
- }
-
- /** If the field is not yet known, adds it. If it is known, checks to make
- * sure that the isIndexed flag is the same as was given previously for this
- * field. If not - marks it as being indexed. Same goes for the TermVector
- * parameters.
- *
- * @param name The name of the field
- * @param isIndexed true if the field is indexed
- * @param storeTermVector true if the term vector should be stored
- */
- synchronized public void addOrUpdate(String name, boolean isIndexed, boolean storeTermVector) {
- addOrUpdate(name, isIndexed, storeTermVector, false);
- }
-
+
/** If the field is not yet known, adds it. If it is known, checks to make
- * sure that the isIndexed flag is the same as was given previously for this
- * field. If not - marks it as being indexed. Same goes for the TermVector
- * parameters.
- *
- * @param name The name of the field
- * @param isIndexed true if the field is indexed
- * @param storeTermVector true if the term vector should be stored
- * @param omitNorms true if the norms for the indexed field should be omitted
- */
- synchronized public void addOrUpdate(String name, boolean isIndexed, boolean storeTermVector,
- boolean omitNorms) {
- addOrUpdate(name, isIndexed, storeTermVector, omitNorms, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null, null);
- }
-
- /** If the field is not yet known, adds it. If it is known, checks to make
- * sure that the isIndexed flag is the same as was given previously for this
- * field. If not - marks it as being indexed. Same goes for the TermVector
- * parameters.
- *
- * @param name The name of the field
- * @param isIndexed true if the field is indexed
- * @param storeTermVector true if the term vector should be stored
- * @param omitNorms true if the norms for the indexed field should be omitted
- * @param storePayloads true if payloads should be stored for this field
- * @param indexOptions if term freqs should be omitted for this field
- */
- synchronized public FieldInfo addOrUpdate(String name, boolean isIndexed, boolean storeTermVector,
- boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues, DocValues.Type normType) {
- return addOrUpdateInternal(name, -1, isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, normType);
- }
-
- // NOTE: this method does not carry over termVector
- // booleans nor docValuesType; the indexer chain
- // (TermVectorsConsumerPerField, DocFieldProcessor) must
- // set these fields when they succeed in consuming
- // the document:
- public FieldInfo addOrUpdate(String name, IndexableFieldType fieldType) {
- // TODO: really, indexer shouldn't even call this
- // method (it's only called from DocFieldProcessor);
- // rather, each component in the chain should update
- // what it "owns". EG fieldType.indexOptions() should
- // be updated by maybe FreqProxTermsWriterPerField:
- return addOrUpdateInternal(name, -1, fieldType.indexed(), false,
- fieldType.omitNorms(), false,
- fieldType.indexOptions(), null, null);
- }
-
- synchronized private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean isIndexed,
- boolean storeTermVector,
- boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues, DocValues.Type normType) {
- if (globalFieldNumbers == null) {
- throw new IllegalStateException("FieldInfos are read-only, create a new instance with a global field map to make modifications to FieldInfos");
- }
- FieldInfo fi = fieldInfo(name);
- if (fi == null) {
- final int fieldNumber = nextFieldNumber(name, preferredFieldNumber);
- fi = addInternal(name, fieldNumber, isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, normType);
- } else {
- fi.update(isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions);
- if (docValues != null) {
- fi.setDocValuesType(docValues, true);
- }
- if (normType != null) {
- fi.setNormValueType(normType, true);
+ * sure that the isIndexed flag is the same as was given previously for this
+ * field. If not - marks it as being indexed. Same goes for the TermVector
+ * parameters.
+ *
+ * @param name The name of the field
+ * @param isIndexed true if the field is indexed
+ * @param storeTermVector true if the term vector should be stored
+ * @param omitNorms true if the norms for the indexed field should be omitted
+ * @param storePayloads true if payloads should be stored for this field
+ * @param indexOptions if term freqs should be omitted for this field
+ */
+ // TODO: fix testCodecs to do this another way, its the only user of this
+ FieldInfo addOrUpdate(String name, boolean isIndexed, boolean storeTermVector,
+ boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues, DocValues.Type normType) {
+ return addOrUpdateInternal(name, -1, isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, normType);
+ }
+
+ // NOTE: this method does not carry over termVector
+ // booleans nor docValuesType; the indexer chain
+ // (TermVectorsConsumerPerField, DocFieldProcessor) must
+ // set these fields when they succeed in consuming
+ // the document:
+ public FieldInfo addOrUpdate(String name, IndexableFieldType fieldType) {
+ // TODO: really, indexer shouldn't even call this
+ // method (it's only called from DocFieldProcessor);
+ // rather, each component in the chain should update
+ // what it "owns". EG fieldType.indexOptions() should
+ // be updated by maybe FreqProxTermsWriterPerField:
+ return addOrUpdateInternal(name, -1, fieldType.indexed(), false,
+ fieldType.omitNorms(), false,
+ fieldType.indexOptions(), null, null);
+ }
+
+ private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean isIndexed,
+ boolean storeTermVector,
+ boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues, DocValues.Type normType) {
+ FieldInfo fi = fieldInfo(name);
+ if (fi == null) {
+ // get a global number for this field
+ final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber);
+ fi = addInternal(name, fieldNumber, isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, normType);
+ } else {
+ fi.update(isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions);
+ if (docValues != null) {
+ fi.setDocValuesType(docValues);
+ }
+ if (!fi.omitsNorms() && normType != null) {
+ fi.setNormValueType(normType);
+ }
}
+ return fi;
}
- version++;
- return fi;
- }
-
- synchronized public FieldInfo add(FieldInfo fi) {
- // IMPORTANT - reuse the field number if possible for consistent field numbers across segments
- return addOrUpdateInternal(fi.name, fi.number, fi.isIndexed, fi.storeTermVector,
- fi.omitNorms, fi.storePayloads,
- fi.indexOptions, fi.getDocValuesType(), fi.getNormType());
- }
-
- /*
- * NOTE: if you call this method from a public method make sure you check if we are modifiable and throw an exception otherwise
- */
- private FieldInfo addInternal(String name, int fieldNumber, boolean isIndexed,
- boolean storeTermVector, boolean omitNorms, boolean storePayloads,
- IndexOptions indexOptions, DocValues.Type docValuesType, DocValues.Type normType) {
- // don't check modifiable here since we use that to initially build up FIs
- if (globalFieldNumbers != null) {
- globalFieldNumbers.setIfNotSet(fieldNumber, name);
- }
- final FieldInfo fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, normType);
- putInternal(fi);
- return fi;
- }
-
- /**
- * lookup the number of a field by name.
- *
- * @param fieldName field's name
- * @return number of field, or -1 if it does not exist.
- */
- public int fieldNumber(String fieldName) {
- FieldInfo fi = fieldInfo(fieldName);
- return (fi != null) ? fi.number : -1;
- }
-
- public FieldInfo fieldInfo(String fieldName) {
- return byName.get(fieldName);
- }
-
- /**
- * Return the fieldName identified by its number.
- *
- * @param fieldNumber
- * @return the fieldName or an empty string when the field
- * with the given number doesn't exist.
- */
- public String fieldName(int fieldNumber) {
- FieldInfo fi = fieldInfo(fieldNumber);
- return (fi != null) ? fi.name : "";
- }
-
- /**
- * Return the fieldinfo object referenced by the fieldNumber.
- * @param fieldNumber
- * @return the FieldInfo object or null when the given fieldNumber
- * doesn't exist.
- */
- public FieldInfo fieldInfo(int fieldNumber) {
- return (fieldNumber >= 0) ? byNumber.get(fieldNumber) : null;
- }
-
- public Iterator<FieldInfo> iterator() {
- return byNumber.values().iterator();
- }
-
- /**
- * @return number of fields
- */
- public int size() {
- assert byNumber.size() == byName.size();
- return byNumber.size();
- }
-
- /**
- * @return true if at least one field has any vectors
- */
- public boolean hasVectors() {
- if (isReadOnly()) {
- return hasVectors;
- }
- // mutable FIs must check
- for (FieldInfo fi : this) {
- if (fi.storeTermVector) {
- return true;
- }
+
+ public FieldInfo add(FieldInfo fi) {
+ // IMPORTANT - reuse the field number if possible for consistent field numbers across segments
+ return addOrUpdateInternal(fi.name, fi.number, fi.isIndexed(), fi.hasVectors(),
+ fi.omitsNorms(), fi.hasPayloads(),
+ fi.getIndexOptions(), fi.getDocValuesType(), fi.getNormType());
}
- return false;
- }
-
- /**
- * @return true if at least one field has any norms
- */
- public boolean hasNorms() {
- for (FieldInfo fi : this) {
- if (fi.hasNorms()) {
- return true;
- }
+
+ private FieldInfo addInternal(String name, int fieldNumber, boolean isIndexed,
+ boolean storeTermVector, boolean omitNorms, boolean storePayloads,
+ IndexOptions indexOptions, DocValues.Type docValuesType, DocValues.Type normType) {
+ globalFieldNumbers.setIfNotSet(fieldNumber, name);
+ final FieldInfo fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, normType, null);
+ putInternal(fi);
+ return fi;
}
- return false;
- }
- /**
- * Returns <code>true</code> iff this instance is not backed by a
- * {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap}. Instances read from a directory via
- * {@link FieldInfos#FieldInfos(FieldInfo[], boolean, boolean, boolean)} will always be read-only
- * since no {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap} is supplied, otherwise
- * <code>false</code>.
- */
- public final boolean isReadOnly() {
- return globalFieldNumbers == null;
- }
-
- synchronized final long getVersion() {
- return version;
- }
-
- final FieldInfos asReadOnly() {
- if (isReadOnly()) {
- return this;
+ public FieldInfo fieldInfo(String fieldName) {
+ return byName.get(fieldName);
}
- final FieldInfos roFis = new FieldInfos((FieldNumberBiMap)null);
- for (FieldInfo fieldInfo : this) {
- FieldInfo clone = fieldInfo.clone();
- roFis.putInternal(clone);
- roFis.hasVectors |= clone.storeTermVector;
- roFis.hasProx |= clone.isIndexed && clone.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- roFis.hasFreq |= clone.isIndexed && clone.indexOptions != IndexOptions.DOCS_ONLY;
- }
- return roFis;
- }
-
- /**
- * @return true if at least one field has docValues
- */
- public boolean hasDocValues() {
- for (FieldInfo fi : this) {
- if (fi.hasDocValues()) {
- return true;
- }
+
+ final FieldInfos finish() {
+ return new FieldInfos(byName.values().toArray(new FieldInfo[byName.size()]));
}
-
- return false;
- }
-
- /**
- * Creates a new {@link FieldInfo} instance from the given instance. If the given instance is
- * read-only this instance will be read-only too.
- *
- * @see #isReadOnly()
- */
- static FieldInfos from(FieldInfos other) {
- return new FieldInfos(other.globalFieldNumbers);
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java Mon May 28 19:28:10 2012
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.Map;
import org.apache.lucene.codecs.FieldsConsumer;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CollectionUtil;
import org.apache.lucene.util.IOUtils;
@@ -39,7 +38,7 @@ final class FreqProxTermsWriter extends
// Other writers would presumably share alot of this...
@Override
- public void flush(Map<FieldInfo, TermsHashConsumerPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
+ public void flush(Map<String,TermsHashConsumerPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
// Gather all FieldData's that have postings, across all
// ThreadStates
@@ -57,7 +56,7 @@ final class FreqProxTermsWriter extends
// Sort by field name
CollectionUtil.quickSort(allFields);
- final FieldsConsumer consumer = state.codec.postingsFormat().fieldsConsumer(state);
+ final FieldsConsumer consumer = state.segmentInfo.getCodec().postingsFormat().fieldsConsumer(state);
boolean success = false;
@@ -80,13 +79,7 @@ final class FreqProxTermsWriter extends
final FieldInfo fieldInfo = allFields.get(fieldNumber).fieldInfo;
final FreqProxTermsWriterPerField fieldWriter = allFields.get(fieldNumber);
-
- // Aggregate the storePayload as seen by the same
- // field across multiple threads
- if (fieldInfo.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) {
- fieldInfo.storePayloads |= fieldWriter.hasPayloads;
- }
-
+
// If this field has postings then add them to the
// segment
fieldWriter.flush(fieldInfo.name, consumer, state);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Mon May 28 19:28:10 2012
@@ -55,7 +55,7 @@ final class FreqProxTermsWriterPerField
this.fieldInfo = fieldInfo;
docState = termsHashPerField.docState;
fieldState = termsHashPerField.fieldState;
- setIndexOptions(fieldInfo.indexOptions);
+ setIndexOptions(fieldInfo.getIndexOptions());
}
@Override
@@ -68,7 +68,11 @@ final class FreqProxTermsWriterPerField
}
@Override
- void finish() {}
+ void finish() {
+ if (hasPayloads) {
+ fieldInfo.setStorePayloads();
+ }
+ }
boolean hasPayloads;
@@ -83,7 +87,7 @@ final class FreqProxTermsWriterPerField
void reset() {
// Record, up front, whether our in-RAM format will be
// with or without term freqs:
- setIndexOptions(fieldInfo.indexOptions);
+ setIndexOptions(fieldInfo.getIndexOptions());
payloadAttribute = null;
}
@@ -330,7 +334,7 @@ final class FreqProxTermsWriterPerField
// according to this.indexOptions, but then write the
// new segment to the directory according to
// currentFieldIndexOptions:
- final IndexOptions currentFieldIndexOptions = fieldInfo.indexOptions;
+ final IndexOptions currentFieldIndexOptions = fieldInfo.getIndexOptions();
final boolean writeTermFreq = currentFieldIndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
final boolean writePositions = currentFieldIndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
@@ -363,7 +367,7 @@ final class FreqProxTermsWriterPerField
final ByteSliceReader freq = new ByteSliceReader();
final ByteSliceReader prox = new ByteSliceReader();
- FixedBitSet visitedDocs = new FixedBitSet(state.numDocs);
+ FixedBitSet visitedDocs = new FixedBitSet(state.segmentInfo.getDocCount());
long sumTotalTermFreq = 0;
long sumDocFreq = 0;
@@ -441,7 +445,7 @@ final class FreqProxTermsWriterPerField
}
numDocs++;
- assert docID < state.numDocs: "doc=" + docID + " maxDoc=" + state.numDocs;
+ assert docID < state.segmentInfo.getDocCount(): "doc=" + docID + " maxDoc=" + state.segmentInfo.getDocCount();
// NOTE: we could check here if the docID was
// deleted, and skip it. However, this is somewhat
@@ -463,7 +467,7 @@ final class FreqProxTermsWriterPerField
// TODO: can we do this reach-around in a cleaner way????
if (state.liveDocs == null) {
- state.liveDocs = docState.docWriter.codec.liveDocsFormat().newLiveDocs(state.numDocs);
+ state.liveDocs = docState.docWriter.codec.liveDocsFormat().newLiveDocs(state.segmentInfo.getDocCount());
}
if (state.liveDocs.get(docID)) {
state.delCountOnFlush++;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=1343365&r1=1343364&r2=1343365&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java Mon May 28 19:28:10 2012
@@ -150,7 +150,7 @@ final class IndexFileDeleter {
for (String fileName : files) {
- if ((IndexFileNameFilter.INSTANCE.accept(null, fileName)) && !fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
+ if (!fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
// Add this file to refCounts with initial count 0:
getRefCount(fileName);
@@ -189,30 +189,6 @@ final class IndexFileDeleter {
}
}
if (sis != null) {
- final SegmentInfos infos = sis;
- for (SegmentInfo segmentInfo : infos) {
- try {
- /*
- * Force FI to load for each segment since we could see a
- * segments file and load successfully above if the files are
- * still referenced when they are deleted and the os doesn't let
- * you delete them. Yet its likely that fnm files are removed
- * while seg file is still around Since LUCENE-2984 we need FI
- * to find out if a seg has vectors and prox so we need those
- * files to be opened for a commit point.
- */
- segmentInfo.getFieldInfos();
- } catch (FileNotFoundException e) {
- refresh(segmentInfo.name);
- sis = null;
- if (infoStream.isEnabled("IFD")) {
- infoStream.message("IFD", "init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
- }
- }
- }
-
- }
- if (sis != null) {
final CommitPoint commitPoint = new CommitPoint(commitsToDelete, directory, sis);
if (sis.getGeneration() == segmentInfos.getGeneration()) {
currentCommitPoint = commitPoint;
@@ -356,7 +332,7 @@ final class IndexFileDeleter {
for(int i=0;i<files.length;i++) {
String fileName = files[i];
if ((segmentName == null || fileName.startsWith(segmentPrefix1) || fileName.startsWith(segmentPrefix2)) &&
- IndexFileNameFilter.INSTANCE.accept(null, fileName) &&
+ !fileName.endsWith("write.lock") &&
!refCounts.containsKey(fileName) &&
!fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
// Unreferenced file, so remove it
@@ -488,7 +464,7 @@ final class IndexFileDeleter {
assert locked();
// If this is a commit point, also incRef the
// segments_N file:
- for( final String fileName: segmentInfos.files(directory, isCommit) ) {
+ for(final String fileName: segmentInfos.files(directory, isCommit)) {
incRef(fileName);
}
}