You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2011/12/15 06:11:30 UTC
svn commit: r1214623 [5/17] - in /lucene/dev/branches/solrcloud: ./
dev-tools/idea/lucene/contrib/ lucene/ lucene/contrib/
lucene/contrib/demo/src/java/org/apache/lucene/demo/
lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight...
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiReader.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiReader.java Thu Dec 15 05:11:14 2011
@@ -18,139 +18,62 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.lucene.index.codecs.PerDocValues;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.ReaderUtil;
-import org.apache.lucene.util.MapBackedSet;
/** An IndexReader which reads multiple indexes, appending
* their content. */
-public class MultiReader extends IndexReader implements Cloneable {
- protected IndexReader[] subReaders;
- private final ReaderContext topLevelContext;
- private int[] starts; // 1st docno for each segment
- private boolean[] decrefOnClose; // remember which subreaders to decRef on close
- private int maxDoc = 0;
- private int numDocs = -1;
- private boolean hasDeletions = false;
+public class MultiReader extends BaseMultiReader<IndexReader> {
+ private final boolean[] decrefOnClose; // remember which subreaders to decRef on close
/**
* <p>Construct a MultiReader aggregating the named set of (sub)readers.
- * Directory locking for delete, undeleteAll, and setNorm operations is
- * left to the subreaders. </p>
* <p>Note that all subreaders are closed if this Multireader is closed.</p>
* @param subReaders set of (sub)readers
*/
public MultiReader(IndexReader... subReaders) throws IOException {
- topLevelContext = initialize(subReaders, true);
+ this(subReaders, true);
}
/**
* <p>Construct a MultiReader aggregating the named set of (sub)readers.
- * Directory locking for delete, undeleteAll, and setNorm operations is
- * left to the subreaders. </p>
+ * @param subReaders set of (sub)readers
* @param closeSubReaders indicates whether the subreaders should be closed
* when this MultiReader is closed
- * @param subReaders set of (sub)readers
*/
public MultiReader(IndexReader[] subReaders, boolean closeSubReaders) throws IOException {
- topLevelContext = initialize(subReaders, closeSubReaders);
- }
-
- private ReaderContext initialize(IndexReader[] subReaders, boolean closeSubReaders) throws IOException {
- this.subReaders = subReaders.clone();
- starts = new int[subReaders.length + 1]; // build starts array
+ super(subReaders.clone());
decrefOnClose = new boolean[subReaders.length];
for (int i = 0; i < subReaders.length; i++) {
- starts[i] = maxDoc;
- maxDoc += subReaders[i].maxDoc(); // compute maxDocs
-
if (!closeSubReaders) {
subReaders[i].incRef();
decrefOnClose[i] = true;
} else {
decrefOnClose[i] = false;
}
-
- if (subReaders[i].hasDeletions()) {
- hasDeletions = true;
- }
}
- starts[subReaders.length] = maxDoc;
- readerFinishedListeners = new MapBackedSet<ReaderFinishedListener>(new ConcurrentHashMap<ReaderFinishedListener,Boolean>());
- return ReaderUtil.buildReaderContext(this);
}
-
- @Override
- public long getUniqueTermCount() throws IOException {
- throw new UnsupportedOperationException("");
- }
-
- @Override
- public Fields fields() throws IOException {
- throw new UnsupportedOperationException("please use MultiFields.getFields, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level Fields");
+
+ // used only by openIfChaged
+ private MultiReader(IndexReader[] subReaders, boolean[] decrefOnClose)
+ throws IOException {
+ super(subReaders);
+ this.decrefOnClose = decrefOnClose;
}
- /**
- * Tries to reopen the subreaders.
- * <br>
- * If one or more subreaders could be re-opened (i. e. IndexReader.openIfChanged(subReader)
- * returned a new instance), then a new MultiReader instance
- * is returned, otherwise this instance is returned.
- * <p>
- * A re-opened instance might share one or more subreaders with the old
- * instance. Index modification operations result in undefined behavior
- * when performed before the old instance is closed.
- * (see {@link IndexReader#openIfChanged}).
- * <p>
- * If subreaders are shared, then the reference count of those
- * readers is increased to ensure that the subreaders remain open
- * until the last referring reader is closed.
- *
- * @throws CorruptIndexException if the index is corrupt
- * @throws IOException if there is a low-level IO error
- */
@Override
protected synchronized IndexReader doOpenIfChanged() throws CorruptIndexException, IOException {
- return doOpenIfChanged(false);
+ return doReopen(false);
}
- /**
- * Clones the subreaders.
- * (see {@link IndexReader#clone()}).
- * <br>
- * <p>
- * If subreaders are shared, then the reference count of those
- * readers is increased to ensure that the subreaders remain open
- * until the last referring reader is closed.
- */
@Override
public synchronized Object clone() {
try {
- return doOpenIfChanged(true);
+ return doReopen(true);
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
-
- @Override
- public Bits getLiveDocs() {
- throw new UnsupportedOperationException("please use MultiFields.getLiveDocs, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level Bits liveDocs");
- }
- /**
- * If clone is true then we clone each of the subreaders
- * @param doClone
- * @return New IndexReader, or null if open/clone is not necessary
- * @throws CorruptIndexException
- * @throws IOException
- */
- protected IndexReader doOpenIfChanged(boolean doClone) throws CorruptIndexException, IOException {
+ private IndexReader doReopen(boolean doClone) throws CorruptIndexException, IOException {
ensureOpen();
boolean changed = false;
@@ -195,142 +118,31 @@ public class MultiReader extends IndexRe
newDecrefOnClose[i] = true;
}
}
- MultiReader mr = new MultiReader(newSubReaders);
- mr.decrefOnClose = newDecrefOnClose;
- return mr;
+ return new MultiReader(newSubReaders, newDecrefOnClose);
} else {
return null;
}
}
@Override
- public Fields getTermVectors(int docID) throws IOException {
- ensureOpen();
- int i = readerIndex(docID); // find segment num
- return subReaders[i].getTermVectors(docID - starts[i]); // dispatch to segment
- }
-
- @Override
- public int numDocs() {
- // Don't call ensureOpen() here (it could affect performance)
- // NOTE: multiple threads may wind up init'ing
- // numDocs... but that's harmless
- if (numDocs == -1) { // check cache
- int n = 0; // cache miss--recompute
- for (int i = 0; i < subReaders.length; i++)
- n += subReaders[i].numDocs(); // sum from readers
- numDocs = n;
- }
- return numDocs;
- }
-
- @Override
- public int maxDoc() {
- // Don't call ensureOpen() here (it could affect performance)
- return maxDoc;
- }
-
- @Override
- public void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException {
- ensureOpen();
- int i = readerIndex(docID); // find segment num
- subReaders[i].document(docID - starts[i], visitor); // dispatch to segment reader
- }
-
- @Override
- public boolean hasDeletions() {
- ensureOpen();
- return hasDeletions;
- }
-
- @Override
- protected void doDelete(int n) throws CorruptIndexException, IOException {
- numDocs = -1; // invalidate cache
- int i = readerIndex(n); // find segment num
- subReaders[i].deleteDocument(n - starts[i]); // dispatch to segment reader
- hasDeletions = true;
- }
-
- @Override
- protected void doUndeleteAll() throws CorruptIndexException, IOException {
- for (int i = 0; i < subReaders.length; i++)
- subReaders[i].undeleteAll();
-
- hasDeletions = false;
- numDocs = -1; // invalidate cache
- }
-
- private int readerIndex(int n) { // find reader for doc n:
- return DirectoryReader.readerIndex(n, this.starts, this.subReaders.length);
- }
-
- @Override
- public boolean hasNorms(String field) throws IOException {
- ensureOpen();
- for (int i = 0; i < subReaders.length; i++) {
- if (subReaders[i].hasNorms(field)) return true;
- }
- return false;
- }
-
- @Override
- public synchronized byte[] norms(String field) throws IOException {
- throw new UnsupportedOperationException("please use MultiNorms.norms, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level norms");
- }
-
- @Override
- protected void doSetNorm(int n, String field, byte value)
- throws CorruptIndexException, IOException {
- int i = readerIndex(n); // find segment num
- subReaders[i].setNorm(n-starts[i], field, value); // dispatch
- }
-
- @Override
- public int docFreq(Term t) throws IOException {
- ensureOpen();
- int total = 0; // sum freqs in segments
- for (int i = 0; i < subReaders.length; i++)
- total += subReaders[i].docFreq(t);
- return total;
- }
-
- @Override
- public int docFreq(String field, BytesRef t) throws IOException {
- ensureOpen();
- int total = 0; // sum freqs in segments
- for (int i = 0; i < subReaders.length; i++) {
- total += subReaders[i].docFreq(field, t);
- }
- return total;
- }
-
- @Override
- protected void doCommit(Map<String,String> commitUserData) throws IOException {
- for (int i = 0; i < subReaders.length; i++)
- subReaders[i].commit(commitUserData);
- }
-
- @Override
protected synchronized void doClose() throws IOException {
+ IOException ioe = null;
for (int i = 0; i < subReaders.length; i++) {
- if (decrefOnClose[i]) {
- subReaders[i].decRef();
- } else {
- subReaders[i].close();
+ try {
+ if (decrefOnClose[i]) {
+ subReaders[i].decRef();
+ } else {
+ subReaders[i].close();
+ }
+ } catch (IOException e) {
+ if (ioe == null) ioe = e;
}
}
+ // throw the first exception
+ if (ioe != null) throw ioe;
}
@Override
- public Collection<String> getFieldNames (IndexReader.FieldOption fieldNames) {
- ensureOpen();
- return DirectoryReader.getFieldNames(fieldNames, this.subReaders);
- }
-
- /**
- * Checks recursively if all subreaders are up to date.
- */
- @Override
public boolean isCurrent() throws CorruptIndexException, IOException {
ensureOpen();
for (int i = 0; i < subReaders.length; i++) {
@@ -350,36 +162,4 @@ public class MultiReader extends IndexRe
public long getVersion() {
throw new UnsupportedOperationException("MultiReader does not support this method.");
}
-
- @Override
- public IndexReader[] getSequentialSubReaders() {
- return subReaders;
- }
-
- @Override
- public ReaderContext getTopReaderContext() {
- ensureOpen();
- return topLevelContext;
- }
-
- @Override
- public void addReaderFinishedListener(ReaderFinishedListener listener) {
- super.addReaderFinishedListener(listener);
- for(IndexReader sub : subReaders) {
- sub.addReaderFinishedListener(listener);
- }
- }
-
- @Override
- public void removeReaderFinishedListener(ReaderFinishedListener listener) {
- super.removeReaderFinishedListener(listener);
- for(IndexReader sub : subReaders) {
- sub.removeReaderFinishedListener(listener);
- }
- }
-
- @Override
- public PerDocValues perDocValues() throws IOException {
- throw new UnsupportedOperationException("please use MultiPerDocValues#getPerDocs, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level Fields");
- }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/ParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/ParallelReader.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/ParallelReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/ParallelReader.java Thu Dec 15 05:11:14 2011
@@ -17,15 +17,11 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.index.codecs.PerDocValues;
-import org.apache.lucene.index.values.IndexDocValues;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.MapBackedSet;
import java.io.IOException;
import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
/** An IndexReader which reads multiple, parallel indexes. Each index added
@@ -59,7 +55,6 @@ public class ParallelReader extends Inde
private boolean hasDeletions;
private final ParallelFields fields = new ParallelFields();
- private final ParallelPerDocs perDocs = new ParallelPerDocs();
/** Construct a ParallelReader.
* <p>Note that all subreaders are closed if this ParallelReader is closed.</p>
@@ -73,7 +68,6 @@ public class ParallelReader extends Inde
public ParallelReader(boolean closeSubReaders) throws IOException {
super();
this.incRefReaders = !closeSubReaders;
- readerFinishedListeners = new MapBackedSet<ReaderFinishedListener>(new ConcurrentHashMap<ReaderFinishedListener,Boolean>());
}
/** {@inheritDoc} */
@@ -132,7 +126,6 @@ public class ParallelReader extends Inde
if (fieldToReader.get(field) == null) {
fieldToReader.put(field, reader);
this.fields.addField(field, MultiFields.getFields(reader).terms(field));
- this.perDocs.addField(field, reader);
}
}
@@ -247,7 +240,7 @@ public class ParallelReader extends Inde
return doReopen(false);
}
- protected IndexReader doReopen(boolean doClone) throws CorruptIndexException, IOException {
+ private IndexReader doReopen(boolean doClone) throws CorruptIndexException, IOException {
ensureOpen();
boolean reopened = false;
@@ -332,24 +325,6 @@ public class ParallelReader extends Inde
return hasDeletions;
}
- // delete in all readers
- @Override
- protected void doDelete(int n) throws CorruptIndexException, IOException {
- for (final IndexReader reader : readers) {
- reader.deleteDocument(n);
- }
- hasDeletions = true;
- }
-
- // undeleteAll in all readers
- @Override
- protected void doUndeleteAll() throws CorruptIndexException, IOException {
- for (final IndexReader reader : readers) {
- reader.undeleteAll();
- }
- hasDeletions = false;
- }
-
@Override
public void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException {
ensureOpen();
@@ -403,25 +378,6 @@ public class ParallelReader extends Inde
}
@Override
- protected void doSetNorm(int n, String field, byte value)
- throws CorruptIndexException, IOException {
- IndexReader reader = fieldToReader.get(field);
- if (reader!=null) {
- synchronized(normsCache) {
- normsCache.remove(field);
- }
- reader.doSetNorm(n, field, value);
- }
- }
-
- @Override
- public int docFreq(Term term) throws IOException {
- ensureOpen();
- IndexReader reader = fieldToReader.get(term.field());
- return reader==null ? 0 : reader.docFreq(term);
- }
-
- @Override
public int docFreq(String field, BytesRef term) throws IOException {
ensureOpen();
IndexReader reader = fieldToReader.get(field);
@@ -458,12 +414,6 @@ public class ParallelReader extends Inde
}
@Override
- protected void doCommit(Map<String,String> commitUserData) throws IOException {
- for (final IndexReader reader : readers)
- reader.commit(commitUserData);
- }
-
- @Override
protected synchronized void doClose() throws IOException {
for (int i = 0; i < readers.size(); i++) {
if (decrefOnClose.get(i).booleanValue()) {
@@ -491,57 +441,10 @@ public class ParallelReader extends Inde
return topLevelReaderContext;
}
+ // TODO: I suspect this is completely untested!!!!!
@Override
- public void addReaderFinishedListener(ReaderFinishedListener listener) {
- super.addReaderFinishedListener(listener);
- for (IndexReader reader : readers) {
- reader.addReaderFinishedListener(listener);
- }
- }
-
- @Override
- public void removeReaderFinishedListener(ReaderFinishedListener listener) {
- super.removeReaderFinishedListener(listener);
- for (IndexReader reader : readers) {
- reader.removeReaderFinishedListener(listener);
- }
- }
-
- @Override
- public PerDocValues perDocValues() throws IOException {
- ensureOpen();
- return perDocs;
- }
-
- // Single instance of this, per ParallelReader instance
- private static final class ParallelPerDocs extends PerDocValues {
- final TreeMap<String,IndexDocValues> fields = new TreeMap<String,IndexDocValues>();
-
- void addField(String field, IndexReader r) throws IOException {
- PerDocValues perDocs = MultiPerDocValues.getPerDocs(r);
- if (perDocs != null) {
- fields.put(field, perDocs.docValues(field));
- }
- }
-
- @Override
- public void close() throws IOException {
- // nothing to do here
- }
-
- @Override
- public IndexDocValues docValues(String field) throws IOException {
- return fields.get(field);
- }
-
- @Override
- public Collection<String> fields() {
- return fields.keySet();
- }
+ public DocValues docValues(String field) throws IOException {
+ IndexReader reader = fieldToReader.get(field);
+ return reader == null ? null : MultiDocValues.getDocValues(reader, field);
}
}
-
-
-
-
-
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java Thu Dec 15 05:11:14 2011
@@ -62,7 +62,7 @@ public class PersistentSnapshotDeletionP
* keeps a lock on the snapshots directory).
*/
public static Map<String, String> readSnapshotsInfo(Directory dir) throws IOException {
- IndexReader r = IndexReader.open(dir, true);
+ IndexReader r = IndexReader.open(dir);
Map<String, String> snapshots = new HashMap<String, String>();
try {
int numDocs = r.numDocs();
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java Thu Dec 15 05:11:14 2011
@@ -18,18 +18,23 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.lucene.index.SegmentReader.CoreClosedListener;
import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.NormsReader;
+import org.apache.lucene.index.codecs.PerDocProducer;
import org.apache.lucene.index.codecs.PostingsFormat;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.StoredFieldsReader;
-import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.TermVectorsReader;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.MapBackedSet;
/** Holds core readers that are shared (unchanged) when
* SegmentReader is cloned or reopened */
@@ -47,7 +52,8 @@ final class SegmentCoreReaders {
final FieldInfos fieldInfos;
final FieldsProducer fields;
- final PerDocValues perDocProducer;
+ final PerDocProducer perDocProducer;
+ final NormsReader norms;
final Directory dir;
final Directory cfsDir;
@@ -61,7 +67,8 @@ final class SegmentCoreReaders {
CompoundFileDirectory cfsReader;
CompoundFileDirectory storeCFSReader;
-
+ final Set<CoreClosedListener> coreClosedListeners =
+ new MapBackedSet<CoreClosedListener>(new ConcurrentHashMap<CoreClosedListener, Boolean>());
SegmentCoreReaders(SegmentReader owner, Directory dir, SegmentInfo si, IOContext context, int termsIndexDivisor) throws IOException {
@@ -92,6 +99,10 @@ final class SegmentCoreReaders {
// Ask codec for its Fields
fields = format.fieldsProducer(segmentReadState);
assert fields != null;
+ // ask codec for its Norms:
+ // TODO: since we don't write any norms file if there are no norms,
+ // kinda jaky to assume the codec handles the case of no norms file at all gracefully?!
+ norms = codec.normsFormat().normsReader(cfsDir, si, fieldInfos, context, dir);
perDocProducer = codec.docValuesFormat().docsProducer(segmentReadState);
success = true;
} finally {
@@ -126,10 +137,9 @@ final class SegmentCoreReaders {
synchronized void decRef() throws IOException {
if (ref.decrementAndGet() == 0) {
IOUtils.close(fields, perDocProducer, termVectorsReaderOrig,
- fieldsReaderOrig, cfsReader, storeCFSReader);
- // Now, notify any ReaderFinished listeners:
- if (owner != null) {
- owner.notifyReaderFinishedListeners();
+ fieldsReaderOrig, cfsReader, storeCFSReader, norms);
+ for (CoreClosedListener listener : coreClosedListeners) {
+ listener.onClose(owner);
}
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Thu Dec 15 05:11:14 2011
@@ -48,7 +48,7 @@ public final class SegmentInfo implement
// TODO: remove these from this class, for now this is the representation
public static final int NO = -1; // e.g. no norms; no deletes;
public static final int YES = 1; // e.g. have norms; have deletes;
- static final int WITHOUT_GEN = 0; // a file name that has no GEN in it.
+ public static final int WITHOUT_GEN = 0; // a file name that has no GEN in it.
public String name; // unique name in dir
public int docCount; // number of docs in seg
@@ -337,23 +337,10 @@ public final class SegmentInfo implement
}
/**
- * Returns true if this field for this segment has saved a separate norms file (_<segment>_N.sX).
- *
- * @param fieldNumber the field index to check
- */
- public boolean hasSeparateNorms(int fieldNumber) {
- if (normGen == null) {
- return false;
- }
-
- Long gen = normGen.get(fieldNumber);
- return gen != null && gen.longValue() != NO;
- }
-
- /**
- * Returns true if any fields in this segment have separate norms.
+ * @deprecated separate norms are not supported in >= 4.0
*/
- public boolean hasSeparateNorms() {
+ @Deprecated
+ boolean hasSeparateNorms() {
if (normGen == null) {
return false;
} else {
@@ -367,42 +354,6 @@ public final class SegmentInfo implement
return false;
}
- void initNormGen() {
- if (normGen == null) { // normGen is null if this segments file hasn't had any norms set against it yet
- normGen = new HashMap<Integer, Long>();
- }
- }
-
- /**
- * Increment the generation count for the norms file for
- * this field.
- *
- * @param fieldIndex field whose norm file will be rewritten
- */
- void advanceNormGen(int fieldIndex) {
- Long gen = normGen.get(fieldIndex);
- if (gen == null || gen.longValue() == NO) {
- normGen.put(fieldIndex, new Long(YES));
- } else {
- normGen.put(fieldIndex, gen+1);
- }
- clearFilesCache();
- }
-
- /**
- * Get the file name for the norms file for this field.
- *
- * @param number field index
- */
- public String getNormFileName(int number) {
- if (hasSeparateNorms(number)) {
- return IndexFileNames.fileNameFromGeneration(name, IndexFileNames.SEPARATE_NORMS_EXTENSION + number, normGen.get(number));
- } else {
- // single file for all norms
- return IndexFileNames.fileNameFromGeneration(name, IndexFileNames.NORMS_EXTENSION, WITHOUT_GEN);
- }
- }
-
/**
* Mark whether this segment is stored as a compound file.
*
@@ -516,11 +467,6 @@ public final class SegmentInfo implement
return codec;
}
- private void addIfExists(Set<String> files, String fileName) throws IOException {
- if (dir.fileExists(fileName))
- files.add(fileName);
- }
-
/*
* Return all files referenced by this SegmentInfo. The
* returns List is a locally cached List so you should not
@@ -546,9 +492,6 @@ public final class SegmentInfo implement
IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
}
} else {
- for(String ext : IndexFileNames.NON_STORE_INDEX_EXTENSIONS) {
- addIfExists(fileSet, IndexFileNames.segmentFileName(name, "", ext));
- }
codec.files(dir, this, fileSet);
}
@@ -566,15 +509,12 @@ public final class SegmentInfo implement
if (delFileName != null && (delGen >= YES || dir.fileExists(delFileName))) {
fileSet.add(delFileName);
}
-
+
+ // because separate norm files are unconditionally stored outside cfs,
+ // we must explicitly ask for their filenames if we might have separate norms:
+ // remove this when 3.x indexes are no longer supported
if (normGen != null) {
- for (Entry<Integer,Long> entry : normGen.entrySet()) {
- long gen = entry.getValue();
- if (gen >= YES) {
- // Definitely a separate norm file, with generation:
- fileSet.add(IndexFileNames.fileNameFromGeneration(name, IndexFileNames.SEPARATE_NORMS_EXTENSION + entry.getKey(), gen));
- }
- }
+ codec.normsFormat().separateFiles(dir, this, fileSet);
}
files = new ArrayList<String>(fileSet);
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Thu Dec 15 05:11:14 2011
@@ -19,22 +19,23 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexReader.FieldOption;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.FieldInfosWriter;
import org.apache.lucene.index.codecs.FieldsConsumer;
+import org.apache.lucene.index.codecs.NormsWriter;
import org.apache.lucene.index.codecs.StoredFieldsWriter;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.TermVectorsWriter;
-import org.apache.lucene.index.values.ValueType;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;
@@ -122,15 +123,17 @@ final class SegmentMerger {
final SegmentWriteState segmentWriteState = new SegmentWriteState(mergeState.infoStream, directory, segment, mergeState.fieldInfos, mergeState.mergedDocCount, termIndexInterval, codec, null, context);
mergeTerms(segmentWriteState);
mergePerDoc(segmentWriteState);
- mergeNorms();
+
+ if (mergeState.fieldInfos.hasNorms()) {
+ int numMerged = mergeNorms(segmentWriteState);
+ assert numMerged == mergeState.mergedDocCount;
+ }
if (mergeState.fieldInfos.hasVectors()) {
int numMerged = mergeVectors();
assert numMerged == mergeState.mergedDocCount;
}
- // write FIS once merge is done. IDV might change types or drops fields
- FieldInfosWriter fieldInfosWriter = codec.fieldInfosFormat().getFieldInfosWriter();
- fieldInfosWriter.write(directory, segment, mergeState.fieldInfos, context);
+
return mergeState;
}
@@ -183,15 +186,40 @@ final class SegmentMerger {
}
}
}
+
+ // returns an updated typepromoter (tracking type and size) given a previous one,
+ // and a newly encountered docvalues
+ private TypePromoter mergeDocValuesType(TypePromoter previous, DocValues docValues) {
+ TypePromoter incoming = TypePromoter.create(docValues.type(), docValues.getValueSize());
+ if (previous == null) {
+ previous = TypePromoter.getIdentityPromoter();
+ }
+ TypePromoter promoted = previous.promote(incoming);
+ if (promoted == null) {
+ // type is incompatible: promote to BYTES_VAR_STRAIGHT
+ return TypePromoter.create(DocValues.Type.BYTES_VAR_STRAIGHT, TypePromoter.VAR_TYPE_VALUE_SIZE);
+ } else {
+ return promoted;
+ }
+ }
private void mergeFieldInfos() throws IOException {
+ // mapping from all docvalues fields found to their promoted types
+ // this is because FieldInfos does not store the valueSize
+ Map<FieldInfo,TypePromoter> docValuesTypes = new HashMap<FieldInfo,TypePromoter>();
+
for (MergeState.IndexReaderAndLiveDocs readerAndLiveDocs : mergeState.readers) {
final IndexReader reader = readerAndLiveDocs.reader;
if (reader instanceof SegmentReader) {
SegmentReader segmentReader = (SegmentReader) reader;
FieldInfos readerFieldInfos = segmentReader.fieldInfos();
for (FieldInfo fi : readerFieldInfos) {
- mergeState.fieldInfos.add(fi);
+ FieldInfo merged = mergeState.fieldInfos.add(fi);
+ // update the type promotion mapping for this reader
+ if (fi.hasDocValues()) {
+ TypePromoter previous = docValuesTypes.get(merged);
+ docValuesTypes.put(merged, mergeDocValuesType(previous, reader.docValues(fi.name)));
+ }
}
} else {
addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
@@ -206,10 +234,33 @@ final class SegmentMerger {
Collection<String> dvNames = reader.getFieldNames(FieldOption.DOC_VALUES);
mergeState.fieldInfos.addOrUpdate(dvNames, false);
for (String dvName : dvNames) {
- mergeState.fieldInfos.fieldInfo(dvName).setDocValues(reader.docValues(dvName).type());
+ FieldInfo merged = mergeState.fieldInfos.fieldInfo(dvName);
+ DocValues docValues = reader.docValues(dvName);
+ merged.setDocValuesType(docValues.type());
+ TypePromoter previous = docValuesTypes.get(merged);
+ docValuesTypes.put(merged, mergeDocValuesType(previous, docValues));
+ }
+ }
+ }
+
+ // update any promoted doc values types:
+ for (Map.Entry<FieldInfo,TypePromoter> e : docValuesTypes.entrySet()) {
+ FieldInfo fi = e.getKey();
+ TypePromoter promoter = e.getValue();
+ if (promoter == null) {
+ fi.resetDocValuesType(null);
+ } else {
+ assert promoter != TypePromoter.getIdentityPromoter();
+ if (fi.getDocValuesType() != promoter.type()) {
+ // reset the type if we got promoted
+ fi.resetDocValuesType(promoter.type());
}
}
}
+
+ // write the merged infos
+ FieldInfosWriter fieldInfosWriter = codec.fieldInfosFormat().getFieldInfosWriter();
+ fieldInfosWriter.write(directory, segment, mergeState.fieldInfos, context);
}
/**
@@ -329,48 +380,19 @@ final class SegmentMerger {
}
}
- private void mergeNorms() throws IOException {
- IndexOutput output = null;
+ private int mergeNorms(SegmentWriteState segmentWriteState) throws IOException {
+ final NormsWriter writer = codec.normsFormat().normsWriter(segmentWriteState);
+
boolean success = false;
try {
- for (FieldInfo fi : mergeState.fieldInfos) {
- if (fi.isIndexed && !fi.omitNorms) {
- if (output == null) {
- output = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.NORMS_EXTENSION), context);
- output.writeBytes(SegmentNorms.NORMS_HEADER, SegmentNorms.NORMS_HEADER.length);
- }
- for (MergeState.IndexReaderAndLiveDocs reader : mergeState.readers) {
- final int maxDoc = reader.reader.maxDoc();
- byte normBuffer[] = reader.reader.norms(fi.name);
- if (normBuffer == null) {
- // Can be null if this segment doesn't have
- // any docs with this field
- normBuffer = new byte[maxDoc];
- Arrays.fill(normBuffer, (byte)0);
- }
- if (reader.liveDocs == null) {
- //optimized case for segments without deleted docs
- output.writeBytes(normBuffer, maxDoc);
- } else {
- // this segment has deleted docs, so we have to
- // check for every doc if it is deleted or not
- final Bits liveDocs = reader.liveDocs;
- for (int k = 0; k < maxDoc; k++) {
- if (liveDocs.get(k)) {
- output.writeByte(normBuffer[k]);
- }
- }
- }
- mergeState.checkAbort.work(maxDoc);
- }
- }
- }
+ int numMerged = writer.merge(mergeState);
success = true;
+ return numMerged;
} finally {
if (success) {
- IOUtils.close(output);
+ IOUtils.close(writer);
} else {
- IOUtils.closeWhileHandlingException(output);
+ IOUtils.closeWhileHandlingException(writer);
}
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentReader.java Thu Dec 15 05:11:14 2011
@@ -20,54 +20,46 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IndexInput;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.codecs.PerDocProducer;
import org.apache.lucene.index.codecs.StoredFieldsReader;
-import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.TermVectorsReader;
+import org.apache.lucene.search.FieldCache; // javadocs
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.CloseableThreadLocal;
-import org.apache.lucene.util.StringHelper;
/**
* @lucene.experimental
*/
-public class SegmentReader extends IndexReader implements Cloneable {
- protected boolean readOnly;
+public final class SegmentReader extends IndexReader implements Cloneable {
+ private final boolean readOnly;
private SegmentInfo si;
private final ReaderContext readerContext = new AtomicReaderContext(this);
- CloseableThreadLocal<StoredFieldsReader> fieldsReaderLocal = new FieldsReaderLocal();
- CloseableThreadLocal<TermVectorsReader> termVectorsLocal = new CloseableThreadLocal<TermVectorsReader>();
+ final CloseableThreadLocal<StoredFieldsReader> fieldsReaderLocal = new FieldsReaderLocal();
+ final CloseableThreadLocal<TermVectorsReader> termVectorsLocal = new CloseableThreadLocal<TermVectorsReader>();
- volatile BitVector liveDocs;
+ volatile BitVector liveDocs = null;
+ volatile Object combinedCoreAndDeletesKey;
AtomicInteger liveDocsRef = null;
+ boolean hasChanges = false;
private boolean liveDocsDirty = false;
- private boolean normsDirty = false;
- // TODO: we should move this tracking into SegmentInfo;
- // this way SegmentInfo.toString shows pending deletes
+ // TODO: remove deletions from SR
private int pendingDeleteCount;
-
private boolean rollbackHasChanges = false;
private boolean rollbackDeletedDocsDirty = false;
- private boolean rollbackNormsDirty = false;
private SegmentInfo rollbackSegmentInfo;
private int rollbackPendingDeleteCount;
-
- // optionally used for the .nrm file shared by multiple norms
- IndexInput singleNormStream;
- AtomicInteger singleNormRef;
+ // end TODO
SegmentCoreReaders core;
@@ -80,41 +72,39 @@ public class SegmentReader extends Index
return core.getFieldsReaderOrig().clone();
}
}
-
- Map<String,SegmentNorms> norms = new HashMap<String,SegmentNorms>();
/**
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public static SegmentReader get(boolean readOnly, SegmentInfo si, int termInfosIndexDivisor, IOContext context) throws CorruptIndexException, IOException {
- return get(readOnly, si.dir, si, true, termInfosIndexDivisor, context);
+ public static SegmentReader get(SegmentInfo si, int termInfosIndexDivisor, IOContext context) throws CorruptIndexException, IOException {
+ return get(true, si, true, termInfosIndexDivisor, context);
+ }
+
+ // TODO: remove deletions from SR
+ static SegmentReader getRW(SegmentInfo si, boolean doOpenStores, int termInfosIndexDivisor, IOContext context) throws CorruptIndexException, IOException {
+ return get(false, si, doOpenStores, termInfosIndexDivisor, context);
}
/**
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public static SegmentReader get(boolean readOnly,
- Directory dir,
+ private static SegmentReader get(boolean readOnly,
SegmentInfo si,
boolean doOpenStores,
int termInfosIndexDivisor,
IOContext context)
throws CorruptIndexException, IOException {
- SegmentReader instance = new SegmentReader();
- instance.readOnly = readOnly;
- instance.si = si;
+ SegmentReader instance = new SegmentReader(readOnly, si);
boolean success = false;
-
try {
- instance.core = new SegmentCoreReaders(instance, dir, si, context, termInfosIndexDivisor);
+ instance.core = new SegmentCoreReaders(instance, si.dir, si, context, termInfosIndexDivisor);
if (doOpenStores) {
instance.core.openDocStores(si);
}
instance.loadLiveDocs(context);
- instance.openNorms(instance.core.cfsDir, context);
success = true;
} finally {
@@ -130,6 +120,11 @@ public class SegmentReader extends Index
return instance;
}
+ private SegmentReader(boolean readOnly, SegmentInfo si) {
+ this.readOnly = readOnly;
+ this.si = si;
+ }
+
void openDocStores() throws IOException {
core.openDocStores(si);
}
@@ -158,36 +153,18 @@ public class SegmentReader extends Index
private void loadLiveDocs(IOContext context) throws IOException {
// NOTE: the bitvector is stored using the regular directory, not cfs
- if (hasDeletions(si)) {
+ if (si.hasDeletions()) {
liveDocs = new BitVector(directory(), si.getDelFileName(), new IOContext(context, true));
liveDocsRef = new AtomicInteger(1);
assert checkLiveCounts();
if (liveDocs.size() != si.docCount) {
throw new CorruptIndexException("document count mismatch: deleted docs count " + liveDocs.size() + " vs segment doc count " + si.docCount + " segment=" + si.name);
}
- } else
+ } else {
assert si.getDelCount() == 0;
- }
-
- /**
- * Clones the norm bytes. May be overridden by subclasses. New and experimental.
- * @param bytes Byte array to clone
- * @return New BitVector
- */
- protected byte[] cloneNormBytes(byte[] bytes) {
- byte[] cloneBytes = new byte[bytes.length];
- System.arraycopy(bytes, 0, cloneBytes, 0, bytes.length);
- return cloneBytes;
- }
-
- /**
- * Clones the deleteDocs BitVector. May be overridden by subclasses. New and experimental.
- * @param bv BitVector to clone
- * @return New BitVector
- */
- protected BitVector cloneDeletedDocs(BitVector bv) {
- ensureOpen();
- return (BitVector)bv.clone();
+ }
+ // we need a key reflecting actual deletes (if existent or not):
+ combinedCoreAndDeletesKey = new Object();
}
@Override
@@ -199,175 +176,16 @@ public class SegmentReader extends Index
}
}
- @Override
- public final synchronized IndexReader clone(boolean openReadOnly) throws CorruptIndexException, IOException {
- return reopenSegment(si, true, openReadOnly);
+ // used by DirectoryReader:
+ synchronized SegmentReader reopenSegment(SegmentInfo si, boolean doClone) throws CorruptIndexException, IOException {
+ return reopenSegment(si, doClone, true);
}
@Override
- protected synchronized IndexReader doOpenIfChanged()
- throws CorruptIndexException, IOException {
+ protected synchronized IndexReader doOpenIfChanged() throws CorruptIndexException, IOException {
return reopenSegment(si, false, readOnly);
}
-
- @Override
- protected synchronized IndexReader doOpenIfChanged(boolean openReadOnly)
- throws CorruptIndexException, IOException {
- return reopenSegment(si, false, openReadOnly);
- }
-
- synchronized SegmentReader reopenSegment(SegmentInfo si, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException {
- ensureOpen();
- boolean deletionsUpToDate = (this.si.hasDeletions() == si.hasDeletions())
- && (!si.hasDeletions() || this.si.getDelFileName().equals(si.getDelFileName()));
- boolean normsUpToDate = true;
-
- Set<Integer> fieldNormsChanged = new HashSet<Integer>();
- for (FieldInfo fi : core.fieldInfos) {
- int fieldNumber = fi.number;
- if (!this.si.getNormFileName(fieldNumber).equals(si.getNormFileName(fieldNumber))) {
- normsUpToDate = false;
- fieldNormsChanged.add(fieldNumber);
- }
- }
-
- // if we're cloning we need to run through the reopenSegment logic
- // also if both old and new readers aren't readonly, we clone to avoid sharing modifications
- if (normsUpToDate && deletionsUpToDate && !doClone && openReadOnly && readOnly) {
- return null;
- }
-
- // When cloning, the incoming SegmentInfos should not
- // have any changes in it:
- assert !doClone || (normsUpToDate && deletionsUpToDate);
-
- // clone reader
- SegmentReader clone = new SegmentReader();
-
- boolean success = false;
- try {
- core.incRef();
- clone.core = core;
- clone.readOnly = openReadOnly;
- clone.si = si;
- clone.pendingDeleteCount = pendingDeleteCount;
- clone.readerFinishedListeners = readerFinishedListeners;
-
- if (!openReadOnly && hasChanges) {
- // My pending changes transfer to the new reader
- clone.liveDocsDirty = liveDocsDirty;
- clone.normsDirty = normsDirty;
- clone.hasChanges = hasChanges;
- hasChanges = false;
- }
-
- if (doClone) {
- if (liveDocs != null) {
- liveDocsRef.incrementAndGet();
- clone.liveDocs = liveDocs;
- clone.liveDocsRef = liveDocsRef;
- }
- } else {
- if (!deletionsUpToDate) {
- // load deleted docs
- assert clone.liveDocs == null;
- clone.loadLiveDocs(IOContext.READ);
- } else if (liveDocs != null) {
- liveDocsRef.incrementAndGet();
- clone.liveDocs = liveDocs;
- clone.liveDocsRef = liveDocsRef;
- }
- }
-
- clone.norms = new HashMap<String,SegmentNorms>();
-
- // Clone norms
- for (FieldInfo fi : core.fieldInfos) {
- // Clone unchanged norms to the cloned reader
- if (doClone || !fieldNormsChanged.contains(fi.number)) {
- final String curField = fi.name;
- SegmentNorms norm = this.norms.get(curField);
- if (norm != null)
- clone.norms.put(curField, (SegmentNorms) norm.clone());
- }
- }
-
- // If we are not cloning, then this will open anew
- // any norms that have changed:
- clone.openNorms(si.getUseCompoundFile() ? core.getCFSReader() : directory(), IOContext.DEFAULT);
-
- success = true;
- } finally {
- if (!success) {
- // An exception occurred during reopen, we have to decRef the norms
- // that we incRef'ed already and close singleNormsStream and FieldsReader
- clone.decRef();
- }
- }
-
- return clone;
- }
-
- @Override
- protected void doCommit(Map<String,String> commitUserData) throws IOException {
- if (hasChanges) {
- startCommit();
- boolean success = false;
- try {
- commitChanges(commitUserData);
- success = true;
- } finally {
- if (!success) {
- rollbackCommit();
- }
- }
- }
- }
-
- private synchronized void commitChanges(Map<String,String> commitUserData) throws IOException {
- if (liveDocsDirty) { // re-write deleted
- si.advanceDelGen();
-
- assert liveDocs.length() == si.docCount;
-
- // We can write directly to the actual name (vs to a
- // .tmp & renaming it) because the file is not live
- // until segments file is written:
- final String delFileName = si.getDelFileName();
- boolean success = false;
- try {
- liveDocs.write(directory(), delFileName, IOContext.DEFAULT);
- success = true;
- } finally {
- if (!success) {
- try {
- directory().deleteFile(delFileName);
- } catch (Throwable t) {
- // suppress this so we keep throwing the
- // original exception
- }
- }
- }
- si.setDelCount(si.getDelCount()+pendingDeleteCount);
- pendingDeleteCount = 0;
- assert (maxDoc()-liveDocs.count()) == si.getDelCount(): "delete count mismatch during commit: info=" + si.getDelCount() + " vs BitVector=" + (maxDoc()-liveDocs.count());
- } else {
- assert pendingDeleteCount == 0;
- }
-
- if (normsDirty) { // re-write norms
- si.initNormGen();
- for (final SegmentNorms norm : norms.values()) {
- if (norm.dirty) {
- norm.reWrite(si);
- }
- }
- }
- liveDocsDirty = false;
- normsDirty = false;
- hasChanges = false;
- }
-
+
/** @lucene.internal */
public StoredFieldsReader getFieldsReader() {
return fieldsReaderLocal.get();
@@ -375,6 +193,10 @@ public class SegmentReader extends Index
@Override
protected void doClose() throws IOException {
+ if (hasChanges) {
+ doCommit();
+ }
+
termVectorsLocal.close();
fieldsReaderLocal.close();
@@ -384,72 +206,17 @@ public class SegmentReader extends Index
liveDocs = null;
}
- for (final SegmentNorms norm : norms.values()) {
- norm.decRef();
- }
if (core != null) {
core.decRef();
}
}
- static boolean hasDeletions(SegmentInfo si) throws IOException {
- // Don't call ensureOpen() here (it could affect performance)
- return si.hasDeletions();
- }
-
@Override
public boolean hasDeletions() {
// Don't call ensureOpen() here (it could affect performance)
return liveDocs != null;
}
- static boolean usesCompoundFile(SegmentInfo si) throws IOException {
- return si.getUseCompoundFile();
- }
-
- static boolean hasSeparateNorms(SegmentInfo si) throws IOException {
- return si.hasSeparateNorms();
- }
-
- @Override
- protected void doDelete(int docNum) {
- if (liveDocs == null) {
- liveDocs = new BitVector(maxDoc());
- liveDocs.setAll();
- liveDocsRef = new AtomicInteger(1);
- }
- // there is more than 1 SegmentReader with a reference to this
- // liveDocs BitVector so decRef the current liveDocsRef,
- // clone the BitVector, create a new liveDocsRef
- if (liveDocsRef.get() > 1) {
- AtomicInteger oldRef = liveDocsRef;
- liveDocs = cloneDeletedDocs(liveDocs);
- liveDocsRef = new AtomicInteger(1);
- oldRef.decrementAndGet();
- }
- liveDocsDirty = true;
- if (liveDocs.getAndClear(docNum)) {
- pendingDeleteCount++;
- }
- }
-
- @Override
- protected void doUndeleteAll() {
- liveDocsDirty = false;
- if (liveDocs != null) {
- assert liveDocsRef != null;
- liveDocsRef.decrementAndGet();
- liveDocs = null;
- liveDocsRef = null;
- pendingDeleteCount = 0;
- si.clearDelGen();
- si.setDelCount(0);
- } else {
- assert liveDocsRef == null;
- assert pendingDeleteCount == 0;
- }
- }
-
List<String> files() throws IOException {
return new ArrayList<String>(si.files());
}
@@ -547,107 +314,14 @@ public class SegmentReader extends Index
@Override
public boolean hasNorms(String field) {
ensureOpen();
- return norms.containsKey(field);
+ FieldInfo fi = core.fieldInfos.fieldInfo(field);
+ return fi != null && fi.isIndexed && !fi.omitNorms;
}
@Override
public byte[] norms(String field) throws IOException {
ensureOpen();
- final SegmentNorms norm = norms.get(field);
- if (norm == null) {
- // not indexed, or norms not stored
- return null;
- }
- return norm.bytes();
- }
-
- @Override
- protected void doSetNorm(int doc, String field, byte value)
- throws IOException {
- SegmentNorms norm = norms.get(field);
- if (norm == null) {
- // field does not store norms
- throw new IllegalStateException("Cannot setNorm for field " + field + ": norms were omitted");
- }
-
- normsDirty = true;
- norm.copyOnWrite()[doc] = value; // set the value
- }
-
- private void openNorms(Directory cfsDir, IOContext context) throws IOException {
- boolean normsInitiallyEmpty = norms.isEmpty(); // only used for assert
- long nextNormSeek = SegmentNorms.NORMS_HEADER.length; //skip header (header unused for now)
- int maxDoc = maxDoc();
- for (FieldInfo fi : core.fieldInfos) {
- if (norms.containsKey(fi.name)) {
- // in case this SegmentReader is being re-opened, we might be able to
- // reuse some norm instances and skip loading them here
- continue;
- }
- if (fi.isIndexed && !fi.omitNorms) {
- Directory d = directory();
- String fileName = si.getNormFileName(fi.number);
- if (!si.hasSeparateNorms(fi.number)) {
- d = cfsDir;
- }
-
- // singleNormFile means multiple norms share this file
- boolean singleNormFile = IndexFileNames.matchesExtension(fileName, IndexFileNames.NORMS_EXTENSION);
- IndexInput normInput = null;
- long normSeek;
-
- if (singleNormFile) {
- normSeek = nextNormSeek;
- if (singleNormStream == null) {
- singleNormStream = d.openInput(fileName, context);
- singleNormRef = new AtomicInteger(1);
- } else {
- singleNormRef.incrementAndGet();
- }
- // All norms in the .nrm file can share a single IndexInput since
- // they are only used in a synchronized context.
- // If this were to change in the future, a clone could be done here.
- normInput = singleNormStream;
- } else {
- normInput = d.openInput(fileName, context);
- // if the segment was created in 3.2 or after, we wrote the header for sure,
- // and don't need to do the sketchy file size check. otherwise, we check
- // if the size is exactly equal to maxDoc to detect a headerless file.
- // NOTE: remove this check in Lucene 5.0!
- String version = si.getVersion();
- final boolean isUnversioned =
- (version == null || StringHelper.getVersionComparator().compare(version, "3.2") < 0)
- && normInput.length() == maxDoc();
- if (isUnversioned) {
- normSeek = 0;
- } else {
- normSeek = SegmentNorms.NORMS_HEADER.length;
- }
- }
-
- norms.put(fi.name, new SegmentNorms(normInput, fi.number, normSeek, this));
- nextNormSeek += maxDoc; // increment also if some norms are separate
- }
- }
- assert singleNormStream == null || !normsInitiallyEmpty || nextNormSeek == singleNormStream.length();
- }
-
- // for testing only
- boolean normsClosed() {
- if (singleNormStream != null) {
- return false;
- }
- for (final SegmentNorms norm : norms.values()) {
- if (norm.refCount > 0) {
- return false;
- }
- }
- return true;
- }
-
- // for testing only
- boolean normsClosed(String field) {
- return norms.get(field).refCount == 0;
+ return core.norms.norms(field);
}
/**
@@ -689,7 +363,6 @@ public class SegmentReader extends Index
return termVectorsReader.get(docID);
}
- /** {@inheritDoc} */
@Override
public String toString() {
final StringBuilder buffer = new StringBuilder();
@@ -724,28 +397,6 @@ public class SegmentReader extends Index
si = info;
}
- void startCommit() {
- rollbackSegmentInfo = (SegmentInfo) si.clone();
- rollbackHasChanges = hasChanges;
- rollbackDeletedDocsDirty = liveDocsDirty;
- rollbackNormsDirty = normsDirty;
- rollbackPendingDeleteCount = pendingDeleteCount;
- for (SegmentNorms norm : norms.values()) {
- norm.rollbackDirty = norm.dirty;
- }
- }
-
- void rollbackCommit() {
- si.reset(rollbackSegmentInfo);
- hasChanges = rollbackHasChanges;
- liveDocsDirty = rollbackDeletedDocsDirty;
- normsDirty = rollbackNormsDirty;
- pendingDeleteCount = rollbackPendingDeleteCount;
- for (SegmentNorms norm : norms.values()) {
- norm.dirty = norm.rollbackDirty;
- }
- }
-
/** Returns the directory this index resides in. */
@Override
public Directory directory() {
@@ -764,23 +415,228 @@ public class SegmentReader extends Index
}
@Override
+ public Object getCombinedCoreAndDeletesKey() {
+ return combinedCoreAndDeletesKey;
+ }
+
+ @Override
public int getTermInfosIndexDivisor() {
return core.termsIndexDivisor;
}
-
+
@Override
- protected void readerFinished() {
- // Do nothing here -- we have more careful control on
- // when to notify that a SegmentReader has finished,
- // because a given core is shared across many cloned
- // SegmentReaders. We only notify once that core is no
- // longer used (all SegmentReaders sharing it have been
- // closed).
+ public DocValues docValues(String field) throws IOException {
+ ensureOpen();
+ final PerDocProducer perDoc = core.perDocProducer;
+ if (perDoc == null) {
+ return null;
+ }
+ return perDoc.docValues(field);
+ }
+
+ /**
+ * Clones the deleteDocs BitVector. May be overridden by subclasses. New and experimental.
+ * @param bv BitVector to clone
+ * @return New BitVector
+ */
+ // TODO: remove deletions from SR
+ BitVector cloneDeletedDocs(BitVector bv) {
+ ensureOpen();
+ return (BitVector)bv.clone();
}
- @Override
- public PerDocValues perDocValues() throws IOException {
+ // TODO: remove deletions from SR
+ final synchronized IndexReader clone(boolean openReadOnly) throws CorruptIndexException, IOException {
+ return reopenSegment(si, true, openReadOnly);
+ }
+
+ // TODO: remove deletions from SR
+ private synchronized SegmentReader reopenSegment(SegmentInfo si, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException {
+ ensureOpen();
+ boolean deletionsUpToDate = (this.si.hasDeletions() == si.hasDeletions())
+ && (!si.hasDeletions() || this.si.getDelFileName().equals(si.getDelFileName()));
+
+ // if we're cloning we need to run through the reopenSegment logic
+ // also if both old and new readers aren't readonly, we clone to avoid sharing modifications
+ if (deletionsUpToDate && !doClone && openReadOnly && readOnly) {
+ return null;
+ }
+
+ // When cloning, the incoming SegmentInfos should not
+ // have any changes in it:
+ assert !doClone || (deletionsUpToDate);
+
+ // clone reader
+ SegmentReader clone = new SegmentReader(openReadOnly, si);
+
+ boolean success = false;
+ try {
+ core.incRef();
+ clone.core = core;
+ clone.pendingDeleteCount = pendingDeleteCount;
+ clone.combinedCoreAndDeletesKey = combinedCoreAndDeletesKey;
+
+ if (!openReadOnly && hasChanges) {
+ // My pending changes transfer to the new reader
+ clone.liveDocsDirty = liveDocsDirty;
+ clone.hasChanges = hasChanges;
+ hasChanges = false;
+ }
+
+ if (doClone) {
+ if (liveDocs != null) {
+ liveDocsRef.incrementAndGet();
+ clone.liveDocs = liveDocs;
+ clone.liveDocsRef = liveDocsRef;
+ }
+ } else {
+ if (!deletionsUpToDate) {
+ // load deleted docs
+ assert clone.liveDocs == null;
+ clone.loadLiveDocs(IOContext.READ);
+ } else if (liveDocs != null) {
+ liveDocsRef.incrementAndGet();
+ clone.liveDocs = liveDocs;
+ clone.liveDocsRef = liveDocsRef;
+ }
+ }
+ success = true;
+ } finally {
+ if (!success) {
+ // An exception occurred during reopen, we have to decRef the norms
+ // that we incRef'ed already and close singleNormsStream and FieldsReader
+ clone.decRef();
+ }
+ }
+
+ return clone;
+ }
+
+ // TODO: remove deletions from SR
+ void doCommit() throws IOException {
+ assert hasChanges;
+ startCommit();
+ boolean success = false;
+ try {
+ commitChanges();
+ success = true;
+ } finally {
+ if (!success) {
+ rollbackCommit();
+ }
+ }
+ }
+
+ // TODO: remove deletions from SR
+ private void startCommit() {
+ rollbackSegmentInfo = (SegmentInfo) si.clone();
+ rollbackHasChanges = hasChanges;
+ rollbackDeletedDocsDirty = liveDocsDirty;
+ rollbackPendingDeleteCount = pendingDeleteCount;
+ }
+
+ // TODO: remove deletions from SR
+ private void rollbackCommit() {
+ si.reset(rollbackSegmentInfo);
+ hasChanges = rollbackHasChanges;
+ liveDocsDirty = rollbackDeletedDocsDirty;
+ pendingDeleteCount = rollbackPendingDeleteCount;
+ }
+
+ // TODO: remove deletions from SR
+ private synchronized void commitChanges() throws IOException {
+ if (liveDocsDirty) { // re-write deleted
+ si.advanceDelGen();
+
+ assert liveDocs.length() == si.docCount;
+
+ // We can write directly to the actual name (vs to a
+ // .tmp & renaming it) because the file is not live
+ // until segments file is written:
+ final String delFileName = si.getDelFileName();
+ boolean success = false;
+ try {
+ liveDocs.write(directory(), delFileName, IOContext.DEFAULT);
+ success = true;
+ } finally {
+ if (!success) {
+ try {
+ directory().deleteFile(delFileName);
+ } catch (Throwable t) {
+ // suppress this so we keep throwing the
+ // original exception
+ }
+ }
+ }
+ si.setDelCount(si.getDelCount()+pendingDeleteCount);
+ pendingDeleteCount = 0;
+ assert (maxDoc()-liveDocs.count()) == si.getDelCount(): "delete count mismatch during commit: info=" + si.getDelCount() + " vs BitVector=" + (maxDoc()-liveDocs.count());
+ } else {
+ assert pendingDeleteCount == 0;
+ }
+
+ liveDocsDirty = false;
+ hasChanges = false;
+ }
+
+ // TODO: remove deletions from SR
+ synchronized void deleteDocument(int docNum) throws IOException {
+ ensureOpen();
+ hasChanges = true;
+ doDelete(docNum);
+ }
+
+ // TODO: remove deletions from SR
+ void doDelete(int docNum) {
+ if (liveDocs == null) {
+ liveDocs = new BitVector(maxDoc());
+ liveDocs.setAll();
+ liveDocsRef = new AtomicInteger(1);
+ }
+ // there is more than 1 SegmentReader with a reference to this
+ // liveDocs BitVector so decRef the current liveDocsRef,
+ // clone the BitVector, create a new liveDocsRef
+ if (liveDocsRef.get() > 1) {
+ AtomicInteger oldRef = liveDocsRef;
+ liveDocs = cloneDeletedDocs(liveDocs);
+ liveDocsRef = new AtomicInteger(1);
+ oldRef.decrementAndGet();
+ }
+ // we need a key reflecting actual deletes (if existent or not):
+ combinedCoreAndDeletesKey = new Object();
+ // liveDocs are now dirty:
+ liveDocsDirty = true;
+ if (liveDocs.getAndClear(docNum)) {
+ pendingDeleteCount++;
+ }
+ }
+
+ /**
+ * Called when the shared core for this SegmentReader
+ * is closed.
+ * <p>
+ * This listener is called only once all SegmentReaders
+ * sharing the same core are closed. At this point it
+ * is safe for apps to evict this reader from any caches
+ * keyed on {@link #getCoreCacheKey}. This is the same
+ * interface that {@link FieldCache} uses, internally,
+ * to evict entries.</p>
+ *
+ * @lucene.experimental
+ */
+ public static interface CoreClosedListener {
+ public void onClose(SegmentReader owner);
+ }
+
+ /** Expert: adds a CoreClosedListener to this reader's shared core */
+ public void addCoreClosedListener(CoreClosedListener listener) {
+ ensureOpen();
+ core.coreClosedListeners.add(listener);
+ }
+
+ /** Expert: removes a CoreClosedListener from this reader's shared core */
+ public void removeCoreClosedListener(CoreClosedListener listener) {
ensureOpen();
- return core.perDocProducer;
+ core.coreClosedListeners.remove(listener);
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java Thu Dec 15 05:11:14 2011
@@ -26,7 +26,6 @@ import org.apache.lucene.util.ReaderUtil
import org.apache.lucene.index.DirectoryReader; // javadoc
import org.apache.lucene.index.MultiReader; // javadoc
-import org.apache.lucene.index.codecs.PerDocValues;
/**
* This class forces a composite reader (eg a {@link
@@ -35,11 +34,12 @@ import org.apache.lucene.index.codecs.Pe
* IndexReader#getSequentialSubReaders}) to emulate an
* atomic reader. This requires implementing the postings
* APIs on-the-fly, using the static methods in {@link
- * MultiFields}, by stepping through the sub-readers to
- * merge fields/terms, appending docs, etc.
+ * MultiFields}, {@link MultiNorms}, {@link MultiDocValues},
+ * by stepping through the sub-readers to merge fields/terms,
+ * appending docs, etc.
*
* <p>If you ever hit an UnsupportedOperationException saying
- * "please use MultiFields.XXX instead", the simple
+ * "please use MultiXXX.YYY instead", the simple
* but non-performant workaround is to wrap your reader
* using this class.</p>
*
@@ -72,9 +72,9 @@ public final class SlowMultiReaderWrappe
}
@Override
- public PerDocValues perDocValues() throws IOException {
+ public DocValues docValues(String field) throws IOException {
ensureOpen();
- return MultiPerDocValues.getPerDocs(in);
+ return MultiDocValues.getDocValues(in, field);
}
@Override
@@ -109,13 +109,4 @@ public final class SlowMultiReaderWrappe
ensureOpen();
return readerContext;
}
-
- @Override
- protected void doSetNorm(int n, String field, byte value)
- throws CorruptIndexException, IOException {
- synchronized(normsCache) {
- normsCache.remove(field);
- }
- in.doSetNorm(n, field, value);
- }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java Thu Dec 15 05:11:14 2011
@@ -113,7 +113,6 @@ final class TermVectorsConsumerPerField
// of a given field in the doc. At this point we flush
// our hash into the DocWriter.
- assert fieldInfo.storeTermVector;
assert termsWriter.vectorFieldsInOrder(fieldInfo);
TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
@@ -150,8 +149,9 @@ final class TermVectorsConsumerPerField
}
termsHashPerField.reset();
+
// commit the termVectors once successful success - FI will otherwise reset them
- fieldInfo.commitVectors();
+ fieldInfo.setStoreTermVectors(doVectorPositions, doVectorOffsets);
}
void shrinkHash() {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java Thu Dec 15 05:11:14 2011
@@ -26,7 +26,6 @@ import java.util.Comparator;
import java.util.List;
import java.util.ArrayList;
-import org.apache.lucene.util.InfoStream;
/**
* Merges segments of approximately equal size, subject to
@@ -346,7 +345,9 @@ public class TieredMergePolicy extends M
final boolean maxMergeIsRunning = mergingBytes >= maxMergedSegmentBytes;
- message(" allowedSegmentCount=" + allowedSegCountInt + " vs count=" + infosSorted.size() + " (eligible count=" + eligible.size() + ") tooBigCount=" + tooBigCount);
+ if (verbose()) {
+ message(" allowedSegmentCount=" + allowedSegCountInt + " vs count=" + infosSorted.size() + " (eligible count=" + eligible.size() + ") tooBigCount=" + tooBigCount);
+ }
if (eligible.size() == 0) {
return spec;
@@ -386,7 +387,9 @@ public class TieredMergePolicy extends M
}
final MergeScore score = score(candidate, hitTooLarge, mergingBytes);
- message(" maybe=" + writer.get().segString(candidate) + " score=" + score.getScore() + " " + score.getExplanation() + " tooLarge=" + hitTooLarge + " size=" + String.format("%.3f MB", totAfterMergeBytes/1024./1024.));
+ if (verbose()) {
+ message(" maybe=" + writer.get().segString(candidate) + " score=" + score.getScore() + " " + score.getExplanation() + " tooLarge=" + hitTooLarge + " size=" + String.format("%.3f MB", totAfterMergeBytes/1024./1024.));
+ }
// If we are already running a max sized merge
// (maxMergeIsRunning), don't allow another max
@@ -649,9 +652,7 @@ public class TieredMergePolicy extends M
}
private void message(String message) {
- if (verbose()) {
- writer.get().infoStream.message("TMP", message);
- }
+ writer.get().infoStream.message("TMP", message);
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java Thu Dec 15 05:11:14 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
*/
import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.InfoStream;
import java.io.IOException;
import java.util.ArrayList;
@@ -90,7 +89,9 @@ public class UpgradeIndexMergePolicy ext
}
}
- if (verbose()) message("findForcedMerges: segmentsToUpgrade=" + oldSegments);
+ if (verbose()) {
+ message("findForcedMerges: segmentsToUpgrade=" + oldSegments);
+ }
if (oldSegments.isEmpty())
return null;
@@ -107,9 +108,10 @@ public class UpgradeIndexMergePolicy ext
}
if (!oldSegments.isEmpty()) {
- if (verbose())
+ if (verbose()) {
message("findForcedMerges: " + base.getClass().getSimpleName() +
" does not want to merge all old segments, merge remaining ones into new segment: " + oldSegments);
+ }
final List<SegmentInfo> newInfos = new ArrayList<SegmentInfo>();
for (final SegmentInfo si : segmentInfos) {
if (oldSegments.containsKey(si)) {
@@ -152,9 +154,6 @@ public class UpgradeIndexMergePolicy ext
}
private void message(String message) {
- if (verbose()) {
- writer.get().infoStream.message("UPGMP", message);
- }
+ writer.get().infoStream.message("UPGMP", message);
}
-
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java Thu Dec 15 05:11:14 2011
@@ -51,6 +51,7 @@ public abstract class Codec implements N
// TODO: segmentInfosFormat should be allowed to declare additional files
// if it wants, in addition to segments_N
docValuesFormat().files(dir, info, files);
+ normsFormat().files(dir, info, files);
}
/** Encodes/decodes postings */
@@ -71,6 +72,9 @@ public abstract class Codec implements N
/** Encodes/decodes segments file */
public abstract SegmentInfosFormat segmentInfosFormat();
+ /** Encodes/decodes document normalization values */
+ public abstract NormsFormat normsFormat();
+
/** looks up a codec by name */
public static Codec forName(String name) {
return loader.lookup(name);
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java Thu Dec 15 05:11:14 2011
@@ -17,77 +17,52 @@ package org.apache.lucene.index.codecs;
* limitations under the License.
*/
import java.io.IOException;
-import java.util.Collection;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MergeState;
-import org.apache.lucene.index.values.IndexDocValues;
-import org.apache.lucene.index.values.PerDocFieldValues;
-import org.apache.lucene.index.values.Writer;
+import org.apache.lucene.index.DocValue;
import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.Counter;
/**
- * Abstract API that consumes {@link PerDocFieldValues}.
+ * Abstract API that consumes {@link DocValue}s.
* {@link DocValuesConsumer} are always associated with a specific field and
* segments. Concrete implementations of this API write the given
- * {@link PerDocFieldValues} into a implementation specific format depending on
+ * {@link DocValue} into a implementation specific format depending on
* the fields meta-data.
*
* @lucene.experimental
*/
public abstract class DocValuesConsumer {
- protected final Counter bytesUsed;
-
- /**
- * Creates a new {@link DocValuesConsumer}.
- *
- * @param bytesUsed
- * bytes-usage tracking reference used by implementation to track
- * internally allocated memory. All tracked bytes must be released
- * once {@link #finish(int)} has been called.
- */
- protected DocValuesConsumer(Counter bytesUsed) {
- this.bytesUsed = bytesUsed == null ? Counter.newCounter() : bytesUsed;
- }
-
/**
- * Adds the given {@link PerDocFieldValues} instance to this
+ * Adds the given {@link DocValue} instance to this
* {@link DocValuesConsumer}
*
* @param docID
* the document ID to add the value for. The docID must always
* increase or be <tt>0</tt> if it is the first call to this method.
- * @param docValues
- * the values to add
+ * @param docValue
+ * the value to add
* @throws IOException
* if an {@link IOException} occurs
*/
- public abstract void add(int docID, PerDocFieldValues docValues)
+ public abstract void add(int docID, DocValue docValue)
throws IOException;
/**
* Called when the consumer of this API is doc with adding
- * {@link PerDocFieldValues} to this {@link DocValuesConsumer}
+ * {@link DocValue} to this {@link DocValuesConsumer}
*
* @param docCount
* the total number of documents in this {@link DocValuesConsumer}.
* Must be greater than or equal the last given docID to
- * {@link #add(int, PerDocFieldValues)}.
+ * {@link #add(int, DocValue)}.
* @throws IOException
*/
public abstract void finish(int docCount) throws IOException;
/**
- * Gathers files associated with this {@link DocValuesConsumer}
- *
- * @param files
- * the of files to add the consumers files to.
- */
- public abstract void files(Collection<String> files) throws IOException;
-
- /**
* Merges the given {@link org.apache.lucene.index.MergeState} into
* this {@link DocValuesConsumer}.
*
@@ -95,18 +70,18 @@ public abstract class DocValuesConsumer
* the state to merge
* @param docValues docValues array containing one instance per reader (
* {@link org.apache.lucene.index.MergeState#readers}) or <code>null</code> if the reader has
- * no {@link IndexDocValues} instance.
+ * no {@link DocValues} instance.
* @throws IOException
* if an {@link IOException} occurs
*/
- public void merge(MergeState mergeState, IndexDocValues[] docValues) throws IOException {
+ public void merge(MergeState mergeState, DocValues[] docValues) throws IOException {
assert mergeState != null;
boolean hasMerged = false;
for(int readerIDX=0;readerIDX<mergeState.readers.size();readerIDX++) {
final org.apache.lucene.index.MergeState.IndexReaderAndLiveDocs reader = mergeState.readers.get(readerIDX);
if (docValues[readerIDX] != null) {
hasMerged = true;
- merge(new Writer.SingleSubMergeState(docValues[readerIDX], mergeState.docBase[readerIDX], reader.reader.maxDoc(),
+ merge(new SingleSubMergeState(docValues[readerIDX], mergeState.docBase[readerIDX], reader.reader.maxDoc(),
reader.liveDocs));
}
}
@@ -124,6 +99,9 @@ public abstract class DocValuesConsumer
* @throws IOException
* if an {@link IOException} occurs
*/
+ // TODO: can't we have a default implementation here that merges naively with our apis?
+ // this is how stored fields and term vectors work. its a pain to have to impl merging
+ // (should be an optimization to override it)
protected abstract void merge(SingleSubMergeState mergeState) throws IOException;
/**
@@ -137,7 +115,7 @@ public abstract class DocValuesConsumer
* the source reader for this MergeState - merged values should be read from
* this instance
*/
- public final IndexDocValues reader;
+ public final DocValues reader;
/** the absolute docBase for this MergeState within the resulting segment */
public final int docBase;
/** the number of documents in this MergeState */
@@ -145,7 +123,7 @@ public abstract class DocValuesConsumer
/** the not deleted bits for this MergeState */
public final Bits liveDocs;
- public SingleSubMergeState(IndexDocValues reader, int docBase, int docCount, Bits liveDocs) {
+ public SingleSubMergeState(DocValues reader, int docBase, int docCount, Bits liveDocs) {
assert reader != null;
this.reader = reader;
this.docBase = docBase;
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java Thu Dec 15 05:11:14 2011
@@ -27,6 +27,6 @@ import org.apache.lucene.store.Directory
public abstract class DocValuesFormat {
public abstract PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException;
- public abstract PerDocValues docsProducer(SegmentReadState state) throws IOException;
+ public abstract PerDocProducer docsProducer(SegmentReadState state) throws IOException;
public abstract void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java Thu Dec 15 05:11:14 2011
@@ -26,23 +26,24 @@ import java.util.TreeMap;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.values.Bytes;
-import org.apache.lucene.index.values.Floats;
-import org.apache.lucene.index.values.IndexDocValues;
-import org.apache.lucene.index.values.Ints;
-import org.apache.lucene.index.values.ValueType;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValues.Type; // javadocs
+import org.apache.lucene.index.codecs.lucene40.values.Bytes;
+import org.apache.lucene.index.codecs.lucene40.values.Floats;
+import org.apache.lucene.index.codecs.lucene40.values.Ints;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BytesRef;
/**
- * Abstract base class for PerDocValues implementations
+ * Abstract base class for PerDocProducer implementations
* @lucene.experimental
*/
-public abstract class DocValuesReaderBase extends PerDocValues {
+// TODO: this needs to go under lucene40 codec (its specific to its impl)
+public abstract class DocValuesReaderBase extends PerDocProducer {
protected abstract void closeInternal(Collection<? extends Closeable> closeables) throws IOException;
- protected abstract Map<String, IndexDocValues> docValues();
+ protected abstract Map<String, DocValues> docValues();
@Override
public void close() throws IOException {
@@ -50,24 +51,19 @@ public abstract class DocValuesReaderBas
}
@Override
- public IndexDocValues docValues(String field) throws IOException {
+ public DocValues docValues(String field) throws IOException {
return docValues().get(field);
}
-
- @Override
- public Collection<String> fields() {
- return docValues().keySet();
- }
public Comparator<BytesRef> getComparator() throws IOException {
return BytesRef.getUTF8SortedAsUnicodeComparator();
}
// Only opens files... doesn't actually load any values
- protected TreeMap<String, IndexDocValues> load(FieldInfos fieldInfos,
+ protected TreeMap<String, DocValues> load(FieldInfos fieldInfos,
String segment, int docCount, Directory dir, IOContext context)
throws IOException {
- TreeMap<String, IndexDocValues> values = new TreeMap<String, IndexDocValues>();
+ TreeMap<String, DocValues> values = new TreeMap<String, DocValues>();
boolean success = false;
try {
@@ -79,7 +75,7 @@ public abstract class DocValuesReaderBas
final String id = DocValuesWriterBase.docValuesId(segment,
fieldInfo.number);
values.put(field,
- loadDocValues(docCount, dir, id, fieldInfo.getDocValues(), context));
+ loadDocValues(docCount, dir, id, fieldInfo.getDocValuesType(), context));
}
}
success = true;
@@ -93,26 +89,26 @@ public abstract class DocValuesReaderBas
}
/**
- * Loads a {@link IndexDocValues} instance depending on the given {@link ValueType}.
- * Codecs that use different implementations for a certain {@link ValueType} can
+ * Loads a {@link DocValues} instance depending on the given {@link Type}.
+ * Codecs that use different implementations for a certain {@link Type} can
* simply override this method and return their custom implementations.
*
* @param docCount
* number of documents in the segment
* @param dir
- * the {@link Directory} to load the {@link IndexDocValues} from
+ * the {@link Directory} to load the {@link DocValues} from
* @param id
* the unique file ID within the segment
* @param type
* the type to load
- * @return a {@link IndexDocValues} instance for the given type
+ * @return a {@link DocValues} instance for the given type
* @throws IOException
* if an {@link IOException} occurs
* @throws IllegalArgumentException
- * if the given {@link ValueType} is not supported
+ * if the given {@link Type} is not supported
*/
- protected IndexDocValues loadDocValues(int docCount, Directory dir, String id,
- ValueType type, IOContext context) throws IOException {
+ protected DocValues loadDocValues(int docCount, Directory dir, String id,
+ DocValues.Type type, IOContext context) throws IOException {
switch (type) {
case FIXED_INTS_16:
case FIXED_INTS_32:
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java Thu Dec 15 05:11:14 2011
@@ -22,7 +22,8 @@ import java.util.Comparator;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.PerDocWriteState;
-import org.apache.lucene.index.values.Writer;
+import org.apache.lucene.index.codecs.lucene40.values.Writer;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BytesRef;
@@ -32,6 +33,7 @@ import org.apache.lucene.util.Counter;
* Abstract base class for PerDocConsumer implementations
* @lucene.experimental
*/
+//TODO: this needs to go under lucene40 codec (its specific to its impl)
public abstract class DocValuesWriterBase extends PerDocConsumer {
protected final String segmentName;
protected final String segmentSuffix;
@@ -52,8 +54,8 @@ public abstract class DocValuesWriterBas
}
@Override
- public DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
- return Writer.create(field.getDocValues(),
+ public DocValuesConsumer addValuesField(DocValues.Type valueType, FieldInfo field) throws IOException {
+ return Writer.create(valueType,
docValuesId(segmentName, field.number),
getDirectory(), getComparator(), bytesUsed, context);
}