You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/06/25 01:07:38 UTC
svn commit: r1139458 [4/24] - in /lucene/dev/branches/solr2193: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/
dev-tools/idea/solr/ dev-tools/idea/solr/contrib/clustering/
dev-tools/maven/ dev-tools/maven/modules/analysis/icu/ dev-t...
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/IndexWriter.java Fri Jun 24 23:07:10 2011
@@ -46,15 +46,18 @@ import org.apache.lucene.index.codecs.Co
import org.apache.lucene.search.Query;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.BufferedIndexInput;
+import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util.MapBackedSet;
+import org.apache.lucene.util.TwoPhaseCommit;
/**
An <code>IndexWriter</code> creates and maintains an index.
@@ -190,7 +193,7 @@ import org.apache.lucene.util.MapBackedS
* referenced by the "front" of the index). For this, IndexFileDeleter
* keeps track of the last non commit checkpoint.
*/
-public class IndexWriter implements Closeable {
+public class IndexWriter implements Closeable, TwoPhaseCommit {
/**
* Name of the write lock in the index.
*/
@@ -235,7 +238,7 @@ public class IndexWriter implements Clos
private DocumentsWriter docWriter;
final IndexFileDeleter deleter;
- private Set<SegmentInfo> segmentsToOptimize = new HashSet<SegmentInfo>(); // used by optimize to note those needing optimization
+ private Map<SegmentInfo,Boolean> segmentsToOptimize = new HashMap<SegmentInfo,Boolean>(); // used by optimize to note those needing optimization
private int optimizeMaxNumSegments;
private Lock writeLock;
@@ -1663,7 +1666,9 @@ public class IndexWriter implements Clos
synchronized(this) {
resetMergeExceptions();
segmentsToOptimize.clear();
- segmentsToOptimize.addAll(segmentInfos.asSet());
+ for(SegmentInfo info : segmentInfos) {
+ segmentsToOptimize.put(info, Boolean.TRUE);
+ }
optimizeMaxNumSegments = maxNumSegments;
// Now mark all pending & running merges as optimize
@@ -1671,11 +1676,13 @@ public class IndexWriter implements Clos
for(final MergePolicy.OneMerge merge : pendingMerges) {
merge.optimize = true;
merge.maxNumSegmentsOptimize = maxNumSegments;
+ segmentsToOptimize.put(merge.info, Boolean.TRUE);
}
for ( final MergePolicy.OneMerge merge: runningMerges ) {
merge.optimize = true;
merge.maxNumSegmentsOptimize = maxNumSegments;
+ segmentsToOptimize.put(merge.info, Boolean.TRUE);
}
}
@@ -1887,8 +1894,7 @@ public class IndexWriter implements Clos
final MergePolicy.MergeSpecification spec;
if (optimize) {
- spec = mergePolicy.findMergesForOptimize(segmentInfos, maxNumSegmentsOptimize, Collections.unmodifiableSet(segmentsToOptimize));
-
+ spec = mergePolicy.findMergesForOptimize(segmentInfos, maxNumSegmentsOptimize, Collections.unmodifiableMap(segmentsToOptimize));
if (spec != null) {
final int numMerges = spec.merges.size();
for(int i=0;i<numMerges;i++) {
@@ -1999,6 +2005,9 @@ public class IndexWriter implements Clos
// will always write to a new generation ("write
// once").
segmentInfos.rollbackSegmentInfos(rollbackSegments);
+ if (infoStream != null ) {
+ message("rollback: infos=" + segString(segmentInfos));
+ }
docWriter.abort();
@@ -2183,13 +2192,19 @@ public class IndexWriter implements Clos
String compoundFileName = IndexFileNames.segmentFileName(newSegment.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION);
message("creating compound file " + compoundFileName);
// Now build compound file
- CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, compoundFileName);
- for(String fileName : newSegment.files()) {
- cfsWriter.addFile(fileName);
+ final Directory cfsDir = directory.createCompoundOutput(compoundFileName);
+ IOException prior = null;
+ try {
+ for(String fileName : newSegment.files()) {
+ directory.copy(cfsDir, fileName, fileName);
+ }
+ } catch(IOException ex) {
+ prior = ex;
+ } finally {
+ IOUtils.closeSafely(prior, cfsDir);
}
-
// Perform the merge
- cfsWriter.close();
+
synchronized(this) {
deleter.deleteNewFiles(newSegment.files());
}
@@ -2439,6 +2454,8 @@ public class IndexWriter implements Clos
flush(false, true);
String mergedName = newSegmentName();
+ // TODO: somehow we should fix this merge so it's
+ // abortable so that IW.close(false) is able to stop it
SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(),
mergedName, null, payloadProcessorProvider,
globalFieldNumberMap.newFieldInfos(SegmentCodecsBuilder.create(codecs)));
@@ -2456,6 +2473,11 @@ public class IndexWriter implements Clos
boolean useCompoundFile;
synchronized(this) { // Guard segmentInfos
+ if (stopMerges) {
+ deleter.deleteNewFiles(info.files());
+ return;
+ }
+ ensureOpen();
useCompoundFile = mergePolicy.useCompoundFile(segmentInfos, info);
}
@@ -2471,6 +2493,11 @@ public class IndexWriter implements Clos
// Register the new segment
synchronized(this) {
+ if (stopMerges) {
+ deleter.deleteNewFiles(info.files());
+ return;
+ }
+ ensureOpen();
segmentInfos.add(info);
checkpoint();
}
@@ -2483,21 +2510,22 @@ public class IndexWriter implements Clos
private void copySegmentIntoCFS(SegmentInfo info, String segName) throws IOException {
String segFileName = IndexFileNames.segmentFileName(segName, "", IndexFileNames.COMPOUND_FILE_EXTENSION);
Collection<String> files = info.files();
- CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, segFileName);
- for (String file : files) {
- String newFileName = segName + IndexFileNames.stripSegmentName(file);
- if (!IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION)
- && !IndexFileNames.isSeparateNormsFile(file)) {
- cfsWriter.addFile(file, info.dir);
- } else {
- assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists";
- info.dir.copy(directory, file, newFileName);
+ final CompoundFileDirectory cfsdir = directory.createCompoundOutput(segFileName);
+ try {
+ for (String file : files) {
+ String newFileName = segName + IndexFileNames.stripSegmentName(file);
+ if (!IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION)
+ && !IndexFileNames.isSeparateNormsFile(file)) {
+ info.dir.copy(cfsdir, file, file);
+ } else {
+ assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists";
+ info.dir.copy(directory, file, newFileName);
+ }
}
+ } finally {
+ IOUtils.closeSafely(true, cfsdir);
}
- // Create the .cfs
- cfsWriter.close();
-
info.dir = directory;
info.name = segName;
info.setUseCompoundFile(true);
@@ -3026,7 +3054,9 @@ public class IndexWriter implements Clos
if (merge.optimize) {
// cascade the optimize:
- segmentsToOptimize.add(merge.info);
+ if (!segmentsToOptimize.containsKey(merge.info)) {
+ segmentsToOptimize.put(merge.info, Boolean.FALSE);
+ }
}
return true;
@@ -3070,12 +3100,13 @@ public class IndexWriter implements Clos
*
* @lucene.experimental
*/
- public final void merge(MergePolicy.OneMerge merge)
+ public void merge(MergePolicy.OneMerge merge)
throws CorruptIndexException, IOException {
boolean success = false;
final long t0 = System.currentTimeMillis();
+ //System.out.println(Thread.currentThread().getName() + ": merge start: size=" + (merge.estimatedMergeBytes/1024./1024.) + " MB\n merge=" + merge.segString(directory) + "\n idx=" + segString());
try {
try {
@@ -3116,6 +3147,7 @@ public class IndexWriter implements Clos
if (infoStream != null && merge.info != null) {
message("merge time " + (System.currentTimeMillis()-t0) + " msec for " + merge.info.docCount + " docs");
}
+ //System.out.println(Thread.currentThread().getName() + ": merge end");
}
/** Hook that's called when the specified merge is complete. */
@@ -3149,7 +3181,7 @@ public class IndexWriter implements Clos
if (info.dir != directory) {
isExternal = true;
}
- if (segmentsToOptimize.contains(info)) {
+ if (segmentsToOptimize.containsKey(info)) {
merge.optimize = true;
merge.maxNumSegmentsOptimize = optimizeMaxNumSegments;
}
@@ -3492,6 +3524,8 @@ public class IndexWriter implements Clos
synchronized(this) {
deleter.deleteFile(compoundFileName);
+
+ deleter.deleteFile(IndexFileNames.segmentFileName(mergedName, "", IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
deleter.deleteNewFiles(merge.info.files());
}
}
@@ -3734,6 +3768,8 @@ public class IndexWriter implements Clos
assert testPoint("midStartCommit");
+ boolean pendingCommitSet = false;
+
try {
// This call can take a long time -- 10s of seconds
// or more. We do it without sync:
@@ -3753,6 +3789,7 @@ public class IndexWriter implements Clos
toSync.prepareCommit(directory);
pendingCommit = toSync;
+ pendingCommitSet = true;
pendingCommitChangeCount = myChangeCount;
}
@@ -3770,7 +3807,7 @@ public class IndexWriter implements Clos
// double-write a segments_N file.
segmentInfos.updateGeneration(toSync);
- if (pendingCommit == null) {
+ if (!pendingCommitSet) {
if (infoStream != null) {
message("hit exception committing segments file");
}
@@ -3846,6 +3883,7 @@ public class IndexWriter implements Clos
}
synchronized boolean nrtIsCurrent(SegmentInfos infos) {
+ //System.out.println("IW.nrtIsCurrent " + (infos.version == segmentInfos.version && !docWriter.anyChanges() && !bufferedDeletesStream.any()));
return infos.version == segmentInfos.version && !docWriter.anyChanges() && !bufferedDeletesStream.any();
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java Fri Jun 24 23:07:10 2011
@@ -21,7 +21,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
-import java.util.Set;
+import java.util.Map;
/** <p>This class implements a {@link MergePolicy} that tries
* to merge segments into levels of exponentially
@@ -201,20 +201,23 @@ public abstract class LogMergePolicy ext
}
}
- protected boolean isOptimized(SegmentInfos infos, int maxNumSegments, Set<SegmentInfo> segmentsToOptimize) throws IOException {
+ protected boolean isOptimized(SegmentInfos infos, int maxNumSegments, Map<SegmentInfo,Boolean> segmentsToOptimize) throws IOException {
final int numSegments = infos.size();
int numToOptimize = 0;
SegmentInfo optimizeInfo = null;
+ boolean segmentIsOriginal = false;
for(int i=0;i<numSegments && numToOptimize <= maxNumSegments;i++) {
final SegmentInfo info = infos.info(i);
- if (segmentsToOptimize.contains(info)) {
+ final Boolean isOriginal = segmentsToOptimize.get(info);
+ if (isOriginal != null) {
+ segmentIsOriginal = isOriginal;
numToOptimize++;
optimizeInfo = info;
}
}
return numToOptimize <= maxNumSegments &&
- (numToOptimize != 1 || isOptimized(optimizeInfo));
+ (numToOptimize != 1 || !segmentIsOriginal || isOptimized(optimizeInfo));
}
/** Returns true if this single info is optimized (has no
@@ -282,7 +285,7 @@ public abstract class LogMergePolicy ext
private MergeSpecification findMergesForOptimizeMaxNumSegments(SegmentInfos infos, int maxNumSegments, int last) throws IOException {
MergeSpecification spec = new MergeSpecification();
final List<SegmentInfo> segments = infos.asList();
-
+
// First, enroll all "full" merges (size
// mergeFactor) to potentially be run concurrently:
while (last - maxNumSegments + 1 >= mergeFactor) {
@@ -346,11 +349,11 @@ public abstract class LogMergePolicy ext
* in use may make use of concurrency. */
@Override
public MergeSpecification findMergesForOptimize(SegmentInfos infos,
- int maxNumSegments, Set<SegmentInfo> segmentsToOptimize) throws IOException {
+ int maxNumSegments, Map<SegmentInfo,Boolean> segmentsToOptimize) throws IOException {
assert maxNumSegments > 0;
if (verbose()) {
- message("findMergesForOptimize: maxNumSegs=" + maxNumSegments + " segsToOptimize= "+ segmentsToOptimize);
+ message("findMergesForOptimize: maxNumSegs=" + maxNumSegments + " segsToOptimize="+ segmentsToOptimize);
}
// If the segments are already optimized (e.g. there's only 1 segment), or
@@ -368,7 +371,7 @@ public abstract class LogMergePolicy ext
int last = infos.size();
while (last > 0) {
final SegmentInfo info = infos.info(--last);
- if (segmentsToOptimize.contains(info)) {
+ if (segmentsToOptimize.get(info) != null) {
last++;
break;
}
@@ -398,7 +401,7 @@ public abstract class LogMergePolicy ext
break;
}
}
-
+
if (anyTooLarge) {
return findMergesForOptimizeSizeLimit(infos, maxNumSegments, last);
} else {
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/MergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/MergePolicy.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/MergePolicy.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/MergePolicy.java Fri Jun 24 23:07:10 2011
@@ -24,7 +24,7 @@ import org.apache.lucene.util.SetOnce.Al
import java.io.IOException;
import java.util.List;
import java.util.ArrayList;
-import java.util.Set;
+import java.util.Map;
/**
* <p>Expert: a MergePolicy determines the sequence of
@@ -51,7 +51,7 @@ import java.util.Set;
* ConcurrentMergeScheduler} they will be run concurrently.</p>
*
* <p>The default MergePolicy is {@link
- * LogByteSizeMergePolicy}.</p>
+ * TieredMergePolicy}.</p>
*
* @lucene.experimental
*/
@@ -297,10 +297,15 @@ public abstract class MergePolicy implem
* is always 1)
* @param segmentsToOptimize
* contains the specific SegmentInfo instances that must be merged
- * away. This may be a subset of all SegmentInfos.
+ * away. This may be a subset of all
+ * SegmentInfos. If the value is True for a
+ * given SegmentInfo, that means this segment was
+ * an original segment present in the
+ * to-be-optimized index; else, it was a segment
+ * produced by a cascaded merge.
*/
public abstract MergeSpecification findMergesForOptimize(
- SegmentInfos segmentInfos, int maxSegmentCount, Set<SegmentInfo> segmentsToOptimize)
+ SegmentInfos segmentInfos, int maxSegmentCount, Map<SegmentInfo,Boolean> segmentsToOptimize)
throws CorruptIndexException, IOException;
/**
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/MultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/MultiFields.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/MultiFields.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/MultiFields.java Fri Jun 24 23:07:10 2011
@@ -18,15 +18,16 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.Map;
-import java.util.List;
import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.lucene.util.ReaderUtil;
-import org.apache.lucene.util.ReaderUtil.Gather; // for javadocs
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.MultiBits;
+import org.apache.lucene.util.ReaderUtil.Gather; // for javadocs
+import org.apache.lucene.util.ReaderUtil;
/**
* Exposes flex API, merged from flex API of sub-segments.
@@ -99,49 +100,11 @@ public final class MultiFields extends F
}
}
- private static class MultiReaderBits implements Bits {
- private final int[] starts;
- private final IndexReader[] readers;
- private final Bits[] delDocs;
-
- public MultiReaderBits(int[] starts, IndexReader[] readers) {
- assert readers.length == starts.length-1;
- this.starts = starts;
- this.readers = readers;
- delDocs = new Bits[readers.length];
- for(int i=0;i<readers.length;i++) {
- delDocs[i] = readers[i].getDeletedDocs();
- }
- }
-
- public boolean get(int doc) {
- final int sub = ReaderUtil.subIndex(doc, starts);
- Bits dels = delDocs[sub];
- if (dels == null) {
- // NOTE: this is not sync'd but multiple threads can
- // come through here; I think this is OK -- worst
- // case is more than 1 thread ends up filling in the
- // sub Bits
- dels = readers[sub].getDeletedDocs();
- if (dels == null) {
- return false;
- } else {
- delDocs[sub] = dels;
- }
- }
- return dels.get(doc-starts[sub]);
- }
-
- public int length() {
- return starts[starts.length-1];
- }
- }
-
public static Bits getDeletedDocs(IndexReader r) {
Bits result;
if (r.hasDeletions()) {
- final List<IndexReader> readers = new ArrayList<IndexReader>();
+ final List<Bits> delDocs = new ArrayList<Bits>();
final List<Integer> starts = new ArrayList<Integer>();
try {
@@ -149,7 +112,7 @@ public final class MultiFields extends F
@Override
protected void add(int base, IndexReader r) throws IOException {
// record all delDocs, even if they are null
- readers.add(r);
+ delDocs.add(r.getDeletedDocs());
starts.add(base);
}
}.run();
@@ -159,16 +122,12 @@ public final class MultiFields extends F
throw new RuntimeException(ioe);
}
- assert readers.size() > 0;
- if (readers.size() == 1) {
+ assert delDocs.size() > 0;
+ if (delDocs.size() == 1) {
// Only one actual sub reader -- optimize this case
- result = readers.get(0).getDeletedDocs();
+ result = delDocs.get(0);
} else {
- int[] startsArray = new int[starts.size()];
- for(int i=0;i<startsArray.length;i++) {
- startsArray[i] = starts.get(i);
- }
- result = new MultiReaderBits(startsArray, readers.toArray(new IndexReader[readers.size()]));
+ result = new MultiBits(delDocs, starts);
}
} else {
@@ -187,7 +146,7 @@ public final class MultiFields extends F
return fields.terms(field);
}
}
-
+
/** Returns {@link DocsEnum} for the specified field &
* term. This may return null if the term does not
* exist. */
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java Fri Jun 24 23:07:10 2011
@@ -17,6 +17,7 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import org.apache.lucene.index.values.MultiIndexDocValues;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.ReaderUtil;
@@ -38,10 +39,14 @@ public final class MultiFieldsEnum exte
// Holds sub-readers containing field we are currently
// on, popped from queue.
private final FieldsEnumWithSlice[] top;
+ private final FieldsEnumWithSlice[] enumWithSlices;
+
private int numTop;
// Re-used TermsEnum
private final MultiTermsEnum terms;
+ private final MultiIndexDocValues docValues;
+
private String currentField;
@@ -50,7 +55,9 @@ public final class MultiFieldsEnum exte
public MultiFieldsEnum(FieldsEnum[] subs, ReaderUtil.Slice[] subSlices) throws IOException {
terms = new MultiTermsEnum(subSlices);
queue = new FieldMergeQueue(subs.length);
+ docValues = new MultiIndexDocValues();
top = new FieldsEnumWithSlice[subs.length];
+ List<FieldsEnumWithSlice> enumWithSlices = new ArrayList<FieldsEnumWithSlice>();
// Init q
for(int i=0;i<subs.length;i++) {
@@ -59,10 +66,13 @@ public final class MultiFieldsEnum exte
if (field != null) {
// this FieldsEnum has at least one field
final FieldsEnumWithSlice sub = new FieldsEnumWithSlice(subs[i], subSlices[i], i);
+ enumWithSlices.add(sub);
sub.current = field;
queue.add(sub);
}
}
+ this.enumWithSlices = enumWithSlices.toArray(FieldsEnumWithSlice.EMPTY_ARRAY);
+
}
@Override
@@ -84,7 +94,7 @@ public final class MultiFieldsEnum exte
if (queue.size() > 0) {
while(true) {
top[numTop++] = queue.pop();
- if (queue.size() == 0 || (queue.top()).current != top[0].current) {
+ if (queue.size() == 0 || !(queue.top()).current.equals(top[0].current)) {
break;
}
}
@@ -114,6 +124,7 @@ public final class MultiFieldsEnum exte
}
public final static class FieldsEnumWithSlice {
+ public static final FieldsEnumWithSlice[] EMPTY_ARRAY = new FieldsEnumWithSlice[0];
final FieldsEnum fields;
final ReaderUtil.Slice slice;
final int index;
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/MultiReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/MultiReader.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/MultiReader.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/MultiReader.java Fri Jun 24 23:07:10 2011
@@ -24,6 +24,7 @@ import java.util.concurrent.ConcurrentHa
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.ReaderUtil;
@@ -403,4 +404,9 @@ public class MultiReader extends IndexRe
sub.removeReaderFinishedListener(listener);
}
}
+
+ @Override
+ public PerDocValues perDocValues() throws IOException {
+ throw new UnsupportedOperationException("please use MultiPerDocValues#getPerDocs, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level Fields");
+ }
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/NoMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/NoMergePolicy.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/NoMergePolicy.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/NoMergePolicy.java Fri Jun 24 23:07:10 2011
@@ -18,7 +18,7 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.Set;
+import java.util.Map;
/**
* A {@link MergePolicy} which never returns merges to execute (hence it's
@@ -59,7 +59,7 @@ public final class NoMergePolicy extends
@Override
public MergeSpecification findMergesForOptimize(SegmentInfos segmentInfos,
- int maxSegmentCount, Set<SegmentInfo> segmentsToOptimize)
+ int maxSegmentCount, Map<SegmentInfo,Boolean> segmentsToOptimize)
throws CorruptIndexException, IOException { return null; }
@Override
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/ParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/ParallelReader.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/ParallelReader.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/ParallelReader.java Fri Jun 24 23:07:10 2011
@@ -21,6 +21,8 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.document.FieldSelectorResult;
import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.index.codecs.PerDocValues;
+import org.apache.lucene.index.values.IndexDocValues;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.MapBackedSet;
@@ -60,7 +62,8 @@ public class ParallelReader extends Inde
private int numDocs;
private boolean hasDeletions;
- private ParallelFields fields = new ParallelFields();
+ private final ParallelFields fields = new ParallelFields();
+ private final ParallelPerDocs perDocs = new ParallelPerDocs();
/** Construct a ParallelReader.
* <p>Note that all subreaders are closed if this ParallelReader is closed.</p>
@@ -134,6 +137,7 @@ public class ParallelReader extends Inde
fieldToReader.put(field, reader);
}
this.fields.addField(field, reader);
+ this.perDocs.addField(field, reader);
}
if (!ignoreStoredFields)
@@ -180,6 +184,7 @@ public class ParallelReader extends Inde
return TermsEnum.EMPTY;
}
}
+
}
// Single instance of this, per ParallelReader instance
@@ -187,7 +192,8 @@ public class ParallelReader extends Inde
final HashMap<String,Terms> fields = new HashMap<String,Terms>();
public void addField(String field, IndexReader r) throws IOException {
- fields.put(field, MultiFields.getFields(r).terms(field));
+ Fields multiFields = MultiFields.getFields(r);
+ fields.put(field, multiFields.terms(field));
}
@Override
@@ -199,8 +205,8 @@ public class ParallelReader extends Inde
return fields.get(field);
}
}
-
- @Override
+
+ @Override
public Bits getDeletedDocs() {
return MultiFields.getDeletedDocs(readers.get(0));
}
@@ -563,6 +569,36 @@ public class ParallelReader extends Inde
reader.removeReaderFinishedListener(listener);
}
}
+
+ @Override
+ public PerDocValues perDocValues() throws IOException {
+ return perDocs;
+ }
+
+ // Single instance of this, per ParallelReader instance
+ private static final class ParallelPerDocs extends PerDocValues {
+ final TreeMap<String,IndexDocValues> fields = new TreeMap<String,IndexDocValues>();
+
+ void addField(String field, IndexReader r) throws IOException {
+ PerDocValues perDocs = MultiPerDocValues.getPerDocs(r);
+ fields.put(field, perDocs.docValues(field));
+ }
+
+ //@Override -- not until Java 1.6
+ public void close() throws IOException {
+ // nothing to do here
+ }
+
+ @Override
+ public IndexDocValues docValues(String field) throws IOException {
+ return fields.get(field);
+ }
+
+ @Override
+ public Collection<String> fields() {
+ return fields.keySet();
+ }
+ }
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java Fri Jun 24 23:07:10 2011
@@ -19,16 +19,22 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
+import java.util.TreeMap;
import java.util.TreeSet;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.TermsConsumer;
+import org.apache.lucene.index.codecs.DocValuesConsumer;
+import org.apache.lucene.index.values.IndexDocValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
@@ -64,7 +70,7 @@ final class PerFieldCodecWrapper extends
for (int i = 0; i < codecs.length; i++) {
boolean success = false;
try {
- consumers.add(codecs[i].fieldsConsumer(new SegmentWriteState(state, "" + i)));
+ consumers.add(codecs[i].fieldsConsumer(new SegmentWriteState(state, i)));
success = true;
} finally {
if (!success) {
@@ -99,13 +105,13 @@ final class PerFieldCodecWrapper extends
boolean success = false;
try {
for (FieldInfo fi : fieldInfos) {
- if (fi.isIndexed) { // TODO this does not work for non-indexed fields
+ if (fi.isIndexed) {
fields.add(fi.name);
assert fi.getCodecId() != FieldInfo.UNASSIGNED_CODEC_ID;
Codec codec = segmentCodecs.codecs[fi.getCodecId()];
if (!producers.containsKey(codec)) {
producers.put(codec, codec.fieldsProducer(new SegmentReadState(dir,
- si, fieldInfos, readBufferSize, indexDivisor, ""+fi.getCodecId())));
+ si, fieldInfos, readBufferSize, indexDivisor, fi.getCodecId())));
}
codecs.put(fi.name, producers.get(codec));
}
@@ -120,6 +126,7 @@ final class PerFieldCodecWrapper extends
}
}
}
+
private final class FieldsIterator extends FieldsEnum {
private final Iterator<String> it;
@@ -161,7 +168,7 @@ final class PerFieldCodecWrapper extends
FieldsProducer fields = codecs.get(field);
return fields == null ? null : fields.terms(field);
}
-
+
@Override
public void close() throws IOException {
IOUtils.closeSafely(false, codecs.values());
@@ -184,7 +191,7 @@ final class PerFieldCodecWrapper extends
}
@Override
- public void files(Directory dir, SegmentInfo info, String codecId, Set<String> files)
+ public void files(Directory dir, SegmentInfo info, int codecId, Set<String> files)
throws IOException {
// ignore codecid since segmentCodec will assign it per codec
segmentCodecs.files(dir, info, files);
@@ -196,4 +203,91 @@ final class PerFieldCodecWrapper extends
codec.getExtensions(extensions);
}
}
+
+ @Override
+ public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+ return new PerDocConsumers(state);
+ }
+
+ @Override
+ public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+ return new PerDocProducers(state.dir, state.fieldInfos, state.segmentInfo,
+ state.readBufferSize, state.termsIndexDivisor);
+ }
+
+ private final class PerDocProducers extends PerDocValues {
+ private final TreeMap<String, PerDocValues> codecs = new TreeMap<String, PerDocValues>();
+
+ public PerDocProducers(Directory dir, FieldInfos fieldInfos, SegmentInfo si,
+ int readBufferSize, int indexDivisor) throws IOException {
+ final Map<Codec, PerDocValues> producers = new HashMap<Codec, PerDocValues>();
+ boolean success = false;
+ try {
+ for (FieldInfo fi : fieldInfos) {
+ if (fi.hasDocValues()) {
+ assert fi.getCodecId() != FieldInfo.UNASSIGNED_CODEC_ID;
+ Codec codec = segmentCodecs.codecs[fi.getCodecId()];
+ if (!producers.containsKey(codec)) {
+ producers.put(codec, codec.docsProducer(new SegmentReadState(dir,
+ si, fieldInfos, readBufferSize, indexDivisor, fi.getCodecId())));
+ }
+ codecs.put(fi.name, producers.get(codec));
+ }
+ }
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeSafely(true, producers.values());
+ }
+ }
+ }
+
+ @Override
+ public Collection<String> fields() {
+ return codecs.keySet();
+ }
+ @Override
+ public IndexDocValues docValues(String field) throws IOException {
+ final PerDocValues perDocProducer = codecs.get(field);
+ if (perDocProducer == null) {
+ return null;
+ }
+ return perDocProducer.docValues(field);
+ }
+
+ public void close() throws IOException {
+ IOUtils.closeSafely(false, codecs.values());
+ }
+ }
+
+ private final class PerDocConsumers extends PerDocConsumer {
+ private final PerDocConsumer[] consumers;
+ private final Codec[] codecs;
+ private final PerDocWriteState state;
+
+ public PerDocConsumers(PerDocWriteState state) throws IOException {
+ assert segmentCodecs == state.segmentCodecs;
+ this.state = state;
+ codecs = segmentCodecs.codecs;
+ consumers = new PerDocConsumer[codecs.length];
+ }
+
+ public void close() throws IOException {
+ IOUtils.closeSafely(false, consumers);
+ }
+
+ @Override
+ public DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
+ final int codecId = field.getCodecId();
+ assert codecId != FieldInfo.UNASSIGNED_CODEC_ID;
+ PerDocConsumer perDoc = consumers[codecId];
+ if (perDoc == null) {
+ perDoc = codecs[codecId].docsConsumer(new PerDocWriteState(state, codecId));
+ assert perDoc != null;
+ consumers[codecId] = perDoc;
+ }
+ return perDoc.addValuesField(field);
+ }
+
+ }
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java Fri Jun 24 23:07:10 2011
@@ -102,7 +102,7 @@ final class SegmentCodecs implements Clo
throws IOException {
final Codec[] codecArray = codecs;
for (int i = 0; i < codecArray.length; i++) {
- codecArray[i].files(dir, info, ""+i, files);
+ codecArray[i].files(dir, info, i, files);
}
}
@@ -148,13 +148,6 @@ final class SegmentCodecs implements Clo
return this;
}
- SegmentCodecsBuilder addAll(FieldInfos infos) {
- for (FieldInfo fieldInfo : infos) {
- tryAddAndSet(fieldInfo);
- }
- return this;
- }
-
SegmentCodecs build() {
return new SegmentCodecs(provider, codecs.toArray(Codec.EMPTY));
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java Fri Jun 24 23:07:10 2011
@@ -20,7 +20,10 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.index.codecs.PerDocValues;
+import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
/** Holds core readers that are shared (unchanged) when
@@ -39,7 +42,8 @@ final class SegmentCoreReaders {
final FieldInfos fieldInfos;
final FieldsProducer fields;
-
+ final PerDocValues perDocProducer;
+
final Directory dir;
final Directory cfsDir;
final int readBufferSize;
@@ -49,8 +53,10 @@ final class SegmentCoreReaders {
FieldsReader fieldsReaderOrig;
TermVectorsReader termVectorsReaderOrig;
- CompoundFileReader cfsReader;
- CompoundFileReader storeCFSReader;
+ CompoundFileDirectory cfsReader;
+ CompoundFileDirectory storeCFSReader;
+
+
SegmentCoreReaders(SegmentReader owner, Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor) throws IOException {
@@ -68,7 +74,7 @@ final class SegmentCoreReaders {
try {
Directory dir0 = dir;
if (si.getUseCompoundFile()) {
- cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
+ cfsReader = dir.openCompoundInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
dir0 = cfsReader;
}
cfsDir = dir0;
@@ -76,11 +82,12 @@ final class SegmentCoreReaders {
fieldInfos = si.getFieldInfos();
this.termsIndexDivisor = termsIndexDivisor;
-
+ final Codec codec = segmentCodecs.codec();
+ final SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si, fieldInfos, readBufferSize, termsIndexDivisor);
// Ask codec for its Fields
- fields = segmentCodecs.codec().fieldsProducer(new SegmentReadState(cfsDir, si, fieldInfos, readBufferSize, termsIndexDivisor));
+ fields = codec.fieldsProducer(segmentReadState);
assert fields != null;
-
+ perDocProducer = codec.docsProducer(segmentReadState);
success = true;
} finally {
if (!success) {
@@ -119,6 +126,10 @@ final class SegmentCoreReaders {
fields.close();
}
+ if (perDocProducer != null) {
+ perDocProducer.close();
+ }
+
if (termVectorsReaderOrig != null) {
termVectorsReaderOrig.close();
}
@@ -151,7 +162,7 @@ final class SegmentCoreReaders {
if (si.getDocStoreOffset() != -1) {
if (si.getDocStoreIsCompoundFile()) {
assert storeCFSReader == null;
- storeCFSReader = new CompoundFileReader(dir,
+ storeCFSReader = dir.openCompoundInput(
IndexFileNames.segmentFileName(si.getDocStoreSegment(), "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION),
readBufferSize);
storeDir = storeCFSReader;
@@ -165,7 +176,7 @@ final class SegmentCoreReaders {
// was not used, but then we are asked to open doc
// stores after the segment has switched to CFS
if (cfsReader == null) {
- cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
+ cfsReader = dir.openCompoundInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
}
storeDir = cfsReader;
assert storeDir != null;
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Fri Jun 24 23:07:10 2011
@@ -17,6 +17,7 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
@@ -29,10 +30,12 @@ import java.util.Set;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter;
+import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.StringHelper;
/**
* Information about a segment such as it's name, directory, and files related
@@ -244,7 +247,7 @@ public final class SegmentInfo implement
}
final Directory dirToTest;
if (isCompoundFile) {
- dirToTest = new CompoundFileReader(dir, IndexFileNames.segmentFileName(storesSegment, "", ext));
+ dirToTest = dir.openCompoundInput(IndexFileNames.segmentFileName(storesSegment, "", ext), BufferedIndexInput.BUFFER_SIZE);
} else {
dirToTest = dir;
}
@@ -262,8 +265,8 @@ public final class SegmentInfo implement
if (fieldInfos == null) {
Directory dir0 = dir;
if (isCompoundFile && checkCompoundFile) {
- dir0 = new CompoundFileReader(dir, IndexFileNames.segmentFileName(name,
- "", IndexFileNames.COMPOUND_FILE_EXTENSION));
+ dir0 = dir.openCompoundInput(IndexFileNames.segmentFileName(name,
+ "", IndexFileNames.COMPOUND_FILE_EXTENSION), BufferedIndexInput.BUFFER_SIZE);
}
try {
fieldInfos = new FieldInfos(dir0, IndexFileNames.segmentFileName(name,
@@ -453,7 +456,7 @@ public final class SegmentInfo implement
this.isCompoundFile = isCompoundFile;
clearFilesCache();
}
-
+
/**
* Returns true if this segment is stored as a compound
* file; else, false.
@@ -616,6 +619,10 @@ public final class SegmentInfo implement
if (useCompoundFile) {
fileSet.add(IndexFileNames.segmentFileName(name, "", IndexFileNames.COMPOUND_FILE_EXTENSION));
+ if (version != null && StringHelper.getVersionComparator().compare("3.4", version) <= 0) {
+ fileSet.add(IndexFileNames.segmentFileName(name, "",
+ IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
+ }
} else {
for(String ext : IndexFileNames.NON_STORE_INDEX_EXTENSIONS) {
addIfExists(fileSet, IndexFileNames.segmentFileName(name, "", ext));
@@ -652,7 +659,7 @@ public final class SegmentInfo implement
if (delFileName != null && (delGen >= YES || dir.fileExists(delFileName))) {
fileSet.add(delFileName);
}
-
+
if (normGen != null) {
for (Entry<Integer,Long> entry : normGen.entrySet()) {
long gen = entry.getValue();
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Fri Jun 24 23:07:10 2011
@@ -29,6 +29,9 @@ import org.apache.lucene.index.MergePoli
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.MergeState;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
+import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
@@ -116,7 +119,6 @@ final class SegmentMerger {
if (fieldInfos.hasVectors())
mergeVectors();
-
return mergedDocs;
}
@@ -131,18 +133,20 @@ final class SegmentMerger {
// Now merge all added files
Collection<String> files = info.files();
- CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName, checkAbort);
- for (String file : files) {
- assert !IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION)
- : ".del file is not allowed in .cfs: " + file;
- assert !IndexFileNames.isSeparateNormsFile(file)
- : "separate norms file (.s[0-9]+) is not allowed in .cfs: " + file;
- cfsWriter.addFile(file);
+ CompoundFileDirectory cfsDir = directory.createCompoundOutput(fileName);
+ try {
+ for (String file : files) {
+ assert !IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION)
+ : ".del file is not allowed in .cfs: " + file;
+ assert !IndexFileNames.isSeparateNormsFile(file)
+ : "separate norms file (.s[0-9]+) is not allowed in .cfs: " + file;
+ directory.copy(cfsDir, file, file);
+ checkAbort.work(directory.fileLength(file));
+ }
+ } finally {
+ cfsDir.close();
}
- // Perform the merge
- cfsWriter.close();
-
return files;
}
@@ -154,7 +158,7 @@ final class SegmentMerger {
for (String field : names) {
fInfos.addOrUpdate(field, true, storeTermVectors,
storePositionWithTermVector, storeOffsetWithTermVector, !reader
- .hasNorms(field), storePayloads, omitTFAndPositions);
+ .hasNorms(field), storePayloads, omitTFAndPositions, null);
}
}
@@ -222,6 +226,7 @@ final class SegmentMerger {
addIndexed(reader, fieldInfos, reader.getFieldNames(FieldOption.STORES_PAYLOADS), false, false, false, true, false);
addIndexed(reader, fieldInfos, reader.getFieldNames(FieldOption.INDEXED), false, false, false, false, false);
fieldInfos.addOrUpdate(reader.getFieldNames(FieldOption.UNINDEXED), false);
+ fieldInfos.addOrUpdate(reader.getFieldNames(FieldOption.DOC_VALUES), false);
}
}
final SegmentCodecs codecInfo = fieldInfos.buildSegmentCodecs(false);
@@ -477,9 +482,16 @@ final class SegmentMerger {
int docBase = 0;
final List<Fields> fields = new ArrayList<Fields>();
+
final List<ReaderUtil.Slice> slices = new ArrayList<ReaderUtil.Slice>();
final List<Bits> bits = new ArrayList<Bits>();
final List<Integer> bitsStarts = new ArrayList<Integer>();
+
+ // TODO: move this into its own method - this merges currently only docvalues
+ final List<PerDocValues> perDocProducers = new ArrayList<PerDocValues>();
+ final List<ReaderUtil.Slice> perDocSlices = new ArrayList<ReaderUtil.Slice>();
+ final List<Bits> perDocBits = new ArrayList<Bits>();
+ final List<Integer> perDocBitsStarts = new ArrayList<Integer>();
for(IndexReader r : readers) {
final Fields f = r.fields();
@@ -490,10 +502,18 @@ final class SegmentMerger {
bits.add(r.getDeletedDocs());
bitsStarts.add(docBase);
}
+ final PerDocValues producer = r.perDocValues();
+ if (producer != null) {
+ perDocSlices.add(new ReaderUtil.Slice(docBase, maxDoc, fields.size()));
+ perDocProducers.add(producer);
+ perDocBits.add(r.getDeletedDocs());
+ perDocBitsStarts.add(docBase);
+ }
docBase += maxDoc;
}
bitsStarts.add(docBase);
+ perDocBitsStarts.add(docBase);
// we may gather more readers than mergeState.readerCount
mergeState = new MergeState();
@@ -559,6 +579,20 @@ final class SegmentMerger {
} finally {
consumer.close();
}
+ if (!perDocSlices.isEmpty()) {
+ mergeState.multiDeletedDocs = new MultiBits(perDocBits, perDocBitsStarts);
+ final PerDocConsumer docsConsumer = codec
+ .docsConsumer(new PerDocWriteState(segmentWriteState));
+ try {
+ final MultiPerDocValues multiPerDocValues = new MultiPerDocValues(perDocProducers
+ .toArray(PerDocValues.EMPTY_ARRAY), perDocSlices
+ .toArray(ReaderUtil.Slice.EMPTY_ARRAY));
+ docsConsumer.merge(mergeState, multiPerDocValues);
+ } finally {
+ docsConsumer.close();
+ }
+ }
+
}
private MergeState mergeState;
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentReadState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentReadState.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentReadState.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentReadState.java Fri Jun 24 23:07:10 2011
@@ -34,11 +34,11 @@ public class SegmentReadState {
// that must do so), then it should negate this value to
// get the app's terms divisor:
public int termsIndexDivisor;
- public final String codecId;
+ public final int codecId;
public SegmentReadState(Directory dir, SegmentInfo info,
FieldInfos fieldInfos, int readBufferSize, int termsIndexDivisor) {
- this(dir, info, fieldInfos, readBufferSize, termsIndexDivisor, "");
+ this(dir, info, fieldInfos, readBufferSize, termsIndexDivisor, -1);
}
public SegmentReadState(Directory dir,
@@ -46,7 +46,7 @@ public class SegmentReadState {
FieldInfos fieldInfos,
int readBufferSize,
int termsIndexDivisor,
- String codecId) {
+ int codecId) {
this.dir = dir;
this.segmentInfo = info;
this.fieldInfos = fieldInfos;
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentReader.java Fri Jun 24 23:07:10 2011
@@ -29,6 +29,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
@@ -839,4 +840,9 @@ public class SegmentReader extends Index
// longer used (all SegmentReaders sharing it have been
// closed).
}
+
+ @Override
+ public PerDocValues perDocValues() throws IOException {
+ return core.perDocProducer;
+ }
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java Fri Jun 24 23:07:10 2011
@@ -43,7 +43,7 @@ public class SegmentWriteState {
public BitVector deletedDocs;
final SegmentCodecs segmentCodecs;
- public final String codecId;
+ public final int codecId;
/** Expert: The fraction of terms in the "dictionary" which should be stored
* in RAM. Smaller values use more memory, but make searching slightly
@@ -53,7 +53,7 @@ public class SegmentWriteState {
public int termIndexInterval; // TODO: this should be private to the codec, not settable here or in IWC
public SegmentWriteState(PrintStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos,
- int numDocs, int termIndexInterval, SegmentCodecs segmentCodecs, BufferedDeletes segDeletes) {
+ int numDocs, int termIndexInterval, SegmentCodecs segmentCodecs, BufferedDeletes segDeletes) {
this.infoStream = infoStream;
this.segDeletes = segDeletes;
this.directory = directory;
@@ -62,13 +62,13 @@ public class SegmentWriteState {
this.numDocs = numDocs;
this.termIndexInterval = termIndexInterval;
this.segmentCodecs = segmentCodecs;
- codecId = "";
+ codecId = -1;
}
/**
* Create a shallow {@link SegmentWriteState} copy final a codec ID
*/
- SegmentWriteState(SegmentWriteState state, String codecId) {
+ SegmentWriteState(SegmentWriteState state, int codecId) {
infoStream = state.infoStream;
directory = state.directory;
segmentName = state.segmentName;
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java Fri Jun 24 23:07:10 2011
@@ -26,6 +26,7 @@ import org.apache.lucene.util.ReaderUtil
import org.apache.lucene.index.DirectoryReader; // javadoc
import org.apache.lucene.index.MultiReader; // javadoc
+import org.apache.lucene.index.codecs.PerDocValues;
/**
* This class forces a composite reader (eg a {@link
@@ -60,11 +61,21 @@ public final class SlowMultiReaderWrappe
}
@Override
+ public String toString() {
+ return "SlowMultiReaderWrapper(" + in + ")";
+ }
+
+ @Override
public Fields fields() throws IOException {
return MultiFields.getFields(in);
}
@Override
+ public PerDocValues perDocValues() throws IOException {
+ return MultiPerDocValues.getPerDocs(in);
+ }
+
+ @Override
public Bits getDeletedDocs() {
return MultiFields.getDeletedDocs(in);
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/Term.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/Term.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/Term.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/Term.java Fri Jun 24 23:07:10 2011
@@ -17,15 +17,12 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import java.util.Comparator;
-
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.StringHelper;
/**
A Term represents a word from text. This is the unit of search. It is
composed of two elements, the text of the word, as a string, and the name of
- the field that the text occurred in, an interned string.
+ the field that the text occurred in.
Note that terms may represent more than words from text fields, but also
things like dates, email addresses, urls, etc. */
@@ -37,13 +34,14 @@ public final class Term implements Compa
/** Constructs a Term with the given field and bytes.
* <p>Note that a null field or null bytes value results in undefined
* behavior for most Lucene APIs that accept a Term parameter.
+ *
* <p>WARNING: the provided BytesRef is not copied, but used directly.
* Therefore the bytes should not be modified after construction, for
* example, you should clone a copy rather than pass reused bytes from
* a TermsEnum.
*/
public Term(String fld, BytesRef bytes) {
- field = fld == null ? null : StringHelper.intern(fld);
+ field = fld;
this.bytes = bytes;
}
@@ -61,28 +59,10 @@ public final class Term implements Compa
* @param fld
*/
public Term(String fld) {
- this(fld, new BytesRef(), true);
- }
-
- /**
- * WARNING: the provided BytesRef is not copied, but used directly.
- * Therefore the bytes should not be modified after construction, for
- * example, you should clone a copy rather than pass reused bytes from
- * a TermsEnum.
- *
- * @lucene.experimental
- */
- public Term(String fld, BytesRef bytes, boolean intern) {
- field = intern ? StringHelper.intern(fld) : fld; // field names are interned
- this.bytes = bytes; // unless already known to be
+ this(fld, new BytesRef());
}
- /** @lucene.experimental */
- public Term(String fld, String text, boolean intern) {
- this(fld, new BytesRef(text), intern);
- }
-
- /** Returns the field of this term, an interned string. The field indicates
+ /** Returns the field of this term. The field indicates
the part of a document which this term came from. */
public final String field() { return field; }
@@ -94,32 +74,6 @@ public final class Term implements Compa
/** Returns the bytes of this term. */
public final BytesRef bytes() { return bytes; }
- /**
- * Optimized construction of new Terms by reusing same field as this Term
- * - avoids field.intern() overhead
- * <p>WARNING: the provided BytesRef is not copied, but used directly.
- * Therefore the bytes should not be modified after construction, for
- * example, you should clone a copy rather than pass reused bytes from
- * a TermsEnum.
- * @param bytes The bytes of the new term (field is implicitly same as this Term instance)
- * @return A new Term
- */
- public Term createTerm(BytesRef bytes)
- {
- return new Term(field,bytes,false);
- }
-
- /**
- * Optimized construction of new Terms by reusing same field as this Term
- * - avoids field.intern() overhead
- * @param text The text of the new term (field is implicitly same as this Term instance)
- * @return A new Term
- */
- public Term createTerm(String text)
- {
- return new Term(field,text,false);
- }
-
@Override
public boolean equals(Object obj) {
if (this == obj)
@@ -157,26 +111,11 @@ public final class Term implements Compa
The ordering of terms is first by field, then by text.*/
public final int compareTo(Term other) {
- if (field == other.field) // fields are interned
+ if (field.equals(other.field)) {
return bytes.compareTo(other.bytes);
- else
- return field.compareTo(other.field);
- }
-
- @Deprecated
- private static final Comparator<BytesRef> legacyComparator =
- BytesRef.getUTF8SortedAsUTF16Comparator();
-
- /**
- * @deprecated (4.0) For internal backwards compatibility use only
- * @lucene.internal
- */
- @Deprecated
- public final int compareToUTF16(Term other) {
- if (field == other.field) // fields are interned
- return legacyComparator.compare(this.bytes, other.bytes);
- else
+ } else {
return field.compareTo(other.field);
+ }
}
/**
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java Fri Jun 24 23:07:10 2011
@@ -181,9 +181,9 @@ final class TermsHashPerField extends In
// term text into textStart address
// Get the text & hash of this term.
int termID;
- try{
- termID = bytesHash.add(termBytesRef, termAtt.fillBytesRef());
- }catch (MaxBytesLengthExceededException e) {
+ try {
+ termID = bytesHash.add(termBytesRef, termAtt.fillBytesRef());
+ } catch (MaxBytesLengthExceededException e) {
// Not enough room in current block
// Just skip this term, to remain as robust as
// possible during indexing. A TokenFilter
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java Fri Jun 24 23:07:10 2011
@@ -18,7 +18,7 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.Set;
+import java.util.Map;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
@@ -82,6 +82,7 @@ public class TieredMergePolicy extends M
private double expungeDeletesPctAllowed = 10.0;
private boolean useCompoundFile = true;
private double noCFSRatio = 0.1;
+ private double reclaimDeletesWeight = 2.0;
/** Maximum number of segments to be merged at a time
* during "normal" merging. For explicit merging (eg,
@@ -133,6 +134,23 @@ public class TieredMergePolicy extends M
return maxMergedSegmentBytes/1024/1024.;
}
+ /** Controls how aggressively merges that reclaim more
+ * deletions are favored. Higher values favor selecting
+ * merges that reclaim deletions. A value of 0.0 means
+ * deletions don't impact merge selection. */
+ public TieredMergePolicy setReclaimDeletesWeight(double v) {
+ if (v < 0.0) {
+ throw new IllegalArgumentException("reclaimDeletesWeight must be >= 0.0 (got " + v + ")");
+ }
+ reclaimDeletesWeight = v;
+ return this;
+ }
+
+ /** See {@link #setReclaimDeletesWeight}. */
+ public double getReclaimDeletesWeight() {
+ return reclaimDeletesWeight;
+ }
+
/** Segments smaller than this are "rounded up" to this
* size, ie treated as equal (floor) size for merge
* selection. This is to prevent frequent flushing of
@@ -169,8 +187,12 @@ public class TieredMergePolicy extends M
/** Sets the allowed number of segments per tier. Smaller
* values mean more merging but fewer segments.
- * setMaxMergeAtOnce} otherwise you'll hit
- * Default is 10.0. */
+ *
+ * <p><b>NOTE</b>: this value should be >= the {@link
+ * #setMaxMergeAtOnce} otherwise you'll force too much
+ * merging to occur.</p>
+ *
+ * <p>Default is 10.0.</p> */
public TieredMergePolicy setSegmentsPerTier(double v) {
if (v < 2.0) {
throw new IllegalArgumentException("segmentsPerTier must be >= 2.0 (got " + v + ")");
@@ -435,7 +457,7 @@ public class TieredMergePolicy extends M
// Strongly favor merges that reclaim deletes:
final double nonDelRatio = ((double) totAfterMergeBytes)/totBeforeMergeBytes;
- mergeScore *= nonDelRatio;
+ mergeScore *= Math.pow(nonDelRatio, reclaimDeletesWeight);
final double finalMergeScore = mergeScore;
@@ -454,7 +476,7 @@ public class TieredMergePolicy extends M
}
@Override
- public MergeSpecification findMergesForOptimize(SegmentInfos infos, int maxSegmentCount, Set<SegmentInfo> segmentsToOptimize) throws IOException {
+ public MergeSpecification findMergesForOptimize(SegmentInfos infos, int maxSegmentCount, Map<SegmentInfo,Boolean> segmentsToOptimize) throws IOException {
if (verbose()) {
message("findMergesForOptimize maxSegmentCount=" + maxSegmentCount + " infos=" + writer.get().segString(infos) + " segmentsToOptimize=" + segmentsToOptimize);
}
@@ -462,8 +484,11 @@ public class TieredMergePolicy extends M
List<SegmentInfo> eligible = new ArrayList<SegmentInfo>();
boolean optimizeMergeRunning = false;
final Collection<SegmentInfo> merging = writer.get().getMergingSegments();
+ boolean segmentIsOriginal = false;
for(SegmentInfo info : infos) {
- if (segmentsToOptimize.contains(info)) {
+ final Boolean isOriginal = segmentsToOptimize.get(info);
+ if (isOriginal != null) {
+ segmentIsOriginal = isOriginal;
if (!merging.contains(info)) {
eligible.add(info);
} else {
@@ -477,7 +502,7 @@ public class TieredMergePolicy extends M
}
if ((maxSegmentCount > 1 && eligible.size() <= maxSegmentCount) ||
- (maxSegmentCount == 1 && eligible.size() == 1 && isOptimized(eligible.get(0)))) {
+ (maxSegmentCount == 1 && eligible.size() == 1 && (!segmentIsOriginal || isOptimized(eligible.get(0))))) {
if (verbose()) {
message("already optimized");
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java Fri Jun 24 23:07:10 2011
@@ -21,9 +21,9 @@ import org.apache.lucene.util.Constants;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
+import java.util.Map;
+import java.util.HashMap;
/** This {@link MergePolicy} is used for upgrading all existing segments of
* an index when calling {@link IndexWriter#optimize()}.
@@ -79,12 +79,13 @@ public class UpgradeIndexMergePolicy ext
}
@Override
- public MergeSpecification findMergesForOptimize(SegmentInfos segmentInfos, int maxSegmentCount, Set<SegmentInfo> segmentsToOptimize) throws CorruptIndexException, IOException {
+ public MergeSpecification findMergesForOptimize(SegmentInfos segmentInfos, int maxSegmentCount, Map<SegmentInfo,Boolean> segmentsToOptimize) throws CorruptIndexException, IOException {
// first find all old segments
- final HashSet<SegmentInfo> oldSegments = new HashSet<SegmentInfo>();
+ final Map<SegmentInfo,Boolean> oldSegments = new HashMap<SegmentInfo,Boolean>();
for (final SegmentInfo si : segmentInfos) {
- if (segmentsToOptimize.contains(si) && shouldUpgradeSegment(si)) {
- oldSegments.add(si);
+ final Boolean v =segmentsToOptimize.get(si);
+ if (v != null && shouldUpgradeSegment(si)) {
+ oldSegments.put(si, v);
}
}
@@ -93,14 +94,14 @@ public class UpgradeIndexMergePolicy ext
if (oldSegments.isEmpty())
return null;
- MergeSpecification spec = base.findMergesForOptimize(segmentInfos, maxSegmentCount, oldSegments);
+ MergeSpecification spec = base.findMergesForOptimize(segmentInfos, maxSegmentCount, oldSegments);
if (spec != null) {
// remove all segments that are in merge specification from oldSegments,
// the resulting set contains all segments that are left over
// and will be merged to one additional segment:
for (final OneMerge om : spec.merges) {
- oldSegments.removeAll(om.segments);
+ oldSegments.keySet().removeAll(om.segments);
}
}
@@ -110,7 +111,7 @@ public class UpgradeIndexMergePolicy ext
" does not want to merge all old segments, merge remaining ones into new segment: " + oldSegments);
final List<SegmentInfo> newInfos = new ArrayList<SegmentInfo>();
for (final SegmentInfo si : segmentInfos) {
- if (oldSegments.contains(si)) {
+ if (oldSegments.containsKey(si)) {
newInfos.add(si);
}
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java Fri Jun 24 23:07:10 2011
@@ -91,7 +91,7 @@ public class BlockTermsReader extends Fi
@Override
public boolean equals(Object _other) {
FieldAndTerm other = (FieldAndTerm) _other;
- return other.field == field && term.bytesEquals(other.term);
+ return other.field.equals(field) && term.bytesEquals(other.term);
}
@Override
@@ -108,7 +108,7 @@ public class BlockTermsReader extends Fi
//private String segment;
public BlockTermsReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, int readBufferSize,
- int termsCacheSize, String codecId)
+ int termsCacheSize, int codecId)
throws IOException {
this.postingsReader = postingsReader;
@@ -196,7 +196,7 @@ public class BlockTermsReader extends Fi
}
}
- public static void files(Directory dir, SegmentInfo segmentInfo, String id, Collection<String> files) {
+ public static void files(Directory dir, SegmentInfo segmentInfo, int id, Collection<String> files) {
files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, BlockTermsWriter.TERMS_EXTENSION));
}
@@ -312,13 +312,13 @@ public class BlockTermsReader extends Fi
private int blocksSinceSeek;
private byte[] termSuffixes;
- private ByteArrayDataInput termSuffixesReader = new ByteArrayDataInput(null);
+ private ByteArrayDataInput termSuffixesReader = new ByteArrayDataInput();
/* Common prefix used for all terms in this block. */
private int termBlockPrefix;
private byte[] docFreqBytes;
- private final ByteArrayDataInput freqReader = new ByteArrayDataInput(null);
+ private final ByteArrayDataInput freqReader = new ByteArrayDataInput();
private int metaDataUpto;
public SegmentTermsEnum() throws IOException {
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/Codec.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/Codec.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/Codec.java Fri Jun 24 23:07:10 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs;
import java.io.IOException;
import java.util.Set;
+import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
@@ -50,6 +51,10 @@ public abstract class Codec {
* returns, it must hold open any files it will need to
* use; else, those files may be deleted. */
public abstract FieldsProducer fieldsProducer(SegmentReadState state) throws IOException;
+
+ public abstract PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException;
+
+ public abstract PerDocValues docsProducer(SegmentReadState state) throws IOException;
/**
* Gathers files associated with this segment
@@ -59,7 +64,7 @@ public abstract class Codec {
* @param id the codec id within this segment
* @param files the of files to add the codec files to.
*/
- public abstract void files(Directory dir, SegmentInfo segmentInfo, String id, Set<String> files) throws IOException;
+ public abstract void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException;
/** Records all file extensions this codec uses */
public abstract void getExtensions(Set<String> extensions);
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java Fri Jun 24 23:07:10 2011
@@ -43,7 +43,7 @@ public class CodecProvider {
private final Set<String> knownExtensions = new HashSet<String>();
- public final static String[] CORE_CODECS = new String[] {"Standard", "Pulsing", "PreFlex", "SimpleText"};
+ public final static String[] CORE_CODECS = new String[] {"Standard", "Pulsing", "PreFlex", "SimpleText", "Memory"};
public synchronized void register(Codec codec) {
if (codec.name == null) {
@@ -83,8 +83,9 @@ public class CodecProvider {
public synchronized Codec lookup(String name) {
final Codec codec = codecs.get(name);
- if (codec == null)
+ if (codec == null) {
throw new IllegalArgumentException("required codec '" + name + "' not found");
+ }
return codec;
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java Fri Jun 24 23:07:10 2011
@@ -17,6 +17,7 @@ package org.apache.lucene.index.codecs;
* limitations under the License.
*/
+import org.apache.lucene.index.codecs.memory.MemoryCodec;
import org.apache.lucene.index.codecs.preflex.PreFlexCodec;
import org.apache.lucene.index.codecs.pulsing.PulsingCodec;
import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec;
@@ -45,5 +46,6 @@ public class CoreCodecProvider extends C
register(new PreFlexCodec());
register(new PulsingCodec(1));
register(new SimpleTextCodec());
+ register(new MemoryCodec());
}
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java Fri Jun 24 23:07:10 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.index.codecs;
import java.io.IOException;
-import org.apache.lucene.index.CompoundFileReader;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldsReader;
import org.apache.lucene.index.IndexFileNames;
@@ -68,12 +67,12 @@ public class DefaultSegmentInfosReader e
Directory dir = directory;
if (si.getDocStoreOffset() != -1) {
if (si.getDocStoreIsCompoundFile()) {
- dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName(
+ dir = dir.openCompoundInput(IndexFileNames.segmentFileName(
si.getDocStoreSegment(), "",
IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), 1024);
}
} else if (si.getUseCompoundFile()) {
- dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName(
+ dir = dir.openCompoundInput(IndexFileNames.segmentFileName(
si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), 1024);
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java Fri Jun 24 23:07:10 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.TermsEnum;
import java.io.IOException;
import java.io.Closeable;
@@ -35,7 +36,7 @@ public abstract class FieldsConsumer imp
/** Add a new field */
public abstract TermsConsumer addField(FieldInfo field) throws IOException;
-
+
/** Called when we are done adding everything. */
public abstract void close() throws IOException;
@@ -45,8 +46,12 @@ public abstract class FieldsConsumer imp
String field;
while((field = fieldsEnum.next()) != null) {
mergeState.fieldInfo = mergeState.fieldInfos.fieldInfo(field);
- final TermsConsumer termsConsumer = addField(mergeState.fieldInfo);
- termsConsumer.merge(mergeState, fieldsEnum.terms());
+ assert mergeState.fieldInfo != null : "FieldInfo for field is null: "+ field;
+ TermsEnum terms = fieldsEnum.terms();
+ if (terms != null) {
+ final TermsConsumer termsConsumer = addField(mergeState.fieldInfo);
+ termsConsumer.merge(mergeState, terms);
+ }
}
}
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java Fri Jun 24 23:07:10 2011
@@ -17,10 +17,12 @@ package org.apache.lucene.index.codecs;
* limitations under the License.
*/
-import org.apache.lucene.index.Fields;
-
-import java.io.IOException;
import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.Terms;
/** Abstract API that consumes terms, doc, freq, prox and
* payloads postings. Concrete implementations of this
@@ -33,4 +35,28 @@ import java.io.Closeable;
public abstract class FieldsProducer extends Fields implements Closeable {
public abstract void close() throws IOException;
public abstract void loadTermsIndex(int indexDivisor) throws IOException;
+
+ public static final FieldsProducer EMPTY = new FieldsProducer() {
+
+ @Override
+ public Terms terms(String field) throws IOException {
+ return null;
+ }
+
+ @Override
+ public FieldsEnum iterator() throws IOException {
+ return FieldsEnum.EMPTY;
+ }
+
+ @Override
+ public void loadTermsIndex(int indexDivisor) throws IOException {
+
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+ };
+
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java Fri Jun 24 23:07:10 2011
@@ -68,7 +68,7 @@ public class FixedGapTermsIndexReader ex
// start of the field info data
protected long dirOffset;
- public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator<BytesRef> termComp, String codecId)
+ public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator<BytesRef> termComp, int codecId)
throws IOException {
this.termComp = termComp;
@@ -406,7 +406,7 @@ public class FixedGapTermsIndexReader ex
}
}
- public static void files(Directory dir, SegmentInfo info, String id, Collection<String> files) {
+ public static void files(Directory dir, SegmentInfo info, int id, Collection<String> files) {
files.add(IndexFileNames.segmentFileName(info.name, id, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION));
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java Fri Jun 24 23:07:10 2011
@@ -57,8 +57,7 @@ public class VariableGapTermsIndexReader
protected long dirOffset;
final String segment;
-
- public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, String codecId)
+ public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, int codecId)
throws IOException {
in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION));
@@ -159,15 +158,11 @@ public class VariableGapTermsIndexReader
private final class FieldIndexData {
- private final FieldInfo fieldInfo;
private final long indexStart;
-
// Set only if terms index is loaded:
private volatile FST<Long> fst;
public FieldIndexData(FieldInfo fieldInfo, long indexStart) throws IOException {
-
- this.fieldInfo = fieldInfo;
this.indexStart = indexStart;
if (indexDivisor > 0) {
@@ -241,7 +236,7 @@ public class VariableGapTermsIndexReader
}
}
- public static void files(Directory dir, SegmentInfo info, String id, Collection<String> files) {
+ public static void files(Directory dir, SegmentInfo info, int id, Collection<String> files) {
files.add(IndexFileNames.segmentFileName(info.name, id, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION));
}
Modified: lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java Fri Jun 24 23:07:10 2011
@@ -22,11 +22,14 @@ import java.io.IOException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
/** Codec that reads the pre-flex-indexing postings
* format. It does not provide a writer because newly
@@ -66,7 +69,7 @@ public class PreFlexCodec extends Codec
}
@Override
- public void files(Directory dir, SegmentInfo info, String id, Set<String> files) throws IOException {
+ public void files(Directory dir, SegmentInfo info, int id, Set<String> files) throws IOException {
// preflex fields have no codec ID - we ignore it here
PreFlexFields.files(dir, info, files);
}
@@ -78,4 +81,14 @@ public class PreFlexCodec extends Codec
extensions.add(TERMS_EXTENSION);
extensions.add(TERMS_INDEX_EXTENSION);
}
+
+ @Override
+ public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+ throw new UnsupportedOperationException("PerDocConsumer is not supported by Preflex codec");
+ }
+
+ @Override
+ public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+ throw new UnsupportedOperationException("PerDocValues is not supported by Preflex codec");
+ }
}