You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2011/11/03 14:35:20 UTC
svn commit: r1197110 [3/10] - in /lucene/dev/branches/solrcloud: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/contrib/ dev-tools/idea/modules/benchmark/
dev-tools/idea/solr/contrib/langid/ dev-tools/...
Modified: lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java (original)
+++ lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java Thu Nov 3 13:35:07 2011
@@ -18,16 +18,17 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
-import java.util.List;
-import java.util.ArrayList;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
+import org.apache.lucene.index.ConcurrentMergeScheduler;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase;
@@ -63,7 +64,6 @@ public class TestSearcherManager extends
@Override
protected void doAfterWriter(ExecutorService es) throws Exception {
- // SearcherManager needs to see empty commit:
final SearcherWarmer warmer = new SearcherWarmer() {
@Override
public void warm(IndexSearcher s) throws IOException {
@@ -72,11 +72,12 @@ public class TestSearcherManager extends
}
};
if (random.nextBoolean()) {
- mgr = SearcherManager.open(writer, true, warmer, es);
+ mgr = new SearcherManager(writer, true, warmer, es);
isNRT = true;
} else {
+ // SearcherManager needs to see empty commit:
writer.commit();
- mgr = SearcherManager.open(dir, warmer, es);
+ mgr = new SearcherManager(dir, warmer, es);
isNRT = false;
}
@@ -178,8 +179,9 @@ public class TestSearcherManager extends
public void testIntermediateClose() throws IOException, InterruptedException {
Directory dir = newDirectory();
+ // Test can deadlock if we use SMS:
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergeScheduler(new ConcurrentMergeScheduler()));
writer.addDocument(new Document());
writer.commit();
final CountDownLatch awaitEnterWarm = new CountDownLatch(1);
@@ -196,8 +198,8 @@ public class TestSearcherManager extends
}
}
};
- final SearcherManager searcherManager = random.nextBoolean() ? SearcherManager.open(dir,
- warmer, es) : SearcherManager.open(writer, random.nextBoolean(), warmer, es);
+ final SearcherManager searcherManager = random.nextBoolean() ? new SearcherManager(dir,
+ warmer, es) : new SearcherManager(writer, random.nextBoolean(), warmer, es);
IndexSearcher searcher = searcherManager.acquire();
try {
assertEquals(1, searcher.getIndexReader().numDocs());
Modified: lucene/dev/branches/solrcloud/lucene/contrib/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/contrib/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/contrib/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java Thu Nov 3 13:35:07 2011
@@ -70,17 +70,16 @@ public class DuplicateFilter extends Fil
}
@Override
- public DocIdSet getDocIdSet(AtomicReaderContext context) throws IOException {
+ public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
if (processingMode == ProcessingMode.PM_FAST_INVALIDATION) {
- return fastBits(context.reader);
+ return fastBits(context.reader, acceptDocs);
} else {
- return correctBits(context.reader);
+ return correctBits(context.reader, acceptDocs);
}
}
- private FixedBitSet correctBits(IndexReader reader) throws IOException {
+ private FixedBitSet correctBits(IndexReader reader, Bits acceptDocs) throws IOException {
FixedBitSet bits = new FixedBitSet(reader.maxDoc()); //assume all are INvalid
- final Bits liveDocs = MultiFields.getLiveDocs(reader);
Terms terms = reader.fields().terms(fieldName);
if (terms == null) {
@@ -94,7 +93,7 @@ public class DuplicateFilter extends Fil
if (currTerm == null) {
break;
} else {
- docs = termsEnum.docs(liveDocs, docs);
+ docs = termsEnum.docs(acceptDocs, docs);
int doc = docs.nextDoc();
if (doc != DocsEnum.NO_MORE_DOCS) {
if (keepMode == KeepMode.KM_USE_FIRST_OCCURRENCE) {
@@ -116,10 +115,9 @@ public class DuplicateFilter extends Fil
return bits;
}
- private FixedBitSet fastBits(IndexReader reader) throws IOException {
+ private FixedBitSet fastBits(IndexReader reader, Bits acceptDocs) throws IOException {
FixedBitSet bits = new FixedBitSet(reader.maxDoc());
bits.set(0, reader.maxDoc()); //assume all are valid
- final Bits liveDocs = MultiFields.getLiveDocs(reader);
Terms terms = reader.fields().terms(fieldName);
if (terms == null) {
@@ -135,7 +133,7 @@ public class DuplicateFilter extends Fil
} else {
if (termsEnum.docFreq() > 1) {
// unset potential duplicates
- docs = termsEnum.docs(liveDocs, docs);
+ docs = termsEnum.docs(acceptDocs, docs);
int doc = docs.nextDoc();
if (doc != DocsEnum.NO_MORE_DOCS) {
if (keepMode == KeepMode.KM_USE_FIRST_OCCURRENCE) {
Modified: lucene/dev/branches/solrcloud/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/geohash/GeoHashDistanceFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/geohash/GeoHashDistanceFilter.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/geohash/GeoHashDistanceFilter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/geohash/GeoHashDistanceFilter.java Thu Nov 3 13:35:07 2011
@@ -26,6 +26,7 @@ import org.apache.lucene.search.Filter;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.FilteredDocIdSet;
import org.apache.lucene.spatial.DistanceUtils;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.spatial.tier.DistanceFilter;
@@ -57,7 +58,7 @@ public class GeoHashDistanceFilter exten
}
@Override
- public DocIdSet getDocIdSet(AtomicReaderContext context) throws IOException {
+ public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
final DocTerms geoHashValues = FieldCache.DEFAULT.getTerms(context.reader, geoHashField);
final BytesRef br = new BytesRef();
@@ -65,7 +66,7 @@ public class GeoHashDistanceFilter exten
final int docBase = nextDocBase;
nextDocBase += context.reader.maxDoc();
- return new FilteredDocIdSet(startingFilter.getDocIdSet(context)) {
+ return new FilteredDocIdSet(startingFilter.getDocIdSet(context, acceptDocs)) {
@Override
public boolean match(int doc) {
Modified: lucene/dev/branches/solrcloud/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java Thu Nov 3 13:35:07 2011
@@ -45,8 +45,7 @@ public class CartesianShapeFilter extend
}
@Override
- public DocIdSet getDocIdSet(final AtomicReaderContext context) throws IOException {
- final Bits liveDocs = context.reader.getLiveDocs();
+ public DocIdSet getDocIdSet(final AtomicReaderContext context, final Bits acceptDocs) throws IOException {
final List<Double> area = shape.getArea();
final int sz = area.size();
@@ -58,7 +57,7 @@ public class CartesianShapeFilter extend
return new DocIdSet() {
@Override
public DocIdSetIterator iterator() throws IOException {
- return context.reader.termDocsEnum(liveDocs, fieldName, bytesRef);
+ return context.reader.termDocsEnum(acceptDocs, fieldName, bytesRef);
}
@Override
@@ -71,7 +70,7 @@ public class CartesianShapeFilter extend
for (int i =0; i< sz; i++) {
double boxId = area.get(i).doubleValue();
NumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(boxId), 0, bytesRef);
- final DocsEnum docsEnum = context.reader.termDocsEnum(liveDocs, fieldName, bytesRef);
+ final DocsEnum docsEnum = context.reader.termDocsEnum(acceptDocs, fieldName, bytesRef);
if (docsEnum == null) continue;
// iterate through all documents
// which have this boxId
Modified: lucene/dev/branches/solrcloud/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/LatLongDistanceFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/LatLongDistanceFilter.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/LatLongDistanceFilter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/LatLongDistanceFilter.java Thu Nov 3 13:35:07 2011
@@ -24,6 +24,7 @@ import org.apache.lucene.search.Filtered
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.DocIdSet;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.spatial.DistanceUtils;
@@ -60,7 +61,7 @@ public class LatLongDistanceFilter exten
}
@Override
- public DocIdSet getDocIdSet(AtomicReaderContext context) throws IOException {
+ public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
final double[] latIndex = FieldCache.DEFAULT.getDoubles(context.reader, latField);
final double[] lngIndex = FieldCache.DEFAULT.getDoubles(context.reader, lngField);
@@ -68,7 +69,7 @@ public class LatLongDistanceFilter exten
final int docBase = nextDocBase;
nextDocBase += context.reader.maxDoc();
- return new FilteredDocIdSet(startingFilter.getDocIdSet(context)) {
+ return new FilteredDocIdSet(startingFilter.getDocIdSet(context, acceptDocs)) {
@Override
protected boolean match(int doc) {
double x = latIndex[doc];
Modified: lucene/dev/branches/solrcloud/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestDistance.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestDistance.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestDistance.java (original)
+++ lucene/dev/branches/solrcloud/lucene/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestDistance.java Thu Nov 3 13:35:07 2011
@@ -105,7 +105,7 @@ public class TestDistance extends Lucene
AtomicReaderContext[] leaves = ReaderUtil.leaves(r.getTopReaderContext());
for (int i = 0; i < leaves.length; i++) {
- f.getDocIdSet(leaves[i]);
+ f.getDocIdSet(leaves[i], leaves[i].reader.getLiveDocs());
}
r.close();
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/Document.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/Document.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/Document.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/Document.java Thu Nov 3 13:35:07 2011
@@ -47,25 +47,7 @@ public final class Document implements I
@Override
public Iterator<IndexableField> iterator() {
-
- return new Iterator<IndexableField>() {
- private int fieldUpto = 0;
-
- @Override
- public boolean hasNext() {
- return fieldUpto < fields.size();
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public IndexableField next() {
- return fields.get(fieldUpto++);
- }
- };
+ return fields.iterator();
}
/**
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/FieldType.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/FieldType.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/FieldType.java Thu Nov 3 13:35:07 2011
@@ -55,7 +55,7 @@ public class FieldType implements Indexa
/**
* Prevents future changes. Note, it is recommended that this is called once
- * the FieldTypes's properties have been set, to prevent unintential state
+ * the FieldTypes's properties have been set, to prevent unintentional state
* changes.
*/
public void freeze() {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java Thu Nov 3 13:35:07 2011
@@ -432,13 +432,13 @@ class BufferedDeletesStream {
}
// Delete by query
- private synchronized long applyQueryDeletes(Iterable<QueryAndLimit> queriesIter, SegmentReader reader) throws IOException {
+ private static long applyQueryDeletes(Iterable<QueryAndLimit> queriesIter, SegmentReader reader) throws IOException {
long delCount = 0;
final AtomicReaderContext readerContext = (AtomicReaderContext) reader.getTopReaderContext();
for (QueryAndLimit ent : queriesIter) {
Query query = ent.query;
int limit = ent.limit;
- final DocIdSet docs = new QueryWrapperFilter(query).getDocIdSet(readerContext);
+ final DocIdSet docs = new QueryWrapperFilter(query).getDocIdSet(readerContext, readerContext.reader.getLiveDocs());
if (docs != null) {
final DocIdSetIterator it = docs.iterator();
if (it != null) {
@@ -448,11 +448,8 @@ class BufferedDeletesStream {
break;
reader.deleteDocument(doc);
- // TODO: we could/should change
- // reader.deleteDocument to return boolean
- // true if it did in fact delete, because here
- // we could be deleting an already-deleted doc
- // which makes this an upper bound:
+ // as we use getLiveDocs() to filter out already deleted documents,
+ // we only delete live documents, so the counting is right:
delCount++;
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/CheckIndex.java Thu Nov 3 13:35:07 2011
@@ -726,6 +726,9 @@ public class CheckIndex {
}
final int docFreq = terms.docFreq();
+ if (docFreq <= 0) {
+ throw new RuntimeException("docfreq: " + docFreq + " is out of bounds");
+ }
status.totFreq += docFreq;
sumDocFreq += docFreq;
@@ -823,6 +826,9 @@ public class CheckIndex {
throw new RuntimeException("term " + term + " docFreq=" + docFreq + " != tot docs w/o deletions " + docCount);
}
if (hasTotalTermFreq) {
+ if (totalTermFreq2 <= 0) {
+ throw new RuntimeException("totalTermFreq: " + totalTermFreq2 + " is out of bounds");
+ }
sumTotalTermFreq += totalTermFreq;
if (totalTermFreq != totalTermFreq2) {
throw new RuntimeException("term " + term + " totalTermFreq=" + totalTermFreq2 + " != recomputed totalTermFreq=" + totalTermFreq);
@@ -939,20 +945,20 @@ public class CheckIndex {
is.search(new TermQuery(new Term(field, lastTerm)), 1);
}
-
- // Test seeking by ord
- if (hasOrd && status.termCount-termCountStart > 0) {
- long termCount;
- try {
- termCount = fields.terms(field).getUniqueTermCount();
- } catch (UnsupportedOperationException uoe) {
- termCount = -1;
- }
-
+
+ // check unique term count
+ long termCount = -1;
+
+ if (status.termCount-termCountStart > 0) {
+ termCount = fields.terms(field).getUniqueTermCount();
+
if (termCount != -1 && termCount != status.termCount - termCountStart) {
throw new RuntimeException("termCount mismatch " + termCount + " vs " + (status.termCount - termCountStart));
}
-
+ }
+
+ // Test seeking by ord
+ if (hasOrd && status.termCount-termCountStart > 0) {
int seekCount = (int) Math.min(10000L, termCount);
if (seekCount > 0) {
BytesRef[] seekTerms = new BytesRef[seekCount];
@@ -995,6 +1001,21 @@ public class CheckIndex {
}
}
+ // for most implementations, this is boring (just the sum across all fields)
+ // but codecs that don't work per-field like preflex actually implement this,
+ // but don't implement it on Terms, so the check isn't redundant.
+ long uniqueTermCountAllFields = reader.getUniqueTermCount();
+
+ // this means something is seriously screwed, e.g. we are somehow getting enclosed in PFCW!!!!!!
+
+ if (uniqueTermCountAllFields == -1) {
+ throw new RuntimeException("invalid termCount: -1");
+ }
+
+ if (status.termCount != uniqueTermCountAllFields) {
+ throw new RuntimeException("termCount mismatch " + uniqueTermCountAllFields + " vs " + (status.termCount));
+ }
+
msg("OK [" + status.termCount + " terms; " + status.totFreq + " terms/docs pairs; " + status.totPos + " tokens]");
if (verbose && status.blockTreeStats != null && infoStream != null && status.termCount > 0) {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Thu Nov 3 13:35:07 2011
@@ -406,8 +406,15 @@ class DirectoryReader extends IndexReade
return doOpenIfChanged(true, commit);
}
- // NOTE: always returns a non-null result (ie new reader)
- // but that could change someday
+ @Override
+ protected final IndexReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) throws CorruptIndexException, IOException {
+ if (writer == this.writer && applyAllDeletes == this.applyAllDeletes) {
+ return doOpenIfChanged();
+ } else {
+ return super.doOpenIfChanged(writer, applyAllDeletes);
+ }
+ }
+
private final IndexReader doOpenFromWriter(boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException {
assert readOnly;
@@ -419,10 +426,18 @@ class DirectoryReader extends IndexReade
throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() cannot currently accept a commit");
}
- // TODO: right now we *always* make a new reader; in
- // the future we could have write make some effort to
- // detect that no changes have occurred
+ if (writer.nrtIsCurrent(segmentInfos)) {
+ return null;
+ }
+
IndexReader reader = writer.getReader(applyAllDeletes);
+
+ // If in fact no changes took place, return null:
+ if (reader.getVersion() == segmentInfos.getVersion()) {
+ reader.decRef();
+ return null;
+ }
+
reader.readerFinishedListeners = readerFinishedListeners;
return reader;
}
@@ -803,7 +818,7 @@ class DirectoryReader extends IndexReade
@Override
public long getUniqueTermCount() throws IOException {
- throw new UnsupportedOperationException("");
+ return -1;
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocTermOrds.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocTermOrds.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocTermOrds.java Thu Nov 3 13:35:07 2011
@@ -101,7 +101,7 @@ import java.util.Comparator;
public class DocTermOrds {
- // Term ords are shifted by this, internally, to reseve
+ // Term ords are shifted by this, internally, to reserve
// values 0 (end term) and 1 (index is a pointer into byte array)
private final static int TNUM_OFFSET = 2;
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Thu Nov 3 13:35:07 2011
@@ -411,7 +411,7 @@ final class DocumentsWriter {
*/
try {
synchronized (ticketQueue) {
- // Each flush is assigned a ticket in the order they accquire the ticketQueue lock
+ // Each flush is assigned a ticket in the order they acquire the ticketQueue lock
ticket = new FlushTicket(flushingDWPT.prepareFlush(), true);
ticketQueue.add(ticket);
}
@@ -578,6 +578,7 @@ final class DocumentsWriter {
}
applyFlushTickets();
}
+ assert !flushingDeleteQueue.anyChanges();
} finally {
assert flushingDeleteQueue == currentFullFlushDelQueue;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java Thu Nov 3 13:35:07 2011
@@ -169,7 +169,13 @@ final class DocumentsWriterDeleteQueue {
boolean anyChanges() {
globalBufferLock.lock();
try {
- return !globalSlice.isEmpty() || globalBufferedDeletes.any();
+ /*
+ * check if all items in the global slice were applied
+ * and if the global slice is up-to-date
+ * and if globalBufferedDeletes has changes
+ */
+ return globalBufferedDeletes.any() || !globalSlice.isEmpty() || globalSlice.sliceTail != tail
+ || tail.next != null;
} finally {
globalBufferLock.unlock();
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/Fields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/Fields.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/Fields.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/Fields.java Thu Nov 3 13:35:07 2011
@@ -32,5 +32,31 @@ public abstract class Fields {
* null if the field does not exist. */
public abstract Terms terms(String field) throws IOException;
+ /** Returns the number of terms for all fields, or -1 if this
+ * measure isn't stored by the codec. Note that, just like
+ * other term measures, this measure does not take deleted
+ * documents into account. */
+ // TODO: deprecate?
+ public long getUniqueTermCount() throws IOException {
+ long numTerms = 0;
+ FieldsEnum it = iterator();
+ while(true) {
+ String field = it.next();
+ if (field == null) {
+ break;
+ }
+ Terms terms = terms(field);
+ if (terms != null) {
+ final long termCount = terms.getUniqueTermCount();
+ if (termCount == -1) {
+ return -1;
+ }
+
+ numTerms += termCount;
+ }
+ }
+ return numTerms;
+ }
+
public final static Fields[] EMPTY_ARRAY = new Fields[0];
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java Thu Nov 3 13:35:07 2011
@@ -561,10 +561,6 @@ public abstract class IndexReader implem
* with the old reader uses "copy on write" semantics to
* ensure the changes are not seen by other readers.
*
- * <p><b>NOTE</b>: If the provided reader is a near real-time
- * reader, this method will return another near-real-time
- * reader.
- *
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
* @return null if there are no changes; else, a new
@@ -672,18 +668,42 @@ public abstract class IndexReader implem
return newReader;
}
+ /**
+ * If the index has changed since it was opened, open and return a new reader;
+ * else, return {@code null}.
+ *
+ * @see #openIfChanged(IndexReader)
+ */
protected IndexReader doOpenIfChanged() throws CorruptIndexException, IOException {
throw new UnsupportedOperationException("This reader does not support reopen().");
}
+ /**
+ * If the index has changed since it was opened, open and return a new reader;
+ * else, return {@code null}.
+ *
+ * @see #openIfChanged(IndexReader, boolean)
+ */
protected IndexReader doOpenIfChanged(boolean openReadOnly) throws CorruptIndexException, IOException {
throw new UnsupportedOperationException("This reader does not support reopen().");
}
+ /**
+ * If the index has changed since it was opened, open and return a new reader;
+ * else, return {@code null}.
+ *
+ * @see #openIfChanged(IndexReader, IndexCommit)
+ */
protected IndexReader doOpenIfChanged(final IndexCommit commit) throws CorruptIndexException, IOException {
throw new UnsupportedOperationException("This reader does not support reopen(IndexCommit).");
}
+ /**
+ * If the index has changed since it was opened, open and return a new reader;
+ * else, return {@code null}.
+ *
+ * @see #openIfChanged(IndexReader, IndexWriter, boolean)
+ */
protected IndexReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) throws CorruptIndexException, IOException {
return writer.getReader(applyAllDeletes);
}
@@ -870,7 +890,7 @@ public abstract class IndexReader implem
* (ie, obtained by a call to {@link
* IndexWriter#getReader}, or by calling {@link #openIfChanged}
* on a near real-time reader), then this method checks if
- * either a new commmit has occurred, or any new
+ * either a new commit has occurred, or any new
* uncommitted changes have taken place via the writer.
* Note that even if the writer has only performed
* merging, this method will still return false.</p>
@@ -1593,26 +1613,17 @@ public abstract class IndexReader implem
/** Returns the number of unique terms (across all fields)
* in this reader.
*
- * @throws UnsupportedOperationException if this count
+ * @return number of unique terms or -1 if this count
* cannot be easily determined (eg Multi*Readers).
* Instead, you should call {@link
* #getSequentialSubReaders} and ask each sub reader for
* its unique term count. */
public long getUniqueTermCount() throws IOException {
- long numTerms = 0;
final Fields fields = fields();
if (fields == null) {
return 0;
}
- FieldsEnum it = fields.iterator();
- while(true) {
- String field = it.next();
- if (field == null) {
- break;
- }
- numTerms += fields.terms(field).getUniqueTermCount();
- }
- return numTerms;
+ return fields.getUniqueTermCount();
}
/** For IndexReader implementations that use
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriter.java Thu Nov 3 13:35:07 2011
@@ -764,7 +764,7 @@ public class IndexWriter implements Clos
if (reader != null) {
// the pulled reader could be from an in-flight merge
// while the info we see has already new applied deletes after a commit
- // we max out the delets since deletes never shrink
+ // we max out the deletes since deletes never shrink
return Math.max(info.getDelCount(), reader.numDeletedDocs());
} else {
return info.getDelCount();
@@ -4073,6 +4073,7 @@ public class IndexWriter implements Clos
synchronized boolean nrtIsCurrent(SegmentInfos infos) {
//System.out.println("IW.nrtIsCurrent " + (infos.version == segmentInfos.version && !docWriter.anyChanges() && !bufferedDeletesStream.any()));
+ ensureOpen();
return infos.version == segmentInfos.version && !docWriter.anyChanges() && !bufferedDeletesStream.any();
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableField.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableField.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableField.java Thu Nov 3 13:35:07 2011
@@ -40,7 +40,7 @@ public interface IndexableField {
// TODO: add attrs to this API?
- /* Field name */
+ /** Field name */
public String name();
// NOTE: if doc/field impl has the notion of "doc level boost"
@@ -49,24 +49,24 @@ public interface IndexableField {
/** Field boost (you must pre-multiply in any doc boost). */
public float boost();
- /* Non-null if this field has a binary value */
+ /** Non-null if this field has a binary value */
public BytesRef binaryValue();
- /* Non-null if this field has a string value */
+ /** Non-null if this field has a string value */
public String stringValue();
- /* Non-null if this field has a Reader value */
+ /** Non-null if this field has a Reader value */
public Reader readerValue();
// Numeric field:
- /* True if this field is numeric */
+ /** True if this field is numeric */
public boolean numeric();
- /* Numeric {@link NumericField.DataType}; only used if
+ /** Numeric {@link org.apache.lucene.document.NumericField.DataType}; only used if
* the field is numeric */
public NumericField.DataType numericDataType();
- /* Numeric value; only used if the field is numeric */
+ /** Numeric value; only used if the field is numeric */
public Number numericValue();
/**
@@ -76,10 +76,10 @@ public interface IndexableField {
*/
public IndexableFieldType fieldType();
- /* Non-null if doc values should be indexed */
+ /** Non-null if doc values should be indexed */
public PerDocFieldValues docValues();
- /* DocValues type; only used if docValues is non-null */
+ /** DocValues type; only used if docValues is non-null */
public ValueType docValuesType();
/**
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java Thu Nov 3 13:35:07 2011
@@ -21,28 +21,28 @@ import org.apache.lucene.index.FieldInfo
public interface IndexableFieldType {
- /* True if this field should be indexed (inverted) */
+ /** True if this field should be indexed (inverted) */
public boolean indexed();
- /* True if the field's value should be stored */
+ /** True if the field's value should be stored */
public boolean stored();
- /* True if this field's value should be analyzed */
+ /** True if this field's value should be analyzed */
public boolean tokenized();
- /* True if term vectors should be indexed */
+ /** True if term vectors should be indexed */
public boolean storeTermVectors();
- /* True if term vector offsets should be indexed */
+ /** True if term vector offsets should be indexed */
public boolean storeTermVectorOffsets();
- /* True if term vector positions should be indexed */
+ /** True if term vector positions should be indexed */
public boolean storeTermVectorPositions();
- /* True if norms should not be indexed */
+ /** True if norms should not be indexed */
public boolean omitNorms();
- /* {@link IndexOptions}, describing what should be
+ /** {@link IndexOptions}, describing what should be
* recorded into the inverted index */
public IndexOptions indexOptions();
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java Thu Nov 3 13:35:07 2011
@@ -29,6 +29,8 @@ import java.io.IOException;
*/
public final class MultiDocsAndPositionsEnum extends DocsAndPositionsEnum {
+ private final MultiTermsEnum parent;
+ final DocsAndPositionsEnum[] subDocsAndPositionsEnum;
private EnumWithSlice[] subs;
int numSubs;
int upto;
@@ -36,7 +38,16 @@ public final class MultiDocsAndPositions
int currentBase;
int doc = -1;
- MultiDocsAndPositionsEnum reset(final EnumWithSlice[] subs, final int numSubs) throws IOException {
+ public MultiDocsAndPositionsEnum(MultiTermsEnum parent, int subReaderCount) {
+ this.parent = parent;
+ subDocsAndPositionsEnum = new DocsAndPositionsEnum[subReaderCount];
+ }
+
+ public boolean canReuse(MultiTermsEnum parent) {
+ return this.parent == parent;
+ }
+
+ public MultiDocsAndPositionsEnum reset(final EnumWithSlice[] subs, final int numSubs) throws IOException {
this.numSubs = numSubs;
this.subs = new EnumWithSlice[subs.length];
for(int i=0;i<subs.length;i++) {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiDocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiDocsEnum.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiDocsEnum.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiDocsEnum.java Thu Nov 3 13:35:07 2011
@@ -27,6 +27,8 @@ import java.io.IOException;
*/
public final class MultiDocsEnum extends DocsEnum {
+ private final MultiTermsEnum parent;
+ final DocsEnum[] subDocsEnum;
private EnumWithSlice[] subs;
int numSubs;
int upto;
@@ -34,6 +36,11 @@ public final class MultiDocsEnum extends
int currentBase;
int doc = -1;
+ public MultiDocsEnum(MultiTermsEnum parent, int subReaderCount) {
+ this.parent = parent;
+ subDocsEnum = new DocsEnum[subReaderCount];
+ }
+
MultiDocsEnum reset(final EnumWithSlice[] subs, final int numSubs) throws IOException {
this.numSubs = numSubs;
@@ -48,6 +55,10 @@ public final class MultiDocsEnum extends
return this;
}
+ public boolean canReuse(MultiTermsEnum parent) {
+ return this.parent == parent;
+ }
+
public int getNumSubs() {
return numSubs;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiTerms.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiTerms.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiTerms.java Thu Nov 3 13:35:07 2011
@@ -96,6 +96,11 @@ public final class MultiTerms extends Te
}
@Override
+ public long getUniqueTermCount() throws IOException {
+ return -1;
+ }
+
+ @Override
public long getSumTotalTermFreq() throws IOException {
long sum = 0;
for(Terms terms : subs) {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java Thu Nov 3 13:35:07 2011
@@ -77,7 +77,7 @@ public final class MultiTermsEnum extend
subDocs = new MultiDocsEnum.EnumWithSlice[slices.length];
subDocsAndPositions = new MultiDocsAndPositionsEnum.EnumWithSlice[slices.length];
for(int i=0;i<slices.length;i++) {
- subs[i] = new TermsEnumWithSlice(slices[i]);
+ subs[i] = new TermsEnumWithSlice(i, slices[i]);
subDocs[i] = new MultiDocsEnum.EnumWithSlice();
subDocs[i].slice = slices[i];
subDocsAndPositions[i] = new MultiDocsAndPositionsEnum.EnumWithSlice();
@@ -347,11 +347,16 @@ public final class MultiTermsEnum extend
@Override
public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
- final MultiDocsEnum docsEnum;
- if (reuse != null) {
+ MultiDocsEnum docsEnum;
+ // Can only reuse if incoming enum is also a MultiDocsEnum
+ if (reuse != null && reuse instanceof MultiDocsEnum) {
docsEnum = (MultiDocsEnum) reuse;
+ // ... and was previously created w/ this MultiTermsEnum:
+ if (!docsEnum.canReuse(this)) {
+ docsEnum = new MultiDocsEnum(this, subs.length);
+ }
} else {
- docsEnum = new MultiDocsEnum();
+ docsEnum = new MultiDocsEnum(this, subs.length);
}
final MultiBits multiLiveDocs;
@@ -390,8 +395,11 @@ public final class MultiTermsEnum extend
b = null;
}
- final DocsEnum subDocsEnum = entry.terms.docs(b, null);
+ assert entry.index < docsEnum.subDocsEnum.length: entry.index + " vs " + docsEnum.subDocsEnum.length + "; " + subs.length;
+ final DocsEnum subDocsEnum = entry.terms.docs(b, docsEnum.subDocsEnum[entry.index]);
+
if (subDocsEnum != null) {
+ docsEnum.subDocsEnum[entry.index] = subDocsEnum;
subDocs[upto].docsEnum = subDocsEnum;
subDocs[upto].slice = entry.subSlice;
@@ -408,11 +416,16 @@ public final class MultiTermsEnum extend
@Override
public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException {
- final MultiDocsAndPositionsEnum docsAndPositionsEnum;
- if (reuse != null) {
+ MultiDocsAndPositionsEnum docsAndPositionsEnum;
+ // Can only reuse if incoming enum is also a MultiDocsAndPositionsEnum
+ if (reuse != null && reuse instanceof MultiDocsAndPositionsEnum) {
docsAndPositionsEnum = (MultiDocsAndPositionsEnum) reuse;
+ // ... and was previously created w/ this MultiTermsEnum:
+ if (!docsAndPositionsEnum.canReuse(this)) {
+ docsAndPositionsEnum = new MultiDocsAndPositionsEnum(this, subs.length);
+ }
} else {
- docsAndPositionsEnum = new MultiDocsAndPositionsEnum();
+ docsAndPositionsEnum = new MultiDocsAndPositionsEnum(this, subs.length);
}
final MultiBits multiLiveDocs;
@@ -452,9 +465,11 @@ public final class MultiTermsEnum extend
b = null;
}
- final DocsAndPositionsEnum subPostings = entry.terms.docsAndPositions(b, null);
+ assert entry.index < docsAndPositionsEnum.subDocsAndPositionsEnum.length: entry.index + " vs " + docsAndPositionsEnum.subDocsAndPositionsEnum.length + "; " + subs.length;
+ final DocsAndPositionsEnum subPostings = entry.terms.docsAndPositions(b, docsAndPositionsEnum.subDocsAndPositionsEnum[entry.index]);
if (subPostings != null) {
+ docsAndPositionsEnum.subDocsAndPositionsEnum[entry.index] = subPostings;
subDocsAndPositions[upto].docsAndPositionsEnum = subPostings;
subDocsAndPositions[upto].slice = entry.subSlice;
upto++;
@@ -479,9 +494,11 @@ public final class MultiTermsEnum extend
private final ReaderUtil.Slice subSlice;
private TermsEnum terms;
public BytesRef current;
+ final int index;
- public TermsEnumWithSlice(ReaderUtil.Slice subSlice) {
+ public TermsEnumWithSlice(int index, ReaderUtil.Slice subSlice) {
this.subSlice = subSlice;
+ this.index = index;
assert subSlice.length >= 0: "length=" + subSlice.length;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/NoMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/NoMergeScheduler.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/NoMergeScheduler.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/NoMergeScheduler.java Thu Nov 3 13:35:07 2011
@@ -23,7 +23,7 @@ import java.io.IOException;
* A {@link MergeScheduler} which never executes any merges. It is also a
* singleton and can be accessed through {@link NoMergeScheduler#INSTANCE}. Use
* it if you want to prevent an {@link IndexWriter} from ever executing merges,
- * irregardless of the {@link MergePolicy} used. Note that you can achieve the
+ * regardless of the {@link MergePolicy} used. Note that you can achieve the
* same thing by using {@link NoMergePolicy}, however with
* {@link NoMergeScheduler} you also ensure that no unnecessary code of any
* {@link MergeScheduler} implementation is ever executed. Hence it is
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/ParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/ParallelReader.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/ParallelReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/ParallelReader.java Thu Nov 3 13:35:07 2011
@@ -569,7 +569,9 @@ public class ParallelReader extends Inde
void addField(String field, IndexReader r) throws IOException {
PerDocValues perDocs = MultiPerDocValues.getPerDocs(r);
- fields.put(field, perDocs.docValues(field));
+ if (perDocs != null) {
+ fields.put(field, perDocs.docValues(field));
+ }
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java Thu Nov 3 13:35:07 2011
@@ -41,10 +41,10 @@ import org.apache.lucene.util.Version;
* a Lucene index. It is highly recommended to use a dedicated directory (and on
* stable storage as well) for persisting the snapshots' information, and not
* reuse the content index directory, or otherwise conflicts and index
- * corruptions will occur.
+ * corruption will occur.
* <p>
* <b>NOTE:</b> you should call {@link #close()} when you're done using this
- * class for safetyness (it will close the {@link IndexWriter} instance used).
+ * class for safety (it will close the {@link IndexWriter} instance used).
*/
public class PersistentSnapshotDeletionPolicy extends SnapshotDeletionPolicy {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java Thu Nov 3 13:35:07 2011
@@ -25,6 +25,7 @@ import java.util.Set;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.index.codecs.preflex.PreFlexCodec;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
@@ -65,7 +66,7 @@ final class SegmentCodecs implements Clo
*/
final Codec[] codecs;
final CodecProvider provider;
- private final Codec codec = new PerFieldCodecWrapper(this);
+ private final Codec codec;
SegmentCodecs(CodecProvider provider, IndexInput input) throws IOException {
this(provider, read(input, provider));
@@ -74,6 +75,11 @@ final class SegmentCodecs implements Clo
SegmentCodecs(CodecProvider provider, Codec... codecs) {
this.provider = provider;
this.codecs = codecs;
+ if (codecs.length == 1 && codecs[0] instanceof PreFlexCodec) {
+ this.codec = codecs[0]; // hack for backwards break... don't wrap the codec in preflex
+ } else {
+ this.codec = new PerFieldCodecWrapper(this);
+ }
}
Codec codec() {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Thu Nov 3 13:35:07 2011
@@ -695,7 +695,7 @@ public final class SegmentInfo implement
* <p>Current format looks like
* <code>_a(3.1):c45/4->_1</code>, which means the segment's
* name is <code>_a</code>; it was created with Lucene 3.1 (or
- * '?' if it's unkown); it's using compound file
+ * '?' if it's unknown); it's using compound file
* format (would be <code>C</code> if not compound); it
* has 45 documents; it has 4 deletions (this part is
* left off when there are no deletions); it's using the
@@ -718,7 +718,7 @@ public final class SegmentInfo implement
}
} catch (Throwable e) {
// Messy: because getHasVectors may be used in an
- // un-thread-safe way, and may attempt to open an fnm
+ // thread-unsafe way, and may attempt to open an fnm
// file that has since (legitimately) been deleted by
// IndexWriter, instead of throwing these exceptions
// up, just add v? to indicate we don't know if this
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Thu Nov 3 13:35:07 2011
@@ -586,6 +586,11 @@ final class SegmentMerger {
private void mergePerDoc() throws IOException {
final PerDocConsumer docsConsumer = codec
.docsConsumer(new PerDocWriteState(segmentWriteState));
+ // TODO: remove this check when 3.x indexes are no longer supported
+ // (3.x indexes don't have docvalues)
+ if (docsConsumer == null) {
+ return;
+ }
boolean success = false;
try {
docsConsumer.merge(mergeState);
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/Terms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/Terms.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/Terms.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/Terms.java Thu Nov 3 13:35:07 2011
@@ -155,10 +155,12 @@ public abstract class Terms {
return termsEnum.docsAndPositions(liveDocs, reuse);
}
- public long getUniqueTermCount() throws IOException {
- throw new UnsupportedOperationException("this reader does not implement getUniqueTermCount()");
- }
-
+ /** Returns the number of terms for this field, or -1 if this
+ * measure isn't stored by the codec. Note that, just like
+ * other term measures, this measure does not take deleted
+ * documents into account. */
+ public abstract long getUniqueTermCount() throws IOException;
+
/** Returns the sum of {@link TermsEnum#totalTermFreq} for
* all terms in this field, or -1 if this measure isn't
* stored by the codec (or if this fields omits term freq
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermsEnum.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermsEnum.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermsEnum.java Thu Nov 3 13:35:07 2011
@@ -147,7 +147,7 @@ public abstract class TermsEnum {
* call this when the enum is unpositioned. This method
* will not return null.
*
- * @param liveDocs set bits are documents that should not
+ * @param liveDocs unset bits are documents that should not
* be returned
* @param reuse pass a prior DocsEnum for possible reuse */
public abstract DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException;
@@ -155,7 +155,10 @@ public abstract class TermsEnum {
/** Get {@link DocsAndPositionsEnum} for the current term.
* Do not call this when the enum is unpositioned.
* This method will only return null if positions were
- * not indexed into the postings by this codec. */
+ * not indexed into the postings by this codec.
+ * @param liveDocs unset bits are documents that should not
+ * be returned
+ * @param reuse pass a prior DocsAndPositionsEnum for possible reuse */
public abstract DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException;
/**
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java Thu Nov 3 13:35:07 2011
@@ -39,7 +39,7 @@ import java.util.ArrayList;
* <p>For normal merging, this policy first computes a
* "budget" of how many segments are allowed by be in the
* index. If the index is over-budget, then the policy
- * sorts segments by decresing size (pro-rating by percent
+ * sorts segments by decreasing size (pro-rating by percent
* deletes), and then finds the least-cost merge. Merge
* cost is measured by a combination of the "skew" of the
* merge (size of largest seg divided by smallest seg),
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java Thu Nov 3 13:35:07 2011
@@ -766,7 +766,7 @@ public class BlockTreeTermsReader extend
arcs[arcIdx] = new FST.Arc<BytesRef>();
}
- // TODO: if the automaon is "smallish" we really
+ // TODO: if the automaton is "smallish" we really
// should use the terms index to seek at least to
// the initial term and likely to subsequent terms
// (or, maybe just fallback to ATE for such cases).
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java Thu Nov 3 13:35:07 2011
@@ -75,7 +75,7 @@ import org.apache.lucene.util.fst.FST;
* queries that rely on advance will (AND BooleanQuery,
* PhraseQuery) will be relatively slow!
*
- * <p><b>NOTE</b>: this codec cannot adress more than ~2.1 GB
+ * <p><b>NOTE</b>: this codec cannot address more than ~2.1 GB
* of postings, because the underlying FST uses an int
* to address the underlying byte[].
*
@@ -684,11 +684,13 @@ public class MemoryCodec extends Codec {
private final long sumTotalTermFreq;
private final long sumDocFreq;
private final int docCount;
+ private final int termCount;
private FST<BytesRef> fst;
private final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
private final FieldInfo field;
- public TermsReader(FieldInfos fieldInfos, IndexInput in) throws IOException {
+ public TermsReader(FieldInfos fieldInfos, IndexInput in, int termCount) throws IOException {
+ this.termCount = termCount;
final int fieldNumber = in.readVInt();
field = fieldInfos.fieldInfo(fieldNumber);
if (field.indexOptions != IndexOptions.DOCS_ONLY) {
@@ -718,6 +720,11 @@ public class MemoryCodec extends Codec {
}
@Override
+ public long getUniqueTermCount() throws IOException {
+ return termCount;
+ }
+
+ @Override
public TermsEnum iterator() {
return new FSTTermsEnum(field, fst);
}
@@ -741,7 +748,7 @@ public class MemoryCodec extends Codec {
if (termCount == 0) {
break;
}
- final TermsReader termsReader = new TermsReader(state.fieldInfos, in);
+ final TermsReader termsReader = new TermsReader(state.fieldInfos, in, termCount);
fields.put(termsReader.field.name, termsReader);
}
} finally {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java Thu Nov 3 13:35:07 2011
@@ -84,11 +84,11 @@ public class PreFlexCodec extends Codec
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- throw new UnsupportedOperationException("PerDocConsumer is not supported by Preflex codec");
+ return null;
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- throw new UnsupportedOperationException("PerDocValues is not supported by Preflex codec");
+ return null;
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java Thu Nov 3 13:35:07 2011
@@ -162,6 +162,11 @@ public class PreFlexFields extends Field
return preTerms.get(field);
}
+ @Override
+ public long getUniqueTermCount() throws IOException {
+ return getTermsDict().size();
+ }
+
synchronized private TermInfosReader getTermsDict() {
if (tis != null) {
return tis;
@@ -241,6 +246,11 @@ public class PreFlexFields extends Field
}
@Override
+ public long getUniqueTermCount() throws IOException {
+ return -1;
+ }
+
+ @Override
public long getSumTotalTermFreq() {
return -1;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java Thu Nov 3 13:35:07 2011
@@ -61,6 +61,7 @@ public final class SegmentTermEnum imple
int skipInterval;
int newSuffixStart;
int maxSkipLevels;
+ private boolean first = true;
SegmentTermEnum(IndexInput i, FieldInfos fis, boolean isi)
throws CorruptIndexException, IOException {
@@ -123,6 +124,7 @@ public final class SegmentTermEnum imple
prevBuffer.reset();
//System.out.println(" ste doSeek prev=" + prevBuffer.toTerm() + " this=" + this);
termInfo.set(ti);
+ first = p == -1;
}
/** Increments the enumeration to the next element. True if one exists.*/
@@ -162,6 +164,13 @@ public final class SegmentTermEnum imple
final int scanTo(Term term) throws IOException {
scanBuffer.set(term);
int count = 0;
+ if (first) {
+ // Always force initial next() in case term is
+ // Term("", "")
+ next();
+ first = false;
+ count++;
+ }
while (scanBuffer.compareTo(termBuffer) > 0 && next()) {
count++;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java Thu Nov 3 13:35:07 2011
@@ -47,9 +47,8 @@ public final class TermInfosReader {
private final SegmentTermEnum origEnum;
private final long size;
- private final Term[] indexTerms;
- private final TermInfo[] indexInfos;
- private final long[] indexPointers;
+ private final TermInfosReaderIndex index;
+ private final int indexLength;
private final int totalIndexInterval;
@@ -118,37 +117,22 @@ public final class TermInfosReader {
if (indexDivisor != -1) {
// Load terms index
totalIndexInterval = origEnum.indexInterval * indexDivisor;
- final SegmentTermEnum indexEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_INDEX_EXTENSION),
- context), fieldInfos, true);
- try {
- int indexSize = 1+((int)indexEnum.size-1)/indexDivisor; // otherwise read index
+ final String indexFileName = IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_INDEX_EXTENSION);
+ final SegmentTermEnum indexEnum = new SegmentTermEnum(directory.openInput(indexFileName,
+ context), fieldInfos, true);
- indexTerms = new Term[indexSize];
- indexInfos = new TermInfo[indexSize];
- indexPointers = new long[indexSize];
-
- for (int i=0;indexEnum.next(); i++) {
- indexTerms[i] = indexEnum.term();
- assert indexTerms[i] != null;
- assert indexTerms[i].text() != null;
- assert indexTerms[i].field() != null;
- indexInfos[i] = indexEnum.termInfo();
- indexPointers[i] = indexEnum.indexPointer;
-
- for (int j = 1; j < indexDivisor; j++)
- if (!indexEnum.next())
- break;
- }
+ try {
+ index = new TermInfosReaderIndex(indexEnum, indexDivisor, dir.fileLength(indexFileName), totalIndexInterval);
+ indexLength = index.length();
} finally {
indexEnum.close();
}
} else {
// Do not load terms index:
totalIndexInterval = -1;
- indexTerms = null;
- indexInfos = null;
- indexPointers = null;
+ index = null;
+ indexLength = -1;
}
success = true;
} finally {
@@ -203,31 +187,6 @@ public final class TermInfosReader {
}
}
- /** Returns the offset of the greatest index entry which is less than or equal to term.*/
- private int getIndexOffset(Term term) {
- int lo = 0; // binary search indexTerms[]
- int hi = indexTerms.length - 1;
-
- while (hi >= lo) {
- int mid = (lo + hi) >>> 1;
- assert indexTerms[mid] != null : "indexTerms = " + indexTerms.length + " mid=" + mid;
- int delta = compareAsUTF16(term, indexTerms[mid]);
- if (delta < 0)
- hi = mid - 1;
- else if (delta > 0)
- lo = mid + 1;
- else
- return mid;
- }
- return hi;
- }
-
- private void seekEnum(SegmentTermEnum enumerator, int indexOffset) throws IOException {
- enumerator.seek(indexPointers[indexOffset],
- ((long) indexOffset * totalIndexInterval) - 1,
- indexTerms[indexOffset], indexInfos[indexOffset]);
- }
-
/** Returns the TermInfo for a Term in the set, or null. */
TermInfo get(Term term) throws IOException {
return get(term, false);
@@ -272,8 +231,8 @@ public final class TermInfosReader {
&& ((enumerator.prev() != null && compareAsUTF16(term, enumerator.prev())> 0)
|| compareAsUTF16(term, enumerator.term()) >= 0)) {
int enumOffset = (int)(enumerator.position/totalIndexInterval)+1;
- if (indexTerms.length == enumOffset // but before end of block
- || compareAsUTF16(term, indexTerms[enumOffset]) < 0) {
+ if (indexLength == enumOffset // but before end of block
+ || index.compareTo(term, enumOffset) < 0) {
// no need to seek
final TermInfo ti;
@@ -309,10 +268,10 @@ public final class TermInfosReader {
indexPos = (int) (tiOrd.termOrd / totalIndexInterval);
} else {
// Must do binary search:
- indexPos = getIndexOffset(term);
+ indexPos = index.getIndexOffset(term);
}
- seekEnum(enumerator, indexPos);
+ index.seekEnum(enumerator, indexPos);
enumerator.scanTo(term);
final TermInfo ti;
@@ -320,13 +279,7 @@ public final class TermInfosReader {
ti = enumerator.termInfo;
if (tiOrd == null) {
if (useCache) {
- // LUCENE-3183: it's possible, if term is Term("",
- // ""), for the STE to be incorrectly un-positioned
- // after scan-to; work around this by not caching in
- // this case:
- if (enumerator.position >= 0) {
- termsCache.put(new CloneableTerm(term), new TermInfoAndOrd(ti, enumerator.position));
- }
+ termsCache.put(new CloneableTerm(term), new TermInfoAndOrd(ti, enumerator.position));
}
} else {
assert sameTermInfo(ti, tiOrd, enumerator);
@@ -358,7 +311,7 @@ public final class TermInfosReader {
}
private void ensureIndexIsRead() {
- if (indexTerms == null) {
+ if (index == null) {
throw new IllegalStateException("terms index was not loaded when this reader was created");
}
}
@@ -368,10 +321,10 @@ public final class TermInfosReader {
if (size == 0) return -1;
ensureIndexIsRead();
- int indexOffset = getIndexOffset(term);
+ int indexOffset = index.getIndexOffset(term);
SegmentTermEnum enumerator = getThreadResources().termEnum;
- seekEnum(enumerator, indexOffset);
+ index.seekEnum(enumerator, indexOffset);
while(compareAsUTF16(term, enumerator.term()) > 0 && enumerator.next()) {}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java Thu Nov 3 13:35:07 2011
@@ -18,6 +18,8 @@ package org.apache.lucene.index.codecs.p
*/
import java.io.IOException;
+import java.util.IdentityHashMap;
+import java.util.Map;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
@@ -29,6 +31,9 @@ import org.apache.lucene.index.codecs.Bl
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CodecUtil;
@@ -172,8 +177,6 @@ public class PulsingPostingsReader exten
}
}
- // TODO: we could actually reuse, by having TL that
- // holds the last wrapped reuse, and vice-versa
@Override
public DocsEnum docs(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsEnum reuse) throws IOException {
PulsingTermState termState = (PulsingTermState) _termState;
@@ -185,20 +188,29 @@ public class PulsingPostingsReader exten
postings = new PulsingDocsEnum(field);
}
} else {
- postings = new PulsingDocsEnum(field);
+ // the 'reuse' is actually the wrapped enum
+ PulsingDocsEnum previous = (PulsingDocsEnum) getOther(reuse);
+ if (previous != null && previous.canReuse(field)) {
+ postings = previous;
+ } else {
+ postings = new PulsingDocsEnum(field);
+ }
+ }
+ if (reuse != postings) {
+ setOther(postings, reuse); // postings.other = reuse
}
return postings.reset(liveDocs, termState);
} else {
- // TODO: not great that we lose reuse of PulsingDocsEnum in this case:
if (reuse instanceof PulsingDocsEnum) {
- return wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, null);
+ DocsEnum wrapped = wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, getOther(reuse));
+ setOther(wrapped, reuse); // wrapped.other = reuse
+ return wrapped;
} else {
return wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, reuse);
}
}
}
- // TODO: -- not great that we can't always reuse
@Override
public DocsAndPositionsEnum docsAndPositions(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException {
if (field.indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
@@ -216,13 +228,23 @@ public class PulsingPostingsReader exten
postings = new PulsingDocsAndPositionsEnum(field);
}
} else {
- postings = new PulsingDocsAndPositionsEnum(field);
+ // the 'reuse' is actually the wrapped enum
+ PulsingDocsAndPositionsEnum previous = (PulsingDocsAndPositionsEnum) getOther(reuse);
+ if (previous != null && previous.canReuse(field)) {
+ postings = previous;
+ } else {
+ postings = new PulsingDocsAndPositionsEnum(field);
+ }
+ }
+ if (reuse != postings) {
+ setOther(postings, reuse); // postings.other = reuse
}
-
return postings.reset(liveDocs, termState);
} else {
if (reuse instanceof PulsingDocsAndPositionsEnum) {
- return wrappedPostingsReader.docsAndPositions(field, termState.wrappedTermState, liveDocs, null);
+ DocsAndPositionsEnum wrapped = wrappedPostingsReader.docsAndPositions(field, termState.wrappedTermState, liveDocs, (DocsAndPositionsEnum) getOther(reuse));
+ setOther(wrapped, reuse); // wrapped.other = reuse
+ return wrapped;
} else {
return wrappedPostingsReader.docsAndPositions(field, termState.wrappedTermState, liveDocs, reuse);
}
@@ -499,4 +521,69 @@ public class PulsingPostingsReader exten
public void close() throws IOException {
wrappedPostingsReader.close();
}
+
+ /** for a docsenum, gets the 'other' reused enum.
+ * Example: Pulsing(Standard).
+ * when doing a term range query you are switching back and forth
+ * between Pulsing and Standard
+ *
+ * The way the reuse works is that Pulsing.other = Standard and
+ * Standard.other = Pulsing.
+ */
+ private DocsEnum getOther(DocsEnum de) {
+ if (de == null) {
+ return null;
+ } else {
+ final AttributeSource atts = de.attributes();
+ return atts.addAttribute(PulsingEnumAttribute.class).enums().get(this);
+ }
+ }
+
+ /**
+ * for a docsenum, sets the 'other' reused enum.
+ * see getOther for an example.
+ */
+ private DocsEnum setOther(DocsEnum de, DocsEnum other) {
+ final AttributeSource atts = de.attributes();
+ return atts.addAttribute(PulsingEnumAttribute.class).enums().put(this, other);
+ }
+
+ /**
+ * A per-docsenum attribute that stores additional reuse information
+ * so that pulsing enums can keep a reference to their wrapped enums,
+ * and vice versa. this way we can always reuse.
+ *
+ * @lucene.internal */
+ public static interface PulsingEnumAttribute extends Attribute {
+ public Map<PulsingPostingsReader,DocsEnum> enums();
+ }
+
+ /** @lucene.internal */
+ public static final class PulsingEnumAttributeImpl extends AttributeImpl implements PulsingEnumAttribute {
+ // we could store 'other', but what if someone 'chained' multiple postings readers,
+ // this could cause problems?
+ // TODO: we should consider nuking this map and just making it so if you do this,
+ // you don't reuse? and maybe pulsingPostingsReader should throw an exc if it wraps
+ // another pulsing, because this is just stupid and wasteful.
+ // we still have to be careful in case someone does Pulsing(Stomping(Pulsing(...
+ private final Map<PulsingPostingsReader,DocsEnum> enums =
+ new IdentityHashMap<PulsingPostingsReader,DocsEnum>();
+
+ public Map<PulsingPostingsReader,DocsEnum> enums() {
+ return enums;
+ }
+
+ @Override
+ public void clear() {
+ // our state is per-docsenum, so this makes no sense.
+ // its best not to clear, in case a wrapped enum has a per-doc attribute or something
+ // and is calling clearAttributes(), so they don't nuke the reuse information!
+ }
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ // this makes no sense for us, because our state is per-docsenum.
+ // we don't want to copy any stuff over to another docsenum ever!
+ }
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java Thu Nov 3 13:35:07 2011
@@ -216,7 +216,7 @@ class SimpleTextFieldsReader extends Fie
@Override
public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
SimpleTextDocsEnum docsEnum;
- if (reuse != null && reuse instanceof SimpleTextDocsEnum && ((SimpleTextDocsEnum) reuse).canReuse(in)) {
+ if (reuse != null && reuse instanceof SimpleTextDocsEnum && ((SimpleTextDocsEnum) reuse).canReuse(SimpleTextFieldsReader.this.in)) {
docsEnum = (SimpleTextDocsEnum) reuse;
} else {
docsEnum = new SimpleTextDocsEnum();
@@ -231,7 +231,7 @@ class SimpleTextFieldsReader extends Fie
}
SimpleTextDocsAndPositionsEnum docsAndPositionsEnum;
- if (reuse != null && reuse instanceof SimpleTextDocsAndPositionsEnum && ((SimpleTextDocsAndPositionsEnum) reuse).canReuse(in)) {
+ if (reuse != null && reuse instanceof SimpleTextDocsAndPositionsEnum && ((SimpleTextDocsAndPositionsEnum) reuse).canReuse(SimpleTextFieldsReader.this.in)) {
docsAndPositionsEnum = (SimpleTextDocsAndPositionsEnum) reuse;
} else {
docsAndPositionsEnum = new SimpleTextDocsAndPositionsEnum();
@@ -249,7 +249,7 @@ class SimpleTextFieldsReader extends Fie
private final IndexInput inStart;
private final IndexInput in;
private boolean omitTF;
- private int docID;
+ private int docID = -1;
private int tf;
private Bits liveDocs;
private final BytesRef scratch = new BytesRef(10);
@@ -268,6 +268,7 @@ class SimpleTextFieldsReader extends Fie
this.liveDocs = liveDocs;
in.seek(fp);
this.omitTF = omitTF;
+ docID = -1;
if (omitTF) {
tf = 1;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Bytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Bytes.java?rev=1197110&r1=1197109&r2=1197110&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Bytes.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Bytes.java Thu Nov 3 13:35:07 2011
@@ -107,7 +107,7 @@ public final class Bytes {
* @param bytesUsed
* an {@link AtomicLong} instance to track the used bytes within the
* {@link Writer}. A call to {@link Writer#finish(int)} will release
- * all internally used resources and frees the memeory tracking
+ * all internally used resources and frees the memory tracking
* reference.
* @param context
* @return a new {@link Writer} instance
@@ -334,17 +334,17 @@ public final class Bytes {
IndexInput indexIn = null;
boolean success = false;
try {
- dataIn = dir.openInput(IndexFileNames.segmentFileName(id, "",
- Writer.DATA_EXTENSION), context);
- version = CodecUtil.checkHeader(dataIn, codecName, maxVersion, maxVersion);
- if (doIndex) {
- indexIn = dir.openInput(IndexFileNames.segmentFileName(id, "",
- Writer.INDEX_EXTENSION), context);
- final int version2 = CodecUtil.checkHeader(indexIn, codecName,
- maxVersion, maxVersion);
- assert version == version2;
- }
- success = true;
+ dataIn = dir.openInput(IndexFileNames.segmentFileName(id, "",
+ Writer.DATA_EXTENSION), context);
+ version = CodecUtil.checkHeader(dataIn, codecName, maxVersion, maxVersion);
+ if (doIndex) {
+ indexIn = dir.openInput(IndexFileNames.segmentFileName(id, "",
+ Writer.INDEX_EXTENSION), context);
+ final int version2 = CodecUtil.checkHeader(indexIn, codecName,
+ maxVersion, maxVersion);
+ assert version == version2;
+ }
+ success = true;
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(dataIn, indexIn);
@@ -393,6 +393,7 @@ public final class Bytes {
protected int lastDocId = -1;
protected int[] docToEntry;
protected final BytesRefHash hash;
+ protected long maxBytes = 0;
protected DerefBytesWriterBase(Directory dir, String id, String codecName,
int codecVersion, Counter bytesUsed, IOContext context)
@@ -433,8 +434,11 @@ public final class Bytes {
int ord = hash.add(bytes);
if (ord < 0) {
ord = (-ord) - 1;
+ } else {
+ maxBytes += bytes.length;
}
+
docToEntry[docID] = ord;
lastDocId = docID;
}
@@ -554,6 +558,8 @@ public final class Bytes {
private final PagedBytes pagedBytes;
protected final PackedInts.Reader docToOrdIndex;
+ protected final PackedInts.Reader ordToOffsetIndex;
+
protected final IndexInput datIn;
protected final IndexInput idxIn;
protected final BytesRef defaultValue = new BytesRef();
@@ -561,12 +567,12 @@ public final class Bytes {
protected final PagedBytes.Reader data;
protected BytesSortedSourceBase(IndexInput datIn, IndexInput idxIn,
- Comparator<BytesRef> comp, long bytesToRead, ValueType type) throws IOException {
- this(datIn, idxIn, comp, new PagedBytes(PAGED_BYTES_BITS), bytesToRead, type);
+ Comparator<BytesRef> comp, long bytesToRead, ValueType type, boolean hasOffsets) throws IOException {
+ this(datIn, idxIn, comp, new PagedBytes(PAGED_BYTES_BITS), bytesToRead, type, hasOffsets);
}
protected BytesSortedSourceBase(IndexInput datIn, IndexInput idxIn,
- Comparator<BytesRef> comp, PagedBytes pagedBytes, long bytesToRead,ValueType type)
+ Comparator<BytesRef> comp, PagedBytes pagedBytes, long bytesToRead, ValueType type, boolean hasOffsets)
throws IOException {
super(type, comp);
assert bytesToRead <= datIn.length() : " file size is less than the expected size diff: "
@@ -576,24 +582,19 @@ public final class Bytes {
this.pagedBytes.copy(datIn, bytesToRead);
data = pagedBytes.freeze(true);
this.idxIn = idxIn;
+ ordToOffsetIndex = hasOffsets ? PackedInts.getReader(idxIn) : null;
docToOrdIndex = PackedInts.getReader(idxIn);
}
@Override
public int ord(int docID) {
+ assert docToOrdIndex.get(docID) < getValueCount();
return (int) docToOrdIndex.get(docID);
}
protected void closeIndexInput() throws IOException {
IOUtils.close(datIn, idxIn);
}
-
- /**
- * Returns the largest doc id + 1 in this doc values source
- */
- public int maxDoc() {
- return docToOrdIndex.size();
- }
}
}