You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ha...@apache.org on 2013/08/30 17:06:49 UTC
svn commit: r1518989 [4/10] - in /lucene/dev/branches/lucene3069: ./
dev-tools/ dev-tools/idea/solr/contrib/velocity/ dev-tools/maven/
dev-tools/maven/solr/core/src/java/ lucene/ lucene/analysis/
lucene/analysis/common/ lucene/analysis/common/src/java/...
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java Fri Aug 30 15:06:42 2013
@@ -71,12 +71,16 @@ abstract class DocumentsWriterPerThreadP
* for indexing anymore.
* @see #isActive()
*/
- private void resetWriter(DocumentsWriterPerThread dwpt) {
+
+ private void deactivate() {
assert this.isHeldByCurrentThread();
- if (dwpt == null) {
- isActive = false;
- }
- this.dwpt = dwpt;
+ isActive = false;
+ reset();
+ }
+
+ private void reset() {
+ assert this.isHeldByCurrentThread();
+ this.dwpt = null;
this.bytesUsed = 0;
this.flushPending = false;
}
@@ -91,6 +95,11 @@ abstract class DocumentsWriterPerThreadP
return isActive;
}
+ boolean isInitialized() {
+ assert this.isHeldByCurrentThread();
+ return isActive() && dwpt != null;
+ }
+
/**
* Returns the number of currently active bytes in this ThreadState's
* {@link DocumentsWriterPerThread}
@@ -121,9 +130,7 @@ abstract class DocumentsWriterPerThreadP
private ThreadState[] threadStates;
private volatile int numThreadStatesActive;
- private SetOnce<FieldNumbers> globalFieldMap = new SetOnce<FieldNumbers>();
- private SetOnce<DocumentsWriter> documentsWriter = new SetOnce<DocumentsWriter>();
-
+
/**
* Creates a new {@link DocumentsWriterPerThreadPool} with a given maximum of {@link ThreadState}s.
*/
@@ -133,14 +140,8 @@ abstract class DocumentsWriterPerThreadP
}
threadStates = new ThreadState[maxNumThreadStates];
numThreadStatesActive = 0;
- }
-
- void initialize(DocumentsWriter documentsWriter, FieldNumbers globalFieldMap, LiveIndexWriterConfig config) {
- this.documentsWriter.set(documentsWriter); // thread pool is bound to DW
- this.globalFieldMap.set(globalFieldMap);
for (int i = 0; i < threadStates.length; i++) {
- final FieldInfos.Builder infos = new FieldInfos.Builder(globalFieldMap);
- threadStates[i] = new ThreadState(new DocumentsWriterPerThread(documentsWriter.directory, documentsWriter, infos, documentsWriter.chain));
+ threadStates[i] = new ThreadState(null);
}
}
@@ -158,9 +159,10 @@ abstract class DocumentsWriterPerThreadP
// should not happen
throw new RuntimeException(e);
}
- clone.documentsWriter = new SetOnce<DocumentsWriter>();
- clone.globalFieldMap = new SetOnce<FieldNumbers>();
clone.threadStates = new ThreadState[threadStates.length];
+ for (int i = 0; i < threadStates.length; i++) {
+ clone.threadStates[i] = new ThreadState(null);
+ }
return clone;
}
@@ -178,6 +180,7 @@ abstract class DocumentsWriterPerThreadP
int getActiveThreadState() {
return numThreadStatesActive;
}
+
/**
* Returns a new {@link ThreadState} iff any new state is available otherwise
@@ -198,8 +201,7 @@ abstract class DocumentsWriterPerThreadP
if (threadState.isActive()) {
// unreleased thread states are deactivated during DW#close()
numThreadStatesActive++; // increment will publish the ThreadState
- assert threadState.dwpt != null;
- threadState.dwpt.initialize();
+ assert threadState.dwpt == null;
unlock = false;
return threadState;
}
@@ -220,7 +222,7 @@ abstract class DocumentsWriterPerThreadP
for (int i = numThreadStatesActive; i < threadStates.length; i++) {
assert threadStates[i].tryLock() : "unreleased threadstate should not be locked";
try {
- assert !threadStates[i].isActive() : "expected unreleased thread state to be inactive";
+ assert !threadStates[i].isInitialized() : "expected unreleased thread state to be inactive";
} finally {
threadStates[i].unlock();
}
@@ -236,24 +238,20 @@ abstract class DocumentsWriterPerThreadP
final ThreadState threadState = threadStates[i];
threadState.lock();
try {
- threadState.resetWriter(null);
+ threadState.deactivate();
} finally {
threadState.unlock();
}
}
}
- DocumentsWriterPerThread replaceForFlush(ThreadState threadState, boolean closed) {
+ DocumentsWriterPerThread reset(ThreadState threadState, boolean closed) {
assert threadState.isHeldByCurrentThread();
- assert globalFieldMap.get() != null;
final DocumentsWriterPerThread dwpt = threadState.dwpt;
if (!closed) {
- final FieldInfos.Builder infos = new FieldInfos.Builder(globalFieldMap.get());
- final DocumentsWriterPerThread newDwpt = new DocumentsWriterPerThread(dwpt, infos);
- newDwpt.initialize();
- threadState.resetWriter(newDwpt);
+ threadState.reset();
} else {
- threadState.resetWriter(null);
+ threadState.deactivate();
}
return dwpt;
}
@@ -328,18 +326,6 @@ abstract class DocumentsWriterPerThreadP
*/
void deactivateThreadState(ThreadState threadState) {
assert threadState.isActive();
- threadState.resetWriter(null);
- }
-
- /**
- * Reinitialized an active {@link ThreadState}. A {@link ThreadState} should
- * only be reinitialized if it is active without any pending documents.
- *
- * @param threadState the state to reinitialize
- */
- void reinitThreadState(ThreadState threadState) {
- assert threadState.isActive;
- assert threadState.dwpt.getNumDocsInRAM() == 0;
- threadState.dwpt.initialize();
+ threadState.deactivate();
}
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java Fri Aug 30 15:06:42 2013
@@ -92,21 +92,22 @@ public final class FieldInfo {
*/
NUMERIC,
/**
- * A per-document byte[].
+ * A per-document byte[]. Values may be larger than
+ * 32766 bytes, but different codecs may enforce their own limits.
*/
BINARY,
/**
* A pre-sorted byte[]. Fields with this type only store distinct byte values
* and store an additional offset pointer per document to dereference the shared
* byte[]. The stored byte[] is presorted and allows access via document id,
- * ordinal and by-value.
+ * ordinal and by-value. Values must be <= 32766 bytes.
*/
SORTED,
/**
* A pre-sorted Set<byte[]>. Fields with this type only store distinct byte values
* and store additional offset pointers per document to dereference the shared
* byte[]s. The stored byte[] is presorted and allows access via document id,
- * ordinal and by-value.
+ * ordinal and by-value. Values must be <= 32766 bytes.
*/
SORTED_SET
};
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java Fri Aug 30 15:06:42 2013
@@ -228,6 +228,11 @@ public class FieldInfos implements Itera
nameToNumber.clear();
docValuesType.clear();
}
+
+ synchronized void setDocValuesType(int number, String name, DocValuesType dvType) {
+ assert containsConsistent(number, name, dvType);
+ docValuesType.put(name, dvType);
+ }
}
static final class Builder {
@@ -287,7 +292,14 @@ public class FieldInfos implements Itera
fi.update(isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions);
if (docValues != null) {
- fi.setDocValuesType(docValues);
+ // only pay the synchronization cost if fi does not already have a DVType
+ boolean updateGlobal = !fi.hasDocValues();
+ fi.setDocValuesType(docValues); // this will also perform the consistency check.
+ if (updateGlobal) {
+ // must also update docValuesType map so it's
+ // aware of this field's DocValueType
+ globalFieldNumbers.setDocValuesType(fi.number, name, docValues);
+ }
}
if (!fi.omitsNorms() && normType != null) {
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java Fri Aug 30 15:06:42 2013
@@ -414,4 +414,10 @@ public class FilterAtomicReader extends
return in.getNormValues(field);
}
+ @Override
+ public Bits getDocsWithField(String field) throws IOException {
+ ensureOpen();
+ return in.getDocsWithField(field);
+ }
+
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java Fri Aug 30 15:06:42 2013
@@ -68,12 +68,11 @@ class FlushByRamOrCountsPolicy extends F
control.setApplyAllDeletes();
}
}
- final DocumentsWriter writer = this.writer.get();
if ((flushOnRAM() &&
control.getDeleteBytesUsed() > (1024*1024*indexWriterConfig.getRAMBufferSizeMB()))) {
control.setApplyAllDeletes();
- if (writer.infoStream.isEnabled("FP")) {
- writer.infoStream.message("FP", "force apply deletes bytesUsed=" + control.getDeleteBytesUsed() + " vs ramBuffer=" + (1024*1024*indexWriterConfig.getRAMBufferSizeMB()));
+ if (infoStream.isEnabled("FP")) {
+ infoStream.message("FP", "force apply deletes bytesUsed=" + control.getDeleteBytesUsed() + " vs ramBuffer=" + (1024*1024*indexWriterConfig.getRAMBufferSizeMB()));
}
}
}
@@ -89,9 +88,8 @@ class FlushByRamOrCountsPolicy extends F
final long limit = (long) (indexWriterConfig.getRAMBufferSizeMB() * 1024.d * 1024.d);
final long totalRam = control.activeBytes() + control.getDeleteBytesUsed();
if (totalRam >= limit) {
- final DocumentsWriter writer = this.writer.get();
- if (writer.infoStream.isEnabled("FP")) {
- writer.infoStream.message("FP", "flush: activeBytes=" + control.activeBytes() + " deleteBytes=" + control.getDeleteBytesUsed() + " vs limit=" + limit);
+ if (infoStream.isEnabled("FP")) {
+ infoStream.message("FP", "flush: activeBytes=" + control.activeBytes() + " deleteBytes=" + control.getDeleteBytesUsed() + " vs limit=" + limit);
}
markLargestWriterPending(control, state, totalRam);
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FlushPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FlushPolicy.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FlushPolicy.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/FlushPolicy.java Fri Aug 30 15:06:42 2013
@@ -20,6 +20,7 @@ import java.util.Iterator;
import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.SetOnce;
/**
@@ -52,8 +53,8 @@ import org.apache.lucene.util.SetOnce;
* @see IndexWriterConfig#setFlushPolicy(FlushPolicy)
*/
abstract class FlushPolicy implements Cloneable {
- protected SetOnce<DocumentsWriter> writer = new SetOnce<DocumentsWriter>();
protected LiveIndexWriterConfig indexWriterConfig;
+ protected InfoStream infoStream;
/**
* Called for each delete term. If this is a delete triggered due to an update
@@ -93,9 +94,9 @@ abstract class FlushPolicy implements Cl
/**
* Called by DocumentsWriter to initialize the FlushPolicy
*/
- protected synchronized void init(DocumentsWriter docsWriter) {
- writer.set(docsWriter);
- indexWriterConfig = docsWriter.indexWriter.getConfig();
+ protected synchronized void init(LiveIndexWriterConfig indexWriterConfig) {
+ this.indexWriterConfig = indexWriterConfig;
+ infoStream = indexWriterConfig.getInfoStream();
}
/**
@@ -127,8 +128,8 @@ abstract class FlushPolicy implements Cl
}
private boolean assertMessage(String s) {
- if (writer.get().infoStream.isEnabled("FP")) {
- writer.get().infoStream.message("FP", s);
+ if (infoStream.isEnabled("FP")) {
+ infoStream.message("FP", s);
}
return true;
}
@@ -142,8 +143,8 @@ abstract class FlushPolicy implements Cl
// should not happen
throw new RuntimeException(e);
}
- clone.writer = new SetOnce<DocumentsWriter>();
clone.indexWriterConfig = null;
+ clone.infoStream = null;
return clone;
}
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Fri Aug 30 15:06:42 2013
@@ -30,6 +30,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Queue;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
@@ -182,7 +183,7 @@ import org.apache.lucene.util.ThreadInte
* referenced by the "front" of the index). For this, IndexFileDeleter
* keeps track of the last non commit checkpoint.
*/
-public class IndexWriter implements Closeable, TwoPhaseCommit {
+public class IndexWriter implements Closeable, TwoPhaseCommit{
private static final int UNBOUNDED_MAX_MERGE_SEGMENTS = -1;
@@ -227,6 +228,7 @@ public class IndexWriter implements Clos
final FieldNumbers globalFieldNumberMap;
private DocumentsWriter docWriter;
+ private final Queue<Event> eventQueue;
final IndexFileDeleter deleter;
// used by forceMerge to note those needing merging
@@ -360,7 +362,7 @@ public class IndexWriter implements Clos
synchronized (fullFlushLock) {
boolean success = false;
try {
- anySegmentFlushed = docWriter.flushAllThreads();
+ anySegmentFlushed = docWriter.flushAllThreads(this);
if (!anySegmentFlushed) {
// prevent double increment since docWriter#doFlush increments the flushcount
// if we flushed anything.
@@ -730,7 +732,9 @@ public class IndexWriter implements Clos
// start with previous field numbers, but new FieldInfos
globalFieldNumberMap = getFieldNumberMap();
- docWriter = new DocumentsWriter(codec, config, directory, this, globalFieldNumberMap, bufferedDeletesStream);
+ config.getFlushPolicy().init(config);
+ docWriter = new DocumentsWriter(this, config, directory);
+ eventQueue = docWriter.eventQueue();
// Default deleter (for backwards compatibility) is
// KeepOnlyLastCommitDeleter:
@@ -961,7 +965,7 @@ public class IndexWriter implements Clos
if (doFlush) {
flush(waitForMerges, true);
} else {
- docWriter.abort(); // already closed -- never sync on IW
+ docWriter.abort(this); // already closed -- never sync on IW
}
} finally {
@@ -1033,7 +1037,7 @@ public class IndexWriter implements Clos
synchronized(this) {
closed = true;
}
- assert oldWriter.perThreadPool.numDeactivatedThreadStates() == oldWriter.perThreadPool.getMaxThreadStates();
+ assert oldWriter.perThreadPool.numDeactivatedThreadStates() == oldWriter.perThreadPool.getMaxThreadStates() : "" + oldWriter.perThreadPool.numDeactivatedThreadStates() + " " + oldWriter.perThreadPool.getMaxThreadStates();
} catch (OutOfMemoryError oom) {
handleOOM(oom, "closeInternal");
} finally {
@@ -1280,9 +1284,10 @@ public class IndexWriter implements Clos
ensureOpen();
try {
boolean success = false;
- boolean anySegmentFlushed = false;
try {
- anySegmentFlushed = docWriter.updateDocuments(docs, analyzer, delTerm);
+ if (docWriter.updateDocuments(docs, analyzer, delTerm)) {
+ processEvents(true, false);
+ }
success = true;
} finally {
if (!success) {
@@ -1291,9 +1296,6 @@ public class IndexWriter implements Clos
}
}
}
- if (anySegmentFlushed) {
- maybeMerge(MergeTrigger.SEGMENT_FLUSH, UNBOUNDED_MAX_MERGE_SEGMENTS);
- }
} catch (OutOfMemoryError oom) {
handleOOM(oom, "updateDocuments");
}
@@ -1313,7 +1315,9 @@ public class IndexWriter implements Clos
public void deleteDocuments(Term term) throws IOException {
ensureOpen();
try {
- docWriter.deleteTerms(term);
+ if (docWriter.deleteTerms(term)) {
+ processEvents(true, false);
+ }
} catch (OutOfMemoryError oom) {
handleOOM(oom, "deleteDocuments(Term)");
}
@@ -1412,7 +1416,9 @@ public class IndexWriter implements Clos
public void deleteDocuments(Term... terms) throws IOException {
ensureOpen();
try {
- docWriter.deleteTerms(terms);
+ if (docWriter.deleteTerms(terms)) {
+ processEvents(true, false);
+ }
} catch (OutOfMemoryError oom) {
handleOOM(oom, "deleteDocuments(Term..)");
}
@@ -1432,7 +1438,9 @@ public class IndexWriter implements Clos
public void deleteDocuments(Query query) throws IOException {
ensureOpen();
try {
- docWriter.deleteQueries(query);
+ if (docWriter.deleteQueries(query)) {
+ processEvents(true, false);
+ }
} catch (OutOfMemoryError oom) {
handleOOM(oom, "deleteDocuments(Query)");
}
@@ -1454,7 +1462,9 @@ public class IndexWriter implements Clos
public void deleteDocuments(Query... queries) throws IOException {
ensureOpen();
try {
- docWriter.deleteQueries(queries);
+ if (docWriter.deleteQueries(queries)) {
+ processEvents(true, false);
+ }
} catch (OutOfMemoryError oom) {
handleOOM(oom, "deleteDocuments(Query..)");
}
@@ -1505,9 +1515,10 @@ public class IndexWriter implements Clos
ensureOpen();
try {
boolean success = false;
- boolean anySegmentFlushed = false;
try {
- anySegmentFlushed = docWriter.updateDocument(doc, analyzer, term);
+ if (docWriter.updateDocument(doc, analyzer, term)) {
+ processEvents(true, false);
+ }
success = true;
} finally {
if (!success) {
@@ -1516,10 +1527,6 @@ public class IndexWriter implements Clos
}
}
}
-
- if (anySegmentFlushed) {
- maybeMerge(MergeTrigger.SEGMENT_FLUSH, UNBOUNDED_MAX_MERGE_SEGMENTS);
- }
} catch (OutOfMemoryError oom) {
handleOOM(oom, "updateDocument");
}
@@ -1730,7 +1737,6 @@ public class IndexWriter implements Clos
// complete
ensureOpen();
}
-
// NOTE: in the ConcurrentMergeScheduler case, when
// doWait is false, we can return immediately while
// background threads accomplish the merging
@@ -2009,8 +2015,9 @@ public class IndexWriter implements Clos
mergeScheduler.close();
bufferedDeletesStream.clear();
+ processEvents(false, true);
docWriter.close(); // mark it as closed first to prevent subsequent indexing actions/flushes
- docWriter.abort(); // don't sync on IW here
+ docWriter.abort(this); // don't sync on IW here
synchronized(this) {
if (pendingCommit != null) {
@@ -2102,7 +2109,8 @@ public class IndexWriter implements Clos
* sure it's just like a fresh index.
*/
try {
- docWriter.lockAndAbortAll();
+ docWriter.lockAndAbortAll(this);
+ processEvents(false, true);
synchronized (this) {
try {
// Abort any running merges
@@ -2135,7 +2143,7 @@ public class IndexWriter implements Clos
}
}
} finally {
- docWriter.unlockAllAfterAbortAll();
+ docWriter.unlockAllAfterAbortAll(this);
}
}
}
@@ -2243,33 +2251,40 @@ public class IndexWriter implements Clos
* Atomically adds the segment private delete packet and publishes the flushed
* segments SegmentInfo to the index writer.
*/
- synchronized void publishFlushedSegment(SegmentInfoPerCommit newSegment,
+ void publishFlushedSegment(SegmentInfoPerCommit newSegment,
FrozenBufferedDeletes packet, FrozenBufferedDeletes globalPacket) throws IOException {
- // Lock order IW -> BDS
- synchronized (bufferedDeletesStream) {
- if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "publishFlushedSegment");
- }
-
- if (globalPacket != null && globalPacket.any()) {
- bufferedDeletesStream.push(globalPacket);
- }
- // Publishing the segment must be synched on IW -> BDS to make the sure
- // that no merge prunes away the seg. private delete packet
- final long nextGen;
- if (packet != null && packet.any()) {
- nextGen = bufferedDeletesStream.push(packet);
- } else {
- // Since we don't have a delete packet to apply we can get a new
- // generation right away
- nextGen = bufferedDeletesStream.getNextGen();
- }
- if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "publish sets newSegment delGen=" + nextGen + " seg=" + segString(newSegment));
+ try {
+ synchronized (this) {
+ // Lock order IW -> BDS
+ synchronized (bufferedDeletesStream) {
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "publishFlushedSegment");
+ }
+
+ if (globalPacket != null && globalPacket.any()) {
+ bufferedDeletesStream.push(globalPacket);
+ }
+ // Publishing the segment must be synched on IW -> BDS to make the sure
+ // that no merge prunes away the seg. private delete packet
+ final long nextGen;
+ if (packet != null && packet.any()) {
+ nextGen = bufferedDeletesStream.push(packet);
+ } else {
+ // Since we don't have a delete packet to apply we can get a new
+ // generation right away
+ nextGen = bufferedDeletesStream.getNextGen();
+ }
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "publish sets newSegment delGen=" + nextGen + " seg=" + segString(newSegment));
+ }
+ newSegment.setBufferedDeletesGen(nextGen);
+ segmentInfos.add(newSegment);
+ checkpoint();
+ }
}
- newSegment.setBufferedDeletesGen(nextGen);
- segmentInfos.add(newSegment);
- checkpoint();
+ } finally {
+ flushCount.incrementAndGet();
+ doAfterFlush();
}
}
@@ -2462,20 +2477,12 @@ public class IndexWriter implements Clos
String mergedName = newSegmentName();
final List<AtomicReader> mergeReaders = new ArrayList<AtomicReader>();
for (IndexReader indexReader : readers) {
- if (indexReader.numDocs() > 0) {
- numDocs += indexReader.numDocs();
- for (AtomicReaderContext ctx : indexReader.leaves()) {
- if (ctx.reader().numDocs() > 0) { // drop empty (or all deleted) segments
- mergeReaders.add(ctx.reader());
- }
- }
+ numDocs += indexReader.numDocs();
+ for (AtomicReaderContext ctx : indexReader.leaves()) {
+ mergeReaders.add(ctx.reader());
}
}
- if (mergeReaders.isEmpty()) { // no segments with documents to add
- return;
- }
-
final IOContext context = new IOContext(new MergeInfo(numDocs, -1, true, -1));
// TODO: somehow we should fix this merge so it's
@@ -2487,6 +2494,10 @@ public class IndexWriter implements Clos
SegmentMerger merger = new SegmentMerger(mergeReaders, info, infoStream, trackingDir,
MergeState.CheckAbort.NONE, globalFieldNumberMap, context);
+
+ if (!merger.shouldMerge()) {
+ return;
+ }
MergeState mergeState;
boolean success = false;
@@ -2709,12 +2720,13 @@ public class IndexWriter implements Clos
boolean flushSuccess = false;
boolean success = false;
try {
- anySegmentsFlushed = docWriter.flushAllThreads();
+ anySegmentsFlushed = docWriter.flushAllThreads(this);
if (!anySegmentsFlushed) {
// prevent double increment since docWriter#doFlush increments the flushcount
// if we flushed anything.
flushCount.incrementAndGet();
}
+ processEvents(false, true);
flushSuccess = true;
synchronized(this) {
@@ -2754,7 +2766,7 @@ public class IndexWriter implements Clos
} catch (OutOfMemoryError oom) {
handleOOM(oom, "prepareCommit");
}
-
+
boolean success = false;
try {
if (anySegmentsFlushed) {
@@ -2769,7 +2781,7 @@ public class IndexWriter implements Clos
}
}
}
-
+
startCommit(toCommit);
}
}
@@ -2954,10 +2966,11 @@ public class IndexWriter implements Clos
synchronized (fullFlushLock) {
boolean flushSuccess = false;
try {
- anySegmentFlushed = docWriter.flushAllThreads();
+ anySegmentFlushed = docWriter.flushAllThreads(this);
flushSuccess = true;
} finally {
docWriter.finishFullFlush(flushSuccess);
+ processEvents(false, true);
}
}
synchronized(this) {
@@ -3733,7 +3746,12 @@ public class IndexWriter implements Clos
MergeState mergeState;
boolean success3 = false;
try {
- mergeState = merger.merge();
+ if (!merger.shouldMerge()) {
+ // would result in a 0 document segment: nothing to merge!
+ mergeState = new MergeState(new ArrayList<AtomicReader>(), merge.info.info, infoStream, checkAbort);
+ } else {
+ mergeState = merger.merge();
+ }
success3 = true;
} finally {
if (!success3) {
@@ -3748,12 +3766,16 @@ public class IndexWriter implements Clos
// Record which codec was used to write the segment
if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "merge codec=" + codec + " docCount=" + merge.info.info.getDocCount() + "; merged segment has " +
+ if (merge.info.info.getDocCount() == 0) {
+ infoStream.message("IW", "merge away fully deleted segments");
+ } else {
+ infoStream.message("IW", "merge codec=" + codec + " docCount=" + merge.info.info.getDocCount() + "; merged segment has " +
(mergeState.fieldInfos.hasVectors() ? "vectors" : "no vectors") + "; " +
(mergeState.fieldInfos.hasNorms() ? "norms" : "no norms") + "; " +
(mergeState.fieldInfos.hasDocValues() ? "docValues" : "no docValues") + "; " +
(mergeState.fieldInfos.hasProx() ? "prox" : "no prox") + "; " +
(mergeState.fieldInfos.hasProx() ? "freqs" : "no freqs"));
+ }
}
// Very important to do this before opening the reader
@@ -3958,8 +3980,8 @@ public class IndexWriter implements Clos
/** Only for testing.
*
* @lucene.internal */
- void keepFullyDeletedSegments() {
- keepFullyDeletedSegments = true;
+ void setKeepFullyDeletedSegments(boolean v) {
+ keepFullyDeletedSegments = v;
}
boolean getKeepFullyDeletedSegments() {
@@ -4302,4 +4324,65 @@ public class IndexWriter implements Clos
synchronized final void flushFailed(SegmentInfo info) throws IOException {
deleter.refresh(info.name);
}
+
+ final int purge(boolean forced) throws IOException {
+ return docWriter.purgeBuffer(this, forced);
+ }
+
+ final void applyDeletesAndPurge(boolean forcePurge) throws IOException {
+ try {
+ purge(forcePurge);
+ } finally {
+ applyAllDeletes();
+ flushCount.incrementAndGet();
+ }
+ }
+ final void doAfterSegmentFlushed(boolean triggerMerge, boolean forcePurge) throws IOException {
+ try {
+ purge(forcePurge);
+ } finally {
+ if (triggerMerge) {
+ maybeMerge(MergeTrigger.SEGMENT_FLUSH, UNBOUNDED_MAX_MERGE_SEGMENTS);
+ }
+ }
+
+ }
+
+ private boolean processEvents(boolean triggerMerge, boolean forcePurge) throws IOException {
+ return processEvents(eventQueue, triggerMerge, forcePurge);
+ }
+
+ private boolean processEvents(Queue<Event> queue, boolean triggerMerge, boolean forcePurge) throws IOException {
+ Event event;
+ boolean processed = false;
+ while((event = queue.poll()) != null) {
+ processed = true;
+ event.process(this, triggerMerge, forcePurge);
+ }
+ return processed;
+ }
+
+ /**
+ * Interface for internal atomic events. See {@link DocumentsWriter} for details. Events are executed concurrently and no order is guaranteed.
+ * Each event should only rely on the serializeability within it's process method. All actions that must happen before or after a certain action must be
+ * encoded inside the {@link #process(IndexWriter, boolean, boolean)} method.
+ *
+ */
+ static interface Event {
+
+ /**
+ * Processes the event. This method is called by the {@link IndexWriter}
+ * passed as the first argument.
+ *
+ * @param writer
+ * the {@link IndexWriter} that executes the event.
+ * @param triggerMerge
+ * <code>false</code> iff this event should not trigger any segment merges
+ * @param clearBuffers
+ * <code>true</code> iff this event should clear all buffers associated with the event.
+ * @throws IOException
+ * if an {@link IOException} occurs
+ */
+ void process(IndexWriter writer, boolean triggerMerge, boolean clearBuffers) throws IOException;
+ }
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java Fri Aug 30 15:06:42 2013
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.lucene.index.MultiTermsEnum.TermsEnumIndex;
import org.apache.lucene.index.MultiTermsEnum.TermsEnumWithSlice;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.packed.AppendingPackedLongBuffer;
import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
@@ -135,6 +136,51 @@ public class MultiDocValues {
};
}
}
+
+ /** Returns a Bits for a reader's docsWithField (potentially merging on-the-fly)
+ * <p>
+ * This is a slow way to access this bitset. Instead, access them per-segment
+ * with {@link AtomicReader#getDocsWithField(String)}
+ * </p>
+ * */
+ public static Bits getDocsWithField(final IndexReader r, final String field) throws IOException {
+ final List<AtomicReaderContext> leaves = r.leaves();
+ final int size = leaves.size();
+ if (size == 0) {
+ return null;
+ } else if (size == 1) {
+ return leaves.get(0).reader().getDocsWithField(field);
+ }
+
+ boolean anyReal = false;
+ boolean anyMissing = false;
+ final Bits[] values = new Bits[size];
+ final int[] starts = new int[size+1];
+ for (int i = 0; i < size; i++) {
+ AtomicReaderContext context = leaves.get(i);
+ Bits v = context.reader().getDocsWithField(field);
+ if (v == null) {
+ v = new Bits.MatchNoBits(context.reader().maxDoc());
+ anyMissing = true;
+ } else {
+ anyReal = true;
+ if (v instanceof Bits.MatchAllBits == false) {
+ anyMissing = true;
+ }
+ }
+ values[i] = v;
+ starts[i] = context.docBase;
+ }
+ starts[size] = r.maxDoc();
+
+ if (!anyReal) {
+ return null;
+ } else if (!anyMissing) {
+ return new Bits.MatchAllBits(r.maxDoc());
+ } else {
+ return new MultiBits(values, starts, false);
+ }
+ }
/** Returns a BinaryDocValues for a reader's docvalues (potentially merging on-the-fly)
* <p>
@@ -404,7 +450,7 @@ public class MultiDocValues {
public int getOrd(int docID) {
int subIndex = ReaderUtil.subIndex(docID, docStarts);
int segmentOrd = values[subIndex].getOrd(docID - docStarts[subIndex]);
- return (int) mapping.getGlobalOrd(subIndex, segmentOrd);
+ return segmentOrd == -1 ? segmentOrd : (int) mapping.getGlobalOrd(subIndex, segmentOrd);
}
@Override
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/NormsConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/NormsConsumerPerField.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/NormsConsumerPerField.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/NormsConsumerPerField.java Fri Aug 30 15:06:42 2013
@@ -44,7 +44,7 @@ final class NormsConsumerPerField extend
if (fieldInfo.isIndexed() && !fieldInfo.omitsNorms()) {
if (consumer == null) {
fieldInfo.setNormValueType(FieldInfo.DocValuesType.NUMERIC);
- consumer = new NumericDocValuesWriter(fieldInfo, docState.docWriter.bytesUsed);
+ consumer = new NumericDocValuesWriter(fieldInfo, docState.docWriter.bytesUsed, false);
}
consumer.addValue(docState.docID, similarity.computeNorm(fieldState));
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java Fri Aug 30 15:06:42 2013
@@ -23,6 +23,8 @@ import java.util.NoSuchElementException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.util.Counter;
+import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.AppendingDeltaPackedLongBuffer;
import org.apache.lucene.util.packed.PackedInts;
@@ -35,14 +37,18 @@ class NumericDocValuesWriter extends Doc
private AppendingDeltaPackedLongBuffer pending;
private final Counter iwBytesUsed;
private long bytesUsed;
+ private final OpenBitSet docsWithField;
private final FieldInfo fieldInfo;
+ private final boolean trackDocsWithField;
- public NumericDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) {
+ public NumericDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed, boolean trackDocsWithField) {
pending = new AppendingDeltaPackedLongBuffer(PackedInts.COMPACT);
- bytesUsed = pending.ramBytesUsed();
+ docsWithField = new OpenBitSet();
+ bytesUsed = pending.ramBytesUsed() + docsWithFieldBytesUsed();
this.fieldInfo = fieldInfo;
this.iwBytesUsed = iwBytesUsed;
iwBytesUsed.addAndGet(bytesUsed);
+ this.trackDocsWithField = trackDocsWithField;
}
public void addValue(int docID, long value) {
@@ -56,12 +62,20 @@ class NumericDocValuesWriter extends Doc
}
pending.add(value);
+ if (trackDocsWithField) {
+ docsWithField.set(docID);
+ }
updateBytesUsed();
}
+
+ private long docsWithFieldBytesUsed() {
+ // size of the long[] + some overhead
+ return RamUsageEstimator.sizeOf(docsWithField.getBits()) + 64;
+ }
private void updateBytesUsed() {
- final long newBytesUsed = pending.ramBytesUsed();
+ final long newBytesUsed = pending.ramBytesUsed() + docsWithFieldBytesUsed();
iwBytesUsed.addAndGet(newBytesUsed - bytesUsed);
bytesUsed = newBytesUsed;
}
@@ -109,14 +123,18 @@ class NumericDocValuesWriter extends Doc
if (!hasNext()) {
throw new NoSuchElementException();
}
- long value;
+ Long value;
if (upto < size) {
- value = iter.next();
+ long v = iter.next();
+ if (!trackDocsWithField || docsWithField.get(upto)) {
+ value = v;
+ } else {
+ value = null;
+ }
} else {
- value = 0;
+ value = trackDocsWithField ? null : MISSING;
}
upto++;
- // TODO: make reusable Number
return value;
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java Fri Aug 30 15:06:42 2013
@@ -286,6 +286,13 @@ public class ParallelAtomicReader extend
}
@Override
+ public Bits getDocsWithField(String field) throws IOException {
+ ensureOpen();
+ AtomicReader reader = fieldToReader.get(field);
+ return reader == null ? null : reader.getDocsWithField(field);
+ }
+
+ @Override
public NumericDocValues getNormValues(String field) throws IOException {
ensureOpen();
AtomicReader reader = fieldToReader.get(field);
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java Fri Aug 30 15:06:42 2013
@@ -36,6 +36,7 @@ import org.apache.lucene.index.SegmentRe
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.CloseableThreadLocal;
import org.apache.lucene.util.IOUtils;
@@ -87,6 +88,13 @@ final class SegmentCoreReaders {
return new HashMap<String,Object>();
}
};
+
+ final CloseableThreadLocal<Map<String,Bits>> docsWithFieldLocal = new CloseableThreadLocal<Map<String,Bits>>() {
+ @Override
+ protected Map<String,Bits> initialValue() {
+ return new HashMap<String,Bits>();
+ }
+ };
final CloseableThreadLocal<Map<String,Object>> normsLocal = new CloseableThreadLocal<Map<String,Object>>() {
@Override
@@ -274,6 +282,30 @@ final class SegmentCoreReaders {
return dvs;
}
+
+ Bits getDocsWithField(String field) throws IOException {
+ FieldInfo fi = fieldInfos.fieldInfo(field);
+ if (fi == null) {
+ // Field does not exist
+ return null;
+ }
+ if (fi.getDocValuesType() == null) {
+ // Field was not indexed with doc values
+ return null;
+ }
+
+ assert dvProducer != null;
+
+ Map<String,Bits> dvFields = docsWithFieldLocal.get();
+
+ Bits dvs = dvFields.get(field);
+ if (dvs == null) {
+ dvs = dvProducer.getDocsWithField(fi);
+ dvFields.put(field, dvs);
+ }
+
+ return dvs;
+ }
NumericDocValues getNormValues(String field) throws IOException {
FieldInfo fi = fieldInfos.fieldInfo(field);
@@ -300,8 +332,8 @@ final class SegmentCoreReaders {
void decRef() throws IOException {
if (ref.decrementAndGet() == 0) {
- IOUtils.close(termVectorsLocal, fieldsReaderLocal, docValuesLocal, normsLocal, fields, dvProducer,
- termVectorsReaderOrig, fieldsReaderOrig, cfsReader, normsProducer);
+ IOUtils.close(termVectorsLocal, fieldsReaderLocal, docValuesLocal, normsLocal, docsWithFieldLocal, fields,
+ dvProducer, termVectorsReaderOrig, fieldsReaderOrig, cfsReader, normsProducer);
notifyCoreClosedListeners();
}
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java Fri Aug 30 15:06:42 2013
@@ -30,6 +30,7 @@ import org.apache.lucene.codecs.TermVect
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;
@@ -52,12 +53,18 @@ final class SegmentMerger {
// note, just like in codec apis Directory 'dir' is NOT the same as segmentInfo.dir!!
SegmentMerger(List<AtomicReader> readers, SegmentInfo segmentInfo, InfoStream infoStream, Directory dir,
- MergeState.CheckAbort checkAbort, FieldInfos.FieldNumbers fieldNumbers, IOContext context) {
+ MergeState.CheckAbort checkAbort, FieldInfos.FieldNumbers fieldNumbers, IOContext context) throws IOException {
mergeState = new MergeState(readers, segmentInfo, infoStream, checkAbort);
directory = dir;
this.codec = segmentInfo.getCodec();
this.context = context;
this.fieldInfosBuilder = new FieldInfos.Builder(fieldNumbers);
+ mergeState.segmentInfo.setDocCount(setDocMaps());
+ }
+
+ /** True if any merging should happen */
+ boolean shouldMerge() {
+ return mergeState.segmentInfo.getDocCount() > 0;
}
/**
@@ -67,14 +74,15 @@ final class SegmentMerger {
* @throws IOException if there is a low-level IO error
*/
MergeState merge() throws IOException {
+ if (!shouldMerge()) {
+ throw new IllegalStateException("Merge would result in 0 document segment");
+ }
// NOTE: it's important to add calls to
// checkAbort.work(...) if you make any changes to this
// method that will spend alot of time. The frequency
// of this check impacts how long
// IndexWriter.close(false) takes to actually stop the
// threads.
-
- mergeState.segmentInfo.setDocCount(setDocMaps());
mergeFieldInfos();
setMatchingSegmentReaders();
long t0 = 0;
@@ -149,24 +157,32 @@ final class SegmentMerger {
if (type != null) {
if (type == DocValuesType.NUMERIC) {
List<NumericDocValues> toMerge = new ArrayList<NumericDocValues>();
+ List<Bits> docsWithField = new ArrayList<Bits>();
for (AtomicReader reader : mergeState.readers) {
NumericDocValues values = reader.getNumericDocValues(field.name);
+ Bits bits = reader.getDocsWithField(field.name);
if (values == null) {
values = NumericDocValues.EMPTY;
+ bits = new Bits.MatchNoBits(reader.maxDoc());
}
toMerge.add(values);
+ docsWithField.add(bits);
}
- consumer.mergeNumericField(field, mergeState, toMerge);
+ consumer.mergeNumericField(field, mergeState, toMerge, docsWithField);
} else if (type == DocValuesType.BINARY) {
List<BinaryDocValues> toMerge = new ArrayList<BinaryDocValues>();
+ List<Bits> docsWithField = new ArrayList<Bits>();
for (AtomicReader reader : mergeState.readers) {
BinaryDocValues values = reader.getBinaryDocValues(field.name);
+ Bits bits = reader.getDocsWithField(field.name);
if (values == null) {
values = BinaryDocValues.EMPTY;
+ bits = new Bits.MatchNoBits(reader.maxDoc());
}
toMerge.add(values);
+ docsWithField.add(bits);
}
- consumer.mergeBinaryField(field, mergeState, toMerge);
+ consumer.mergeBinaryField(field, mergeState, toMerge, docsWithField);
} else if (type == DocValuesType.SORTED) {
List<SortedDocValues> toMerge = new ArrayList<SortedDocValues>();
for (AtomicReader reader : mergeState.readers) {
@@ -209,14 +225,16 @@ final class SegmentMerger {
for (FieldInfo field : mergeState.fieldInfos) {
if (field.hasNorms()) {
List<NumericDocValues> toMerge = new ArrayList<NumericDocValues>();
+ List<Bits> docsWithField = new ArrayList<Bits>();
for (AtomicReader reader : mergeState.readers) {
NumericDocValues norms = reader.getNormValues(field.name);
if (norms == null) {
norms = NumericDocValues.EMPTY;
}
toMerge.add(norms);
+ docsWithField.add(new Bits.MatchAllBits(reader.maxDoc()));
}
- consumer.mergeNumericField(field, mergeState, toMerge);
+ consumer.mergeNumericField(field, mergeState, toMerge, docsWithField);
}
}
success = true;
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Fri Aug 30 15:06:42 2013
@@ -224,6 +224,12 @@ public final class SegmentReader extends
}
@Override
+ public Bits getDocsWithField(String field) throws IOException {
+ ensureOpen();
+ return core.getDocsWithField(field);
+ }
+
+ @Override
public BinaryDocValues getBinaryDocValues(String field) throws IOException {
ensureOpen();
return core.getBinaryDocValues(field);
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java Fri Aug 30 15:06:42 2013
@@ -64,9 +64,7 @@ public final class SlowCompositeReaderWr
}
}
- /** Sole constructor, wrapping the provided {@link
- * CompositeReader}. */
- public SlowCompositeReaderWrapper(CompositeReader reader) throws IOException {
+ private SlowCompositeReaderWrapper(CompositeReader reader) throws IOException {
super();
in = reader;
fields = MultiFields.getFields(in);
@@ -92,6 +90,12 @@ public final class SlowCompositeReaderWr
}
@Override
+ public Bits getDocsWithField(String field) throws IOException {
+ ensureOpen();
+ return MultiDocValues.getDocsWithField(in, field);
+ }
+
+ @Override
public BinaryDocValues getBinaryDocValues(String field) throws IOException {
ensureOpen();
return MultiDocValues.getBinaryValues(in, field);
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java Fri Aug 30 15:06:42 2013
@@ -37,12 +37,13 @@ public abstract class SortedDocValues ex
* Returns the ordinal for the specified docID.
* @param docID document ID to lookup
* @return ordinal for the document: this is dense, starts at 0, then
- * increments by 1 for the next value in sorted order.
+ * increments by 1 for the next value in sorted order. Note that
+ * missing values are indicated by -1.
*/
public abstract int getOrd(int docID);
/** Retrieves the value for the specified ordinal.
- * @param ord ordinal to lookup
+ * @param ord ordinal to lookup (must be >= 0 and < {@link #getValueCount()})
* @param result will be populated with the ordinal's value
* @see #getOrd(int)
*/
@@ -71,7 +72,7 @@ public abstract class SortedDocValues ex
public static final SortedDocValues EMPTY = new SortedDocValues() {
@Override
public int getOrd(int docID) {
- return 0;
+ return -1;
}
@Override
@@ -83,7 +84,7 @@ public abstract class SortedDocValues ex
@Override
public int getValueCount() {
- return 1;
+ return 0;
}
};
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java Fri Aug 30 15:06:42 2013
@@ -30,19 +30,19 @@ import org.apache.lucene.util.BytesRefHa
import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.RamUsageEstimator;
-import org.apache.lucene.util.packed.AppendingPackedLongBuffer;
+import org.apache.lucene.util.packed.AppendingDeltaPackedLongBuffer;
import org.apache.lucene.util.packed.PackedInts;
/** Buffers up pending byte[] per doc, deref and sorting via
* int ord, then flushes when segment flushes. */
class SortedDocValuesWriter extends DocValuesWriter {
final BytesRefHash hash;
- private AppendingPackedLongBuffer pending;
+ private AppendingDeltaPackedLongBuffer pending;
private final Counter iwBytesUsed;
private long bytesUsed; // this currently only tracks differences in 'pending'
private final FieldInfo fieldInfo;
- private static final BytesRef EMPTY = new BytesRef(BytesRef.EMPTY_BYTES);
+ private static final int EMPTY_ORD = -1;
public SortedDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) {
this.fieldInfo = fieldInfo;
@@ -52,7 +52,7 @@ class SortedDocValuesWriter extends DocV
new ByteBlockPool.DirectTrackingAllocator(iwBytesUsed)),
BytesRefHash.DEFAULT_CAPACITY,
new DirectBytesStartArray(BytesRefHash.DEFAULT_CAPACITY, iwBytesUsed));
- pending = new AppendingPackedLongBuffer(PackedInts.COMPACT);
+ pending = new AppendingDeltaPackedLongBuffer(PackedInts.COMPACT);
bytesUsed = pending.ramBytesUsed();
iwBytesUsed.addAndGet(bytesUsed);
}
@@ -70,7 +70,7 @@ class SortedDocValuesWriter extends DocV
// Fill in any holes:
while(pending.size() < docID) {
- addOneValue(EMPTY);
+ pending.add(EMPTY_ORD);
}
addOneValue(value);
@@ -79,8 +79,9 @@ class SortedDocValuesWriter extends DocV
@Override
public void finish(int maxDoc) {
while(pending.size() < maxDoc) {
- addOneValue(EMPTY);
+ pending.add(EMPTY_ORD);
}
+ updateBytesUsed();
}
private void addOneValue(BytesRef value) {
@@ -177,7 +178,7 @@ class SortedDocValuesWriter extends DocV
// iterates over the ords for each doc we have in ram
private class OrdsIterator implements Iterator<Number> {
- final AppendingPackedLongBuffer.Iterator iter = pending.iterator();
+ final AppendingDeltaPackedLongBuffer.Iterator iter = pending.iterator();
final int ordMap[];
final int maxDoc;
int docUpto;
@@ -200,8 +201,7 @@ class SortedDocValuesWriter extends DocV
}
int ord = (int) iter.next();
docUpto++;
- // TODO: make reusable Number
- return ordMap[ord];
+ return ord == -1 ? ord : ordMap[ord];
}
@Override
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java Fri Aug 30 15:06:42 2013
@@ -49,6 +49,14 @@ public class CachingWrapperFilter extend
this.filter = filter;
}
+ /**
+ * Gets the contained filter.
+ * @return the contained filter.
+ */
+ public Filter getFilter() {
+ return filter;
+ }
+
/**
* Provide the DocIdSet to be cached, using the DocIdSet provided
* by the wrapped Filter. <p>This implementation returns the given {@link DocIdSet},
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/FieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/FieldCache.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/FieldCache.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/FieldCache.java Fri Aug 30 15:06:42 2013
@@ -104,26 +104,6 @@ public interface FieldCache {
}
};
}
-
- /** Returns MISSING/-1 ordinal for every document */
- public static final SortedDocValues EMPTY_TERMSINDEX = new SortedDocValues() {
- @Override
- public int getOrd(int docID) {
- return -1;
- }
-
- @Override
- public void lookupOrd(int ord, BytesRef result) {
- result.bytes = MISSING;
- result.offset = 0;
- result.length = 0;
- }
-
- @Override
- public int getValueCount() {
- return 0;
- }
- };
/**
* Placeholder indicating creation of this cache is currently in-progress.
@@ -266,13 +246,10 @@ public interface FieldCache {
}
};
-
/** Checks the internal cache for an appropriate entry, and if none is found,
* reads the terms in <code>field</code> and returns a bit set at the size of
* <code>reader.maxDoc()</code>, with turned on bits for each docid that
- * does have a value for this field. Note that if the field was only indexed
- * as DocValues then this method will not work (it will return a Bits stating
- * that no documents contain the field).
+ * does have a value for this field.
*/
public Bits getDocsWithField(AtomicReader reader, String field) throws IOException;
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java Fri Aug 30 15:06:42 2013
@@ -501,8 +501,7 @@ class FieldCacheImpl implements FieldCac
// field does not exist or has no value
return new Bits.MatchNoBits(reader.maxDoc());
} else if (fieldInfo.hasDocValues()) {
- // doc values are dense
- return new Bits.MatchAllBits(reader.maxDoc());
+ return reader.getDocsWithField(field);
} else if (!fieldInfo.isIndexed()) {
return new Bits.MatchNoBits(reader.maxDoc());
}
@@ -944,13 +943,13 @@ class FieldCacheImpl implements FieldCac
} else {
final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
if (info == null) {
- return EMPTY_TERMSINDEX;
+ return SortedDocValues.EMPTY;
} else if (info.hasDocValues()) {
// we don't try to build a sorted instance from numeric/binary doc
// values because dedup can be very costly
throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
} else if (!info.isIndexed()) {
- return EMPTY_TERMSINDEX;
+ return SortedDocValues.EMPTY;
}
return (SortedDocValues) caches.get(SortedDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio), false);
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java Fri Aug 30 15:06:42 2013
@@ -259,7 +259,7 @@ public class AttributeSource {
* already in this AttributeSource and returns it. Otherwise a
* new instance is created, added to this AttributeSource and returned.
*/
- public final <A extends Attribute> A addAttribute(Class<A> attClass) {
+ public final <T extends Attribute> T addAttribute(Class<T> attClass) {
AttributeImpl attImpl = attributes.get(attClass);
if (attImpl == null) {
if (!(attClass.isInterface() && Attribute.class.isAssignableFrom(attClass))) {
@@ -297,7 +297,7 @@ public class AttributeSource {
* available. If you want to only use the attribute, if it is available (to optimize
* consuming), use {@link #hasAttribute}.
*/
- public final <A extends Attribute> A getAttribute(Class<A> attClass) {
+ public final <T extends Attribute> T getAttribute(Class<T> attClass) {
AttributeImpl attImpl = attributes.get(attClass);
if (attImpl == null) {
throw new IllegalArgumentException("This AttributeSource does not have the attribute '" + attClass.getName() + "'.");
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java Fri Aug 30 15:06:42 2013
@@ -21,6 +21,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexInput;
/** Represents a logical byte[] as a series of pages. You
@@ -34,6 +36,7 @@ import org.apache.lucene.store.IndexInpu
// other "shift/mask big arrays". there are too many of these classes!
public final class PagedBytes {
private final List<byte[]> blocks = new ArrayList<byte[]>();
+ // TODO: these are unused?
private final List<Integer> blockEnd = new ArrayList<Integer>();
private final int blockSize;
private final int blockBits;
@@ -42,6 +45,7 @@ public final class PagedBytes {
private boolean frozen;
private int upto;
private byte[] currentBlock;
+ private final long bytesUsedPerBlock;
private static final byte[] EMPTY_BYTES = new byte[0];
@@ -75,13 +79,13 @@ public final class PagedBytes {
* given length. Iff the slice spans across a block border this method will
* allocate sufficient resources and copy the paged data.
* <p>
- * Slices spanning more than one block are not supported.
+ * Slices spanning more than two blocks are not supported.
* </p>
* @lucene.internal
**/
public void fillSlice(BytesRef b, long start, int length) {
assert length >= 0: "length=" + length;
- assert length <= blockSize+1;
+ assert length <= blockSize+1: "length=" + length;
final int index = (int) (start >> blockBits);
final int offset = (int) (start & blockMask);
b.length = length;
@@ -132,6 +136,7 @@ public final class PagedBytes {
this.blockBits = blockBits;
blockMask = blockSize-1;
upto = blockSize;
+ bytesUsedPerBlock = blockSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + RamUsageEstimator.NUM_BYTES_OBJECT_REF;
}
/** Read this many bytes from in */
@@ -216,6 +221,11 @@ public final class PagedBytes {
}
}
+ /** Return approx RAM usage in bytes. */
+ public long ramBytesUsed() {
+ return (blocks.size() + (currentBlock != null ? 1 : 0)) * bytesUsedPerBlock;
+ }
+
/** Copy bytes in, writing the length as a 1 or 2 byte
* vInt prefix. */
// TODO: this really needs to be refactored into fieldcacheimpl!
@@ -249,4 +259,148 @@ public final class PagedBytes {
return pointer;
}
+
+ public final class PagedBytesDataInput extends DataInput {
+ private int currentBlockIndex;
+ private int currentBlockUpto;
+ private byte[] currentBlock;
+
+ PagedBytesDataInput() {
+ currentBlock = blocks.get(0);
+ }
+
+ @Override
+ public PagedBytesDataInput clone() {
+ PagedBytesDataInput clone = getDataInput();
+ clone.setPosition(getPosition());
+ return clone;
+ }
+
+ /** Returns the current byte position. */
+ public long getPosition() {
+ return (long) currentBlockIndex * blockSize + currentBlockUpto;
+ }
+
+ /** Seek to a position previously obtained from
+ * {@link #getPosition}. */
+ public void setPosition(long pos) {
+ currentBlockIndex = (int) (pos >> blockBits);
+ currentBlock = blocks.get(currentBlockIndex);
+ currentBlockUpto = (int) (pos & blockMask);
+ }
+
+ @Override
+ public byte readByte() {
+ if (currentBlockUpto == blockSize) {
+ nextBlock();
+ }
+ return currentBlock[currentBlockUpto++];
+ }
+
+ @Override
+ public void readBytes(byte[] b, int offset, int len) {
+ assert b.length >= offset + len;
+ final int offsetEnd = offset + len;
+ while (true) {
+ final int blockLeft = blockSize - currentBlockUpto;
+ final int left = offsetEnd - offset;
+ if (blockLeft < left) {
+ System.arraycopy(currentBlock, currentBlockUpto,
+ b, offset,
+ blockLeft);
+ nextBlock();
+ offset += blockLeft;
+ } else {
+ // Last block
+ System.arraycopy(currentBlock, currentBlockUpto,
+ b, offset,
+ left);
+ currentBlockUpto += left;
+ break;
+ }
+ }
+ }
+
+ private void nextBlock() {
+ currentBlockIndex++;
+ currentBlockUpto = 0;
+ currentBlock = blocks.get(currentBlockIndex);
+ }
+ }
+
+ public final class PagedBytesDataOutput extends DataOutput {
+ @Override
+ public void writeByte(byte b) {
+ if (upto == blockSize) {
+ if (currentBlock != null) {
+ blocks.add(currentBlock);
+ blockEnd.add(upto);
+ }
+ currentBlock = new byte[blockSize];
+ upto = 0;
+ }
+ currentBlock[upto++] = b;
+ }
+
+ @Override
+ public void writeBytes(byte[] b, int offset, int length) {
+ assert b.length >= offset + length;
+ if (length == 0) {
+ return;
+ }
+
+ if (upto == blockSize) {
+ if (currentBlock != null) {
+ blocks.add(currentBlock);
+ blockEnd.add(upto);
+ }
+ currentBlock = new byte[blockSize];
+ upto = 0;
+ }
+
+ final int offsetEnd = offset + length;
+ while(true) {
+ final int left = offsetEnd - offset;
+ final int blockLeft = blockSize - upto;
+ if (blockLeft < left) {
+ System.arraycopy(b, offset, currentBlock, upto, blockLeft);
+ blocks.add(currentBlock);
+ blockEnd.add(blockSize);
+ currentBlock = new byte[blockSize];
+ upto = 0;
+ offset += blockLeft;
+ } else {
+ // Last block
+ System.arraycopy(b, offset, currentBlock, upto, left);
+ upto += left;
+ break;
+ }
+ }
+ }
+
+ /** Return the current byte position. */
+ public long getPosition() {
+ return getPointer();
+ }
+ }
+
+ /** Returns a DataInput to read values from this
+ * PagedBytes instance. */
+ public PagedBytesDataInput getDataInput() {
+ if (!frozen) {
+ throw new IllegalStateException("must call freeze() before getDataInput");
+ }
+ return new PagedBytesDataInput();
+ }
+
+ /** Returns a DataOutput that you may use to write into
+ * this PagedBytes instance. If you do this, you should
+ * not call the other writing methods (eg, copy);
+ * results are undefined. */
+ public PagedBytesDataOutput getDataOutput() {
+ if (frozen) {
+ throw new IllegalStateException("cannot get DataOutput after freeze()");
+ }
+ return new PagedBytesDataOutput();
+ }
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/fst/FST.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/fst/FST.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/util/fst/FST.java Fri Aug 30 15:06:42 2013
@@ -1146,7 +1146,6 @@ public final class FST<T> {
/** Finds an arc leaving the incoming arc, replacing the arc in place.
* This returns null if the arc was not found, else the incoming arc. */
public Arc<T> findTargetArc(int labelToMatch, Arc<T> follow, Arc<T> arc, BytesReader in) throws IOException {
- assert assertRootArcs();
if (labelToMatch == END_LABEL) {
if (follow.isFinal()) {
@@ -1168,6 +1167,10 @@ public final class FST<T> {
// Short-circuit if this arc is in the root arc cache:
if (follow.target == startNode && labelToMatch < cachedRootArcs.length) {
+
+ // LUCENE-5152: detect tricky cases where caller
+ // modified previously returned cached root-arcs:
+ assert assertRootArcs();
final Arc<T> result = cachedRootArcs[labelToMatch];
if (result == null) {
return null;
Modified: lucene/dev/branches/lucene3069/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec Fri Aug 30 15:06:42 2013
@@ -16,3 +16,4 @@
org.apache.lucene.codecs.lucene40.Lucene40Codec
org.apache.lucene.codecs.lucene41.Lucene41Codec
org.apache.lucene.codecs.lucene42.Lucene42Codec
+org.apache.lucene.codecs.lucene45.Lucene45Codec
\ No newline at end of file
Modified: lucene/dev/branches/lucene3069/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat Fri Aug 30 15:06:42 2013
@@ -14,3 +14,4 @@
# limitations under the License.
org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat
+org.apache.lucene.codecs.lucene45.Lucene45DocValuesFormat
\ No newline at end of file
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java Fri Aug 30 15:06:42 2013
@@ -17,21 +17,27 @@ package org.apache.lucene;
* limitations under the License.
*/
-import org.apache.lucene.analysis.*;
-import org.apache.lucene.codecs.*;
-import org.apache.lucene.codecs.lucene42.Lucene42Codec;
-import org.apache.lucene.document.*;
-import org.apache.lucene.index.*;
-import org.apache.lucene.search.*;
-import org.apache.lucene.store.*;
-import org.apache.lucene.util.*;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.lucene45.Lucene45Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.BaseDirectoryWrapper;
+import org.apache.lucene.util.LuceneTestCase;
+
/* Intentionally outside of oal.index to verify fully
external codecs work fine */
public class TestExternalCodecs extends LuceneTestCase {
- private static final class CustomPerFieldCodec extends Lucene42Codec {
+ private static final class CustomPerFieldCodec extends Lucene45Codec {
private final PostingsFormat ramFormat = PostingsFormat.forName("RAMOnly");
private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene41");
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java Fri Aug 30 15:06:42 2013
@@ -135,7 +135,8 @@ public class TestMockAnalyzer extends Ba
// LUCENE-5153: test that wrapping an analyzer's reader is allowed
final Random random = random();
- Analyzer a = new AnalyzerWrapper() {
+ final Analyzer delegate = new MockAnalyzer(random);
+ Analyzer a = new AnalyzerWrapper(delegate.getReuseStrategy()) {
@Override
protected Reader wrapReader(String fieldName, Reader reader) {
@@ -149,7 +150,7 @@ public class TestMockAnalyzer extends Ba
@Override
protected Analyzer getWrappedAnalyzer(String fieldName) {
- return new MockAnalyzer(random);
+ return delegate;
}
};
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestLucene40DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestLucene40DocValuesFormat.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestLucene40DocValuesFormat.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestLucene40DocValuesFormat.java Fri Aug 30 15:06:42 2013
@@ -30,5 +30,11 @@ public class TestLucene40DocValuesFormat
protected Codec getCodec() {
return codec;
}
-
+
+ // LUCENE-4583: This codec should throw IAE on huge binary values:
+ @Override
+ protected boolean codecAcceptsHugeBinaryValues(String field) {
+ return false;
+ }
+
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java Fri Aug 30 15:06:42 2013
@@ -67,7 +67,7 @@ public class TestBlockPostingsFormat3 ex
// creates 8 fields with different options and does "duels" of fields against each other
public void test() throws Exception {
Directory dir = newDirectory();
- Analyzer analyzer = new Analyzer(new Analyzer.PerFieldReuseStrategy()) {
+ Analyzer analyzer = new Analyzer(Analyzer.PER_FIELD_REUSE_STRATEGY) {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader);
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/lucene42/TestLucene42DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/lucene42/TestLucene42DocValuesFormat.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/lucene42/TestLucene42DocValuesFormat.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/lucene42/TestLucene42DocValuesFormat.java Fri Aug 30 15:06:42 2013
@@ -24,10 +24,15 @@ import org.apache.lucene.index.BaseCompr
* Tests Lucene42DocValuesFormat
*/
public class TestLucene42DocValuesFormat extends BaseCompressingDocValuesFormatTestCase {
- private final Codec codec = new Lucene42Codec();
+ private final Codec codec = new Lucene42RWCodec();
@Override
protected Codec getCodec() {
return codec;
}
+
+ @Override
+ protected boolean codecAcceptsHugeBinaryValues(String field) {
+ return false;
+ }
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java Fri Aug 30 15:06:42 2013
@@ -25,7 +25,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42Codec;
+import org.apache.lucene.codecs.lucene45.Lucene45Codec;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -46,6 +46,7 @@ import org.apache.lucene.search.TermQuer
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util._TestUtil;
/**
* Basic tests of PerFieldDocValuesFormat
@@ -63,6 +64,11 @@ public class TestPerFieldDocValuesFormat
protected Codec getCodec() {
return codec;
}
+
+ @Override
+ protected boolean codecAcceptsHugeBinaryValues(String field) {
+ return _TestUtil.fieldSupportsHugeBinaryDocValues(field);
+ }
// just a simple trivial test
// TODO: we should come up with a test that somehow checks that segment suffix
@@ -73,9 +79,9 @@ public class TestPerFieldDocValuesFormat
Directory directory = newDirectory();
// we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
- final DocValuesFormat fast = DocValuesFormat.forName("Lucene42");
+ final DocValuesFormat fast = DocValuesFormat.forName("Lucene45");
final DocValuesFormat slow = DocValuesFormat.forName("SimpleText");
- iwc.setCodec(new Lucene42Codec() {
+ iwc.setCodec(new Lucene45Codec() {
@Override
public DocValuesFormat getDocValuesFormatForField(String field) {
if ("dv1".equals(field)) {
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java Fri Aug 30 15:06:42 2013
@@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42Codec;
+import org.apache.lucene.codecs.lucene45.Lucene45Codec;
import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
import org.apache.lucene.codecs.mocksep.MockSepPostingsFormat;
import org.apache.lucene.codecs.pulsing.Pulsing41PostingsFormat;
@@ -200,7 +200,7 @@ public class TestPerFieldPostingsFormat2
}
- public static class MockCodec extends Lucene42Codec {
+ public static class MockCodec extends Lucene45Codec {
final PostingsFormat lucene40 = new Lucene41PostingsFormat();
final PostingsFormat simpleText = new SimpleTextPostingsFormat();
final PostingsFormat mockSep = new MockSepPostingsFormat();
@@ -217,7 +217,7 @@ public class TestPerFieldPostingsFormat2
}
}
- public static class MockCodec2 extends Lucene42Codec {
+ public static class MockCodec2 extends Lucene45Codec {
final PostingsFormat lucene40 = new Lucene41PostingsFormat();
final PostingsFormat simpleText = new SimpleTextPostingsFormat();
@@ -268,7 +268,7 @@ public class TestPerFieldPostingsFormat2
}
public void testSameCodecDifferentInstance() throws Exception {
- Codec codec = new Lucene42Codec() {
+ Codec codec = new Lucene45Codec() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
if ("id".equals(field)) {
@@ -284,7 +284,7 @@ public class TestPerFieldPostingsFormat2
}
public void testSameCodecDifferentParams() throws Exception {
- Codec codec = new Lucene42Codec() {
+ Codec codec = new Lucene45Codec() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
if ("id".equals(field)) {
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/BinaryTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/BinaryTokenStream.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/BinaryTokenStream.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/BinaryTokenStream.java Fri Aug 30 15:06:42 2013
@@ -31,16 +31,19 @@ import org.apache.lucene.analysis.Canned
*/
public final class BinaryTokenStream extends TokenStream {
private final ByteTermAttribute bytesAtt = addAttribute(ByteTermAttribute.class);
+ private final BytesRef bytes;
private boolean available = true;
public BinaryTokenStream(BytesRef bytes) {
- bytesAtt.setBytesRef(bytes);
+ this.bytes = bytes;
}
@Override
public boolean incrementToken() {
if (available) {
+ clearAttributes();
available = false;
+ bytesAtt.setBytesRef(bytes);
return true;
}
return false;
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/Test2BPositions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/Test2BPositions.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/Test2BPositions.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/Test2BPositions.java Fri Aug 30 15:06:42 2013
@@ -88,14 +88,12 @@ public class Test2BPositions extends Luc
private final PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
int index;
- public MyTokenStream() {
- termAtt.setLength(1);
- termAtt.buffer()[0] = 'a';
- }
-
@Override
public boolean incrementToken() {
if (index < 52) {
+ clearAttributes();
+ termAtt.setLength(1);
+ termAtt.buffer()[0] = 'a';
posIncAtt.setPositionIncrement(1+index);
index++;
return true;
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java Fri Aug 30 15:06:42 2013
@@ -85,18 +85,14 @@ public class Test2BPostings extends Luce
public static final class MyTokenStream extends TokenStream {
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
- private final char buffer[];
int index;
- public MyTokenStream() {
- termAtt.setLength(1);
- buffer = termAtt.buffer();
- }
-
@Override
public boolean incrementToken() {
if (index <= 'z') {
- buffer[0] = (char) index++;
+ clearAttributes();
+ termAtt.setLength(1);
+ termAtt.buffer()[0] = (char) index++;
return true;
}
return false;
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/Test2BPostingsBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/Test2BPostingsBytes.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/Test2BPostingsBytes.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/index/Test2BPostingsBytes.java Fri Aug 30 15:06:42 2013
@@ -129,14 +129,12 @@ public class Test2BPostingsBytes extends
int index;
int n;
- public MyTokenStream() {
- termAtt.setLength(1);
- termAtt.buffer()[0] = 'a';
- }
-
@Override
public boolean incrementToken() {
if (index < n) {
+ clearAttributes();
+ termAtt.buffer()[0] = 'a';
+ termAtt.setLength(1);
index++;
return true;
}