You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/03/20 17:10:07 UTC
svn commit: r1458913 [2/2] - in /lucene/dev/branches/lucene4258/lucene:
codecs/src/java/org/apache/lucene/codecs/simpletext/
core/src/java/org/apache/lucene/codecs/
core/src/java/org/apache/lucene/codecs/lucene40/
core/src/java/org/apache/lucene/index/...
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1458913&r1=1458912&r2=1458913&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Wed Mar 20 16:10:06 2013
@@ -51,8 +51,8 @@ public final class SegmentReader extends
private final int numDocs;
final SegmentCoreReaders core;
+ final SegmentCoreReaders[] updates;
- private SegmentCoreReaders[] updates;
private final IOContext context;
private Fields fields;
private FieldInfos fieldInfos;
@@ -73,8 +73,8 @@ public final class SegmentReader extends
IOContext context) throws IOException {
this.si = si;
this.context = context;
- core = new SegmentCoreReaders(this, si, -1, context, termInfosIndexDivisor);
- initUpdates(si, termInfosIndexDivisor, context);
+ core = new SegmentCoreReaders(this, si.info, -1, context, termInfosIndexDivisor);
+ updates = initUpdates(si, termInfosIndexDivisor, context);
boolean success = false;
try {
if (si.hasDeletions()) {
@@ -109,8 +109,8 @@ public final class SegmentReader extends
* loading new live docs from a new deletes file. Used by openIfChanged.
*/
SegmentReader(SegmentInfoPerCommit si, SegmentCoreReaders core,
- IOContext context) throws IOException {
- this(si, context, core, si.info.getCodec().liveDocsFormat()
+ SegmentCoreReaders[] updates, IOContext context) throws IOException {
+ this(si, context, core, updates, si.info.getCodec().liveDocsFormat()
.readLiveDocs(si.info.dir, si, context), si.info.getDocCount()
- si.getDelCount());
}
@@ -121,12 +121,12 @@ public final class SegmentReader extends
* NRT reader
*/
SegmentReader(SegmentInfoPerCommit si, IOContext context,
- SegmentCoreReaders core, Bits liveDocs, int numDocs) {
+ SegmentCoreReaders core, SegmentCoreReaders[] updates, Bits liveDocs, int numDocs) {
this.si = si;
this.context = context;
this.core = core;
core.incRef();
- this.updates = null;
+ this.updates = updates;
// TODO : handle NRT updates, add field liveUpdates
assert liveDocs != null;
@@ -135,17 +135,18 @@ public final class SegmentReader extends
this.numDocs = numDocs;
}
- private void initUpdates(SegmentInfoPerCommit si, int termInfosIndexDivisor,
+ private SegmentCoreReaders[] initUpdates(SegmentInfoPerCommit si, int termInfosIndexDivisor,
IOContext context) throws IOException {
if (si.hasUpdates()) {
- updates = new SegmentCoreReaders[(int) si.getUpdateGen()];
- for (int i = 0; i < updates.length; i++) {
- updates[i] = new SegmentCoreReaders(this, si, i + 1, context,
+ SegmentCoreReaders[] newUpdates = new SegmentCoreReaders[(int) si
+ .getUpdateGen()];
+ for (int i = 0; i < newUpdates.length; i++) {
+ newUpdates[i] = new SegmentCoreReaders(this, si.info, i + 1, context,
termInfosIndexDivisor);
}
- return;
+ return newUpdates;
}
- updates = null;
+ return null;
}
@Override
@@ -453,7 +454,16 @@ public final class SegmentReader extends
@Override
public NumericDocValues getNormValues(String field) throws IOException {
ensureOpen();
- return core.getNormValues(field);
+ NumericDocValues normValues = core.getNormValues(field);
+ if (updates != null) {
+ for (final SegmentCoreReaders updateReader : updates) {
+ NumericDocValues updateNormValues = updateReader.getNormValues(field);
+ if (updateNormValues != null) {
+ normValues = updateNormValues;
+ }
+ }
+ }
+ return normValues;
}
/**
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StackedTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StackedTermsEnum.java?rev=1458913&r1=1458912&r2=1458913&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StackedTermsEnum.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StackedTermsEnum.java Wed Mar 20 16:10:06 2013
@@ -87,7 +87,7 @@ public class StackedTermsEnum extends Te
}
private void init() throws IOException {
- activeEnums = new TreeSet<InnerTermsEnum>();
+ activeEnums = new TreeSet<InnerTermsEnum>(new InnerTermsEnumFullComparator());
for (int i = 0; i < subTerms.length; i++) {
if (subTerms[i] != null) {
final TermsEnum termsEnum = subTerms[i].iterator(null);
@@ -188,6 +188,14 @@ public class StackedTermsEnum extends Te
}
}
+ if (activeMap.isEmpty()) {
+ return null;
+ }
+
+ if (replacements == null && activeMap.size() == 1) {
+ return activeMap.keySet().iterator().next();
+ }
+
return new StackedDocsEnum(activeMap, replacements);
}
@@ -208,6 +216,10 @@ public class StackedTermsEnum extends Te
return null;
}
+ if (replacements == null && activeMap.size() == 1) {
+ return (DocsAndPositionsEnum) activeMap.keySet().iterator().next();
+ }
+
return new StackedDocsEnum(activeMap, replacements);
}
@@ -238,9 +250,29 @@ public class StackedTermsEnum extends Te
@Override
public int compareTo(InnerTermsEnum o) {
- return comparator.compare(this.term, o.term);
+ int diff = comparator.compare(this.term, o.term);
+ if (diff != 0) {
+ return diff;
+ }
+ return this.index - o.index;
+
+ }
+
+ }
+
+ /**
+ * A comparator which
+ */
+ private class InnerTermsEnumFullComparator implements Comparator<InnerTermsEnum> {
+
+ @Override
+ public int compare(InnerTermsEnum arg0, InnerTermsEnum arg1) {
+ int diff = comparator.compare(arg0.term, arg1.term);
+ if (diff != 0) {
+ return diff;
+ }
+ return arg0.index - arg1.index;
}
}
-
}
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java?rev=1458913&r1=1458912&r2=1458913&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java Wed Mar 20 16:10:06 2013
@@ -172,7 +172,8 @@ final class StandardDirectoryReader exte
// Steal the ref returned by SegmentReader ctor:
assert infos.info(i).info.dir == newReaders[i].getSegmentInfo().info.dir;
assert infos.info(i).hasDeletions();
- newReaders[i] = new SegmentReader(infos.info(i), newReaders[i].core, IOContext.READ);
+ newReaders[i] = new SegmentReader(infos.info(i),
+ newReaders[i].core, newReaders[i].updates, IOContext.READ);
}
}
success = true;
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java?rev=1458913&r1=1458912&r2=1458913&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java Wed Mar 20 16:10:06 2013
@@ -19,10 +19,12 @@ package org.apache.lucene.index;
import java.io.IOException;
+import org.apache.lucene.store.Directory;
+
abstract class StoredFieldsConsumer {
- abstract void addField(int docID, StorableField field, FieldInfo fieldInfo) throws IOException;
- abstract void flush(SegmentWriteState state) throws IOException;
- abstract void abort() throws IOException;
- abstract void startDocument() throws IOException;
- abstract void finishDocument() throws IOException;
+ abstract void addField(int docID, StorableField field, FieldInfo fieldInfo) throws IOException;
+ abstract void flush(SegmentWriteState state) throws IOException;
+ abstract void abort() throws IOException;
+ abstract void startDocument() throws IOException;
+ abstract void finishDocument(Directory directory, SegmentInfo info) throws IOException;
}
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StoredFieldsProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StoredFieldsProcessor.java?rev=1458913&r1=1458912&r2=1458913&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StoredFieldsProcessor.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StoredFieldsProcessor.java Wed Mar 20 16:10:06 2013
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.StoredFieldsWriter;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.RamUsageEstimator;
@@ -66,7 +67,7 @@ final class StoredFieldsProcessor extend
// It's possible that all documents seen in this segment
// hit non-aborting exceptions, in which case we will
// not have yet init'd the FieldsWriter:
- initFieldsWriter(state.context);
+ initFieldsWriter(state.context, state.directory, state.segmentInfo);
fill(numDocs);
}
@@ -81,9 +82,9 @@ final class StoredFieldsProcessor extend
}
}
- private synchronized void initFieldsWriter(IOContext context) throws IOException {
+ private synchronized void initFieldsWriter(IOContext context, Directory directory, SegmentInfo info) throws IOException {
if (fieldsWriter == null) {
- fieldsWriter = codec.storedFieldsFormat().fieldsWriter(docWriter.directory, docWriter.getSegmentInfo(), context);
+ fieldsWriter = codec.storedFieldsFormat().fieldsWriter(directory, info, context);
lastDocID = 0;
}
}
@@ -113,10 +114,10 @@ final class StoredFieldsProcessor extend
}
@Override
- void finishDocument() throws IOException {
+ void finishDocument(Directory directory, SegmentInfo info) throws IOException {
assert docWriter.writer.testPoint("StoredFieldsWriter.finishDocument start");
- initFieldsWriter(IOContext.DEFAULT);
+ initFieldsWriter(IOContext.DEFAULT, directory, info);
fill(docState.docID);
if (fieldsWriter != null && numStoredFields > 0) {
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java?rev=1458913&r1=1458912&r2=1458913&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java Wed Mar 20 16:10:06 2013
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Map;
import org.apache.lucene.codecs.TermVectorsWriter;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.ArrayUtil;
@@ -83,16 +84,16 @@ final class TermVectorsConsumer extends
}
}
- private final void initTermVectorsWriter() throws IOException {
+ private final void initTermVectorsWriter(Directory directory, SegmentInfo info) throws IOException {
if (writer == null) {
IOContext context = new IOContext(new FlushInfo(docWriter.getNumDocsInRAM(), docWriter.bytesUsed()));
- writer = docWriter.codec.termVectorsFormat().vectorsWriter(docWriter.directory, docWriter.getSegmentInfo(), context);
+ writer = docWriter.codec.termVectorsFormat().vectorsWriter(directory, info, context);
lastDocID = 0;
}
}
@Override
- void finishDocument(TermsHash termsHash) throws IOException {
+ void finishDocument(TermsHash termsHash, Directory directory, SegmentInfo info) throws IOException {
assert docWriter.writer.testPoint("TermVectorsTermsWriter.finishDocument start");
@@ -100,7 +101,7 @@ final class TermVectorsConsumer extends
return;
}
- initTermVectorsWriter();
+ initTermVectorsWriter(directory, info);
fill(docState.docID);
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermsHash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermsHash.java?rev=1458913&r1=1458912&r2=1458913&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermsHash.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermsHash.java Wed Mar 20 16:10:06 2013
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Counter;
@@ -126,10 +127,10 @@ final class TermsHash extends InvertedDo
}
@Override
- void finishDocument() throws IOException {
- consumer.finishDocument(this);
+ void finishDocument(Directory directory, SegmentInfo info) throws IOException {
+ consumer.finishDocument(this, directory, info);
if (nextTermsHash != null) {
- nextTermsHash.consumer.finishDocument(nextTermsHash);
+ nextTermsHash.consumer.finishDocument(nextTermsHash, directory, info);
}
}
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumer.java?rev=1458913&r1=1458912&r2=1458913&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumer.java Wed Mar 20 16:10:06 2013
@@ -20,10 +20,12 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Map;
+import org.apache.lucene.store.Directory;
+
abstract class TermsHashConsumer {
abstract void flush(Map<String, TermsHashConsumerPerField> fieldsToFlush, final SegmentWriteState state) throws IOException;
abstract void abort();
abstract void startDocument() throws IOException;
- abstract void finishDocument(TermsHash termsHash) throws IOException;
+ abstract void finishDocument(TermsHash termsHash, Directory directory, SegmentInfo info) throws IOException;
abstract public TermsHashConsumerPerField addField(TermsHashPerField termsHashPerField, FieldInfo fieldInfo);
}
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TwoStoredFieldsConsumers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TwoStoredFieldsConsumers.java?rev=1458913&r1=1458912&r2=1458913&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TwoStoredFieldsConsumers.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TwoStoredFieldsConsumers.java Wed Mar 20 16:10:06 2013
@@ -19,6 +19,8 @@ package org.apache.lucene.index;
import java.io.IOException;
+import org.apache.lucene.store.Directory;
+
/** Just switches between two {@link DocFieldConsumer}s. */
class TwoStoredFieldsConsumers extends StoredFieldsConsumer {
@@ -61,8 +63,8 @@ class TwoStoredFieldsConsumers extends S
}
@Override
- void finishDocument() throws IOException {
- first.finishDocument();
- second.finishDocument();
+ void finishDocument(Directory directory, SegmentInfo info) throws IOException {
+ first.finishDocument(directory, info);
+ second.finishDocument(directory, info);
}
}
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/UpdatedSegmentData.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/UpdatedSegmentData.java?rev=1458913&r1=1458912&r2=1458913&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/UpdatedSegmentData.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/UpdatedSegmentData.java Wed Mar 20 16:10:06 2013
@@ -1,20 +1,17 @@
package org.apache.lucene.index;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
-import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.SortedSet;
+import java.util.PriorityQueue;
import java.util.TreeMap;
-import java.util.TreeSet;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.StoredField;
-import org.apache.lucene.index.FieldsUpdate.Operation;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.Bits;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -38,41 +35,56 @@ import org.apache.lucene.index.FieldsUpd
*/
class UpdatedSegmentData {
+ static final FieldInfos EMPTY_FIELD_INFOS = new FieldInfos(new FieldInfo[0]);
+
/** Updates mapped by doc ID, for each do sorted list of updates. */
- private TreeMap<Integer,SortedSet<FieldsUpdate>> updatesMap;
+ private TreeMap<Integer,PriorityQueue<FieldsUpdate>> updatesMap;
- public long generation;
+ /** */
+ private long generation;
- private Map<String,FieldGenerationReplacements> fieldGenerationReplacments;
+ private Map<String,FieldGenerationReplacements> fieldGenerationReplacments = new HashMap<String,FieldGenerationReplacements>();
- private Iterator<Entry<Integer,SortedSet<FieldsUpdate>>> updatesIterator;
+ private Iterator<Entry<Integer,PriorityQueue<FieldsUpdate>>> updatesIterator;
private int currDocID;
private int nextDocID;
private int numDocs;
- private SortedSet<FieldsUpdate> nextUpdate;
+ private PriorityQueue<FieldsUpdate> nextUpdate;
private Analyzer analyzer;
+ private int termsIndexDivisor;
+
UpdatedSegmentData() {
- updatesMap = new TreeMap<Integer,SortedSet<FieldsUpdate>>();
+ updatesMap = new TreeMap<Integer,PriorityQueue<FieldsUpdate>>();
}
- void addUpdate(int docID, FieldsUpdate update) {
- SortedSet<FieldsUpdate> prevUpdates = updatesMap.get(docID);
+ void addUpdate(int docID, FieldsUpdate fieldsUpdate) {
+ if (docID >= fieldsUpdate.docIDUpto) {
+ return;
+ }
+ PriorityQueue<FieldsUpdate> prevUpdates = updatesMap.get(docID);
if (prevUpdates == null) {
- prevUpdates = new TreeSet<FieldsUpdate>();
+ prevUpdates = new PriorityQueue<FieldsUpdate>();
updatesMap.put(docID, prevUpdates);
+ } else {
+ System.out.println();
}
- prevUpdates.add(update);
+ prevUpdates.add(fieldsUpdate);
}
void addUpdates(int docID, FieldsUpdate[] updatesArray) {
- SortedSet<FieldsUpdate> prevUpdates = updatesMap.get(docID);
+ PriorityQueue<FieldsUpdate> prevUpdates = updatesMap.get(docID);
if (prevUpdates == null) {
- prevUpdates = new TreeSet<FieldsUpdate>();
- updatesMap.put(docID, prevUpdates);
+ prevUpdates = new PriorityQueue<FieldsUpdate>();
}
for (int i = 0; i < updatesArray.length; i++) {
- prevUpdates.add(updatesArray[i]);
+ FieldsUpdate fieldsUpdate = updatesArray[i];
+ if (docID < fieldsUpdate.docIDUpto) {
+ prevUpdates.add(fieldsUpdate);
+ }
+ }
+ if (!prevUpdates.isEmpty()) {
+ updatesMap.put(docID, prevUpdates);
}
}
@@ -87,12 +99,16 @@ class UpdatedSegmentData {
* The updates generation.
* @param numDocs
* number of documents in the base segment
+ * @param termsIndexDivisor
+ * Terms index divisor to use in temporary segments
*/
- void startWriting(long generation, int numDocs) {
+ void startWriting(long generation, int numDocs, int termsIndexDivisor) {
this.generation = generation;
this.numDocs = numDocs;
+ this.termsIndexDivisor = termsIndexDivisor;
updatesIterator = updatesMap.entrySet().iterator();
currDocID = 0;
+ fieldGenerationReplacments.clear();
// fetch the first actual updates document if exists
nextDocUpdate();
}
@@ -102,7 +118,7 @@ class UpdatedSegmentData {
*/
private void nextDocUpdate() {
if (updatesIterator.hasNext()) {
- Entry<Integer,SortedSet<FieldsUpdate>> docUpdates = updatesIterator
+ Entry<Integer,PriorityQueue<FieldsUpdate>> docUpdates = updatesIterator
.next();
nextDocID = docUpdates.getKey();
nextUpdate = docUpdates.getValue();
@@ -112,39 +128,6 @@ class UpdatedSegmentData {
}
}
- /**
- * Get the next document to put in the updates index, could be an empty
- * document. Updates the analyzer.
- *
- * @throws IOException
- * If different analyzers were assigned to field updates affecting
- * the next document.
- */
- IndexDocument nextDocument() throws IOException {
- IndexDocument toReturn = null;
- if (currDocID < nextDocID) {
- // empty document required
- if (currDocID == numDocs - 1) {
- // add document with stored field for getting right size of segment when
- // reading stored documents
- toReturn = STORED_FIELD_DOCUMENT;
- } else {
- toReturn = EMPTY_DOCUMENT;
- }
- } else if (currDocID < numDocs) {
- // return an actual updates document...
- toReturn = new UpdatesIndexDocument(nextUpdate);
- // ... and fetch the next one if exists
- nextDocUpdate();
- } else {
- // no more documents required
- return null;
- }
-
- currDocID++;
- return toReturn;
- }
-
Analyzer getAnalyzer() {
return analyzer;
}
@@ -153,136 +136,172 @@ class UpdatedSegmentData {
return fieldGenerationReplacments;
}
- /**
- * An {@link IndexDocument} containing all the updates to a certain document
- * in a stacked segment, taking into account replacements.
- * <p>
- * Constructing an {@link UpdatesIndexDocument} also updates the containing
- * {@link UpdatedSegmentData}'s analyzer and its
- * {@link FieldGenerationReplacements} vectors for the relevant fields.
- */
- private class UpdatesIndexDocument implements IndexDocument {
-
- Map<String,List<IndexableField>> indexablesByField = new HashMap<String,List<IndexableField>>();
- Map<String,List<StorableField>> storablesByField = new HashMap<String,List<StorableField>>();
-
- public UpdatesIndexDocument(SortedSet<FieldsUpdate> fieldsUpdates)
- throws IOException {
- boolean setAnalyzer = true;
- analyzer = null;
- for (FieldsUpdate fieldsUpdate : fieldsUpdates) {
- // set analyzer and check for analyzer conflict
- if (setAnalyzer) {
- analyzer = fieldsUpdate.analyzer;
- setAnalyzer = false;
- } else if (analyzer != fieldsUpdate.analyzer) {
- throw new IOException(
- "two analyzers assigned to one updated document");
- }
-
- if (fieldsUpdate.operation == Operation.REPLACE_FIELDS) {
- // handle fields replacement
- for (IndexableField field : fieldsUpdate.fields.indexableFields()) {
- replaceField(field.name());
- }
- for (StorableField field : fieldsUpdate.fields.storableFields()) {
- replaceField(field.name());
- }
- }
-
- // add new fields
- for (IndexableField field : fieldsUpdate.fields.indexableFields()) {
- List<IndexableField> fieldList = indexablesByField.get(field.name());
- if (fieldList == null) {
- fieldList = new ArrayList<IndexableField>();
- indexablesByField.put(field.name(), fieldList);
- }
- fieldList.add(field);
- }
- for (StorableField field : fieldsUpdate.fields.storableFields()) {
- List<StorableField> fieldList = storablesByField.get(field.name());
- if (fieldList == null) {
- fieldList = new ArrayList<StorableField>();
- storablesByField.put(field.name(), fieldList);
+ AtomicReader nextReader() throws IOException {
+ AtomicReader toReturn = null;
+ if (currDocID < nextDocID) {
+ // empty documents reader required
+ toReturn = new UpdateAtomicReader(nextDocID - currDocID);
+ currDocID = nextDocID;
+ } else if (currDocID < numDocs) {
+ // get the an actual updates reader...
+ FieldsUpdate update = nextUpdate.poll();
+ toReturn = new UpdateAtomicReader(update.directory, update.segmentInfo,
+ IOContext.DEFAULT);
+
+ // ... and if done for this document remove from updates map
+ if (nextUpdate.isEmpty()) {
+ updatesIterator.remove();
+ }
+
+ // add generation replacements if exist
+ if (update.replacedFields != null) {
+ for (String fieldName : update.replacedFields) {
+ FieldGenerationReplacements fieldReplacement = fieldGenerationReplacments
+ .get(fieldName);
+ if (fieldReplacement == null) {
+ fieldReplacement = new FieldGenerationReplacements();
+ fieldGenerationReplacments.put(fieldName, fieldReplacement);
}
- fieldList.add(field);
+ fieldReplacement.set(currDocID, generation);
}
}
+ // move to next doc id
+ nextDocUpdate();
+ currDocID++;
}
- private void replaceField(String fieldName) {
- // remove previous fields
- indexablesByField.remove(fieldName);
- storablesByField.remove(fieldName);
-
- // update field generation replacement vector
- if (fieldGenerationReplacments == null) {
- fieldGenerationReplacments = new HashMap<String,FieldGenerationReplacements>();
- }
- FieldGenerationReplacements fieldReplacement = fieldGenerationReplacments
- .get(fieldName);
- if (fieldReplacement == null) {
- fieldReplacement = new FieldGenerationReplacements();
- fieldGenerationReplacments.put(fieldName, fieldReplacement);
+ return toReturn;
+ }
+
+ private class UpdateAtomicReader extends AtomicReader {
+
+ final private SegmentCoreReaders core;
+ final private int numDocs;
+
+ /**
+ * Constructor with fields directory, for actual updates.
+ *
+ * @param fieldsDir
+ * Directory with inverted fields.
+ * @param segmentInfo
+ * Info of the inverted fields segment.
+ * @param context
+ * IOContext to use.
+ * @throws IOException
+ * If cannot create the reader.
+ */
+ UpdateAtomicReader(Directory fieldsDir, SegmentInfo segmentInfo,
+ IOContext context) throws IOException {
+ core = new SegmentCoreReaders(null, segmentInfo, -1, context,
+ termsIndexDivisor);
+ numDocs = 1;
+ }
+
+ /**
+ * Constructor with fields directory, for actual updates.
+ */
+ UpdateAtomicReader(int numDocs) {
+ core = null;
+ this.numDocs = numDocs;
+ }
+
+ @Override
+ public Fields fields() throws IOException {
+ if (core == null) {
+ return null;
}
- fieldReplacement.set(currDocID, generation);
+ return core.fields;
}
@Override
- public Iterable<IndexableField> indexableFields() {
- List<IndexableField> indexableFields = new ArrayList<IndexableField>();
- for (List<IndexableField> byField : indexablesByField.values()) {
- indexableFields.addAll(byField);
+ public FieldInfos getFieldInfos() {
+ if (core == null) {
+ return EMPTY_FIELD_INFOS;
}
- return indexableFields;
+ return core.fieldInfos;
+ }
+
+ @Override
+ public Bits getLiveDocs() {
+ return null;
}
@Override
- public Iterable<StorableField> storableFields() {
- List<StorableField> storableFields = new ArrayList<StorableField>();
- for (List<StorableField> byField : storablesByField.values()) {
- storableFields.addAll(byField);
+ public Fields getTermVectors(int docID) throws IOException {
+ if (core == null) {
+ return null;
}
- return storableFields;
+ return core.termVectorsLocal.get().get(docID);
}
- }
-
- /**
- * An empty document to be used as filler to maintain doc IDs in stacked
- * segments.
- */
- private static final IndexDocument EMPTY_DOCUMENT = new IndexDocument() {
@Override
- public Iterable<StorableField> storableFields() {
- return Collections.emptyList();
+ public int numDocs() {
+ return numDocs;
}
@Override
- public Iterable<IndexableField> indexableFields() {
- return Collections.emptyList();
+ public int maxDoc() {
+ return numDocs;
}
- };
-
- private static final ArrayList<StorableField> STORED_FIELD_LIST = new ArrayList<StorableField>(
- 1);
- static {
- STORED_FIELD_LIST.add(new StoredField("dummy", ""));
- }
-
- /**
- * A document containing only one stored field to be used as the last document
- * in stacked segments.
- */
- private static final IndexDocument STORED_FIELD_DOCUMENT = new IndexDocument() {
+
+ @Override
+ public void document(int docID, StoredFieldVisitor visitor)
+ throws IOException {
+ if (core == null) {
+ return;
+ }
+ core.fieldsReaderLocal.get().visitDocument(docID, visitor, null);
+ }
+
@Override
- public Iterable<StorableField> storableFields() {
- return STORED_FIELD_LIST;
+ public boolean hasDeletions() {
+ return false;
}
@Override
- public Iterable<IndexableField> indexableFields() {
- return Collections.emptyList();
+ protected void doClose() throws IOException {}
+
+ @Override
+ public NumericDocValues getNumericDocValues(String field)
+ throws IOException {
+ if (core == null) {
+ return null;
+ }
+ return core.getNumericDocValues(field);
+ }
+
+ @Override
+ public BinaryDocValues getBinaryDocValues(String field) throws IOException {
+ if (core == null) {
+ return null;
+ }
+ return core.getBinaryDocValues(field);
+ }
+
+ @Override
+ public SortedDocValues getSortedDocValues(String field) throws IOException {
+ if (core == null) {
+ return null;
+ }
+ return core.getSortedDocValues(field);
}
- };
+
+ @Override
+ public SortedSetDocValues getSortedSetDocValues(String field)
+ throws IOException {
+ if (core == null) {
+ return null;
+ }
+ return core.getSortedSetDocValues(field);
+ }
+
+ @Override
+ public NumericDocValues getNormValues(String field) throws IOException {
+ if (core == null) {
+ return null;
+ }
+ return core.getNormValues(field);
+ }
+
+ }
}
Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldReplacements.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldReplacements.java?rev=1458913&r1=1458912&r2=1458913&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldReplacements.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldReplacements.java Wed Mar 20 16:10:06 2013
@@ -17,13 +17,17 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Locale;
+import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.codecs.simpletext.SimpleTextCodec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.FieldType;
@@ -32,7 +36,10 @@ import org.apache.lucene.document.TextFi
import org.apache.lucene.index.FieldsUpdate.Operation;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IndexData;
import org.apache.lucene.util.LuceneTestCase;
public class TestFieldReplacements extends LuceneTestCase {
@@ -61,6 +68,7 @@ public class TestFieldReplacements exten
public void setUp() throws Exception {
super.setUp();
dir = newDirectory();
+
// init fields data structures
int numFields = 4 + random().nextInt(4);
@@ -77,7 +85,7 @@ public class TestFieldReplacements exten
fieldTokens[i] = tokens.toArray(new String[tokens.size()]);
}
}
-
+
@Override
public void tearDown() throws Exception {
dir.close();
@@ -85,19 +93,21 @@ public class TestFieldReplacements exten
}
public void testEmptyIndex() throws IOException {
- // test performing fields addition and replace on an empty index
+ init(random());
+
+ // test performing fields addition and replacement on an empty index
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random())));
HashSet<Term> usedTerms = new HashSet<Term>();
Operation operation = Operation.REPLACE_FIELDS;
- writer.updateFields(operation, getOperationTerm(usedTerms),
- getFields(usedTerms));
+ writer.updateFields(operation, getOperationTerm(usedTerms, random()),
+ getFields(usedTerms, random()));
operation = Operation.ADD_FIELDS;
- writer.updateFields(operation, getOperationTerm(usedTerms),
- getFields(usedTerms));
+ writer.updateFields(operation, getOperationTerm(usedTerms, random()),
+ getFields(usedTerms, random()));
writer.close();
@@ -106,23 +116,39 @@ public class TestFieldReplacements exten
directoryReader.close();
}
- private void addDocuments() throws IOException {
-
+ private static void init(Random localRandom) {
+ int numFields = 4 + localRandom.nextInt(4);
+ fieldNames = new String[numFields];
+ fieldTokens = new String[numFields][];
+ for (int i = 0; i < numFields; i++) {
+ fieldNames[i] = "f" + i;
+ ArrayList<String> tokens = new ArrayList<String>();
+ final String[] allTokens = loremIpsum.split("\\s");
+ for (int index = localRandom.nextInt(2 + i); index < allTokens.length; index += 1 + localRandom
+ .nextInt(2 + i)) {
+ tokens.add(allTokens[index].toLowerCase());
+ }
+ fieldTokens[i] = tokens.toArray(new String[tokens.size()]);
+ }
+ }
+
+ private static void addDocuments(Directory directory, Random localRandom,
+ int maxDocs) throws IOException {
+ init(localRandom);
HashSet<Term> usedTerms = new HashSet<Term>();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random()));
+ config.setCodec(new SimpleTextCodec());
+ IndexWriter writer = new IndexWriter(directory, config);
// add random documents
- int numDocs = 10 + random().nextInt(50);
+ int numOps = 10 + localRandom.nextInt(50);
int nCommits = 0;
- for (int i = 0; i < numDocs; i++) {
-
- // create fields
- Document fields = getFields(usedTerms);
+ for (int i = 0; i < Math.min(maxDocs, numOps); i++) {
// select operation
- int opIndex = random().nextInt(10);
+ int opIndex = localRandom.nextInt(10);
Operation operation;
if (opIndex <= 1) {
if (opIndex == 0) {
@@ -135,26 +161,29 @@ public class TestFieldReplacements exten
}
// create term if needed
- Term term = getOperationTerm(usedTerms);
+ Term term = getOperationTerm(usedTerms, localRandom);
- writer.updateFields(operation, term, fields);
+ // create fields and update
+ writer.updateFields(operation, term, getFields(usedTerms, localRandom));
} else {
if (opIndex == 2) {
if (VERBOSE_FIELD_REPLACEMENTS) {
System.out.println("REPLACE_DOCUMENTS");
}
- Term term = getOperationTerm(usedTerms);
- writer.replaceDocument(term, fields);
+ Term term = getOperationTerm(usedTerms, localRandom);
+ // create document and replace
+ writer.replaceDocument(term, getFields(usedTerms, localRandom));
} else {
if (VERBOSE_FIELD_REPLACEMENTS) {
System.out.println("ADD_DOCUMENT");
}
- writer.addDocument(fields);
+ // create document and add
+ writer.addDocument(getFields(usedTerms, localRandom));
}
}
// commit about once every 10 docs
- int interCommit = random().nextInt(10);
+ int interCommit = localRandom.nextInt(10);
if (interCommit == 0) {
if (VERBOSE_FIELD_REPLACEMENTS) {
System.out.println("commit " + (++nCommits));
@@ -168,16 +197,16 @@ public class TestFieldReplacements exten
writer.close();
}
- public Document getFields(HashSet<Term> usedTerms) {
+ public static Document getFields(HashSet<Term> usedTerms, Random loaclRandom) {
Document fields = new Document();
- int nFields = 1 + random().nextInt(5);
+ int nFields = 1 + loaclRandom.nextInt(5);
for (int j = 0; j < nFields; j++) {
- boolean indexed = random().nextInt(8) > 0;
- int index = random().nextInt(fieldNames.length);
+ boolean indexed = loaclRandom.nextInt(8) > 0;
+ int index = loaclRandom.nextInt(fieldNames.length);
String fieldName = fieldNames[index];
String value = createFieldValue(fieldTokens[index], fieldName, indexed,
- usedTerms);
+ usedTerms, loaclRandom);
if (indexed) {
fields.add(new TextField(fieldName, value, Store.NO));
@@ -197,20 +226,20 @@ public class TestFieldReplacements exten
return fields;
}
- public Term getOperationTerm(HashSet<Term> usedTerms) {
+ public static Term getOperationTerm(HashSet<Term> usedTerms, Random loaclRandom) {
Term term = null;
- boolean used = random().nextInt(5) < 4;
+ boolean used = loaclRandom.nextInt(5) < 4;
if (used && !usedTerms.isEmpty()) {
final Iterator<Term> iterator = usedTerms.iterator();
- int usedIndex = random().nextInt(usedTerms.size());
+ int usedIndex = loaclRandom.nextInt(usedTerms.size());
for (int j = 0; j < usedIndex; j++) {
iterator.next();
}
term = iterator.next();
} else {
// select term
- int fieldIndex = random().nextInt(fieldNames.length);
- int textIndex = random().nextInt(fieldTokens[fieldIndex].length / 10);
+ int fieldIndex = loaclRandom.nextInt(fieldNames.length);
+ int textIndex = loaclRandom.nextInt(fieldTokens[fieldIndex].length / 10);
term = new Term(fieldNames[fieldIndex],
fieldTokens[fieldIndex][textIndex]);
}
@@ -220,11 +249,11 @@ public class TestFieldReplacements exten
return term;
}
- private String createFieldValue(String[] tokens, String fieldName,
- boolean indexed, HashSet<Term> usedTerms) {
+ private static String createFieldValue(String[] tokens, String fieldName,
+ boolean indexed, HashSet<Term> usedTerms, Random loaclRandom) {
StringBuilder builder = new StringBuilder();
- int index = random().nextInt(Math.min(10, tokens.length));
+ int index = loaclRandom.nextInt(Math.min(10, tokens.length));
while (index < tokens.length) {
builder.append(tokens[index]);
@@ -232,19 +261,217 @@ public class TestFieldReplacements exten
if (indexed) {
usedTerms.add(new Term(fieldName, tokens[index]));
}
- index += 1 + random().nextInt(10);
+ index += 1 + loaclRandom.nextInt(10);
}
return builder.toString();
}
public void testRandomIndexGeneration() throws IOException {
- addDocuments();
+ addDocuments(dir, random(), Integer.MAX_VALUE);
DirectoryReader directoryReader = DirectoryReader.open(dir);
directoryReader.close();
}
- public void testStatisticsAfterFieldUpdates() throws IOException {
+ public void testAddIndexes() throws IOException {
+ addDocuments(dir, random(), Integer.MAX_VALUE);
+ RAMDirectory addedDir = new RAMDirectory();
+ IndexWriter addedIndexWriter = new IndexWriter(addedDir,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ addedIndexWriter.addIndexes(dir);
+ addedIndexWriter.close();
+
+ DirectoryReader updatesReader = DirectoryReader.open(dir);
+ IndexData updatesIndexData = new IndexData(updatesReader);
+ updatesReader.close();
+
+ DirectoryReader addedReader = DirectoryReader.open(addedDir);
+ IndexData addedIndexData = new IndexData(addedReader);
+ addedReader.close();
+ addedDir.close();
+
+ assertEquals("Difference in addIndexes ", updatesIndexData, addedIndexData);
+ }
+
+
+ public void testIndexEquality() throws IOException {
+ // create index through updates
+ addDocuments(dir, new Random(3), Integer.MAX_VALUE);
+
+ DirectoryReader updatesReader = DirectoryReader.open(dir);
+ IndexData updatesIndexData = new IndexData(updatesReader);
+ System.out.println("Updates index data");
+ System.out.println(updatesIndexData.toString(true));
+ System.out.println();
+ updatesReader.close();
+
+ // create the same index directly
+ RAMDirectory directDir = new RAMDirectory();
+ IndexWriter directWriter = new IndexWriter(directDir,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+
+ Document doc = new Document();
+ doc.add(new StoredField("f0","elit, magna volutpat. tation ea dolor consequat, facilisis odio te soluta doming facer qui me consuetudium littera per nunc "));
+ doc.add(new TextField("f4","consectetuer tincidunt erat nostrud hendrerit dignissim claritatem me etiam quam claram, ", Store.NO));
+ doc.add(new TextField("f3","nibh iriure qui liber claritatem. claram, seacula videntur sollemnes ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f4","diam eum nulla nulla claritatem. mutationem claram, ", Store.NO));
+ doc.add(new TextField("f4","amet, erat eum delenit iis claritatem. claram, et fiant ", Store.NO));
+ doc.add(new StoredField("f0","dolore quis duis iriure illum accumsan blandit tempor nihil facer assum. qui lectores dynamicus, claram, quinta qui sollemnes "));
+ doc.add(new StoredField("f1","elit, dolore aliquip dolore et facilisi. nobis placerat demonstraverunt processus qui littera eodem clari, "));
+ doc.add(new TextField("f1","nonummy ad commodo at te eleifend congue doming in demonstraverunt consuetudium est eodem ", Store.NO));
+ doc.add(new TextField("f3","dolore volutpat. exerci nisl consequat, delenit liber nobis qui lectores saepius. et fiant ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f5","magna dolore luptatum claritatem investigationes quod per ", Store.NO));
+ doc.add(new TextField("f2","elit, sed dolore aliquip commodo eum dignissim feugait doming habent insitam; legunt est qui quarta parum ", Store.NO));
+ doc.add(new StoredField("f3","nibh volutpat. in facilisis accumsan luptatum mazim lectores sequitur anteposuerit sollemnes "));
+ doc.add(new TextField("f2","euismod suscipit eum dolor molestie at qui duis doming in lius qui notare nunc ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f4","tincidunt velit facilisis dignissim cum iis claram, ", Store.NO));
+ doc.add(new StoredField("f4","ullamcorper accumsan delenit dolore nihil claritatem. mutationem clari, "));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f3","exerci ea esse consequat, facilisis praesent placerat dynamicus, seacula qui ", Store.NO));
+ doc.add(new TextField("f2","sed nonummy erat duis eum iriure dignissim duis nam assum. insitam; qui quam nunc futurum. ", Store.NO));
+ doc.add(new TextField("f5","velit luptatum augue placerat quam ", Store.NO));
+ doc.add(new TextField("f3","minim commodo facilisis qui imperdiet ii claritas seacula ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f2","tincidunt suscipit dolor eu dignissim delenit congue possim lius anteposuerit in ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f5","consectetuer illum eleifend processus fiant ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f1","volutpat. minim aliquip duis dolore zzril congue in saepius. dynamicus, qui est eodem qui futurum. ", Store.NO));
+ doc.add(new TextField("f0","ut quis duis eum hendrerit dolore odio feugait option doming mazim possim usus claritatem. legunt mirum litterarum qui sollemnes futurum. ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f0","nibh ut ut minim exerci ea duis esse et blandit luptatum facilisi. soluta doming quod typi usus quod dynamicus, consuetudium mirum quam quarta clari, in ", Store.NO));
+ doc.add(new TextField("f4","wisi facilisis claritatem iis lius mutationem qui ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new StoredField("f4","nibh ullamcorper ea dignissim usus mutationem quarta "));
+ doc.add(new StoredField("f4","consectetuer wisi ea illum facilisis assum. mutationem quarta clari, "));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f5","velit tempor processus putamus et typi, ", Store.NO));
+ doc.add(new TextField("f0","nibh dolore exerci eum esse feugiat facilisis iusto dolore cum quod non facit legunt quam claram, litterarum nunc ", Store.NO));
+ doc.add(new TextField("f4","consectetuer ullamcorper eum dignissim dolore assum. est littera in ", Store.NO));
+ doc.add(new TextField("f2","ipsum ad duis dolor eu at nam doming habent eorum me consuetudium decima futurum. ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f3","adipiscing wisi nisl consequat, dignissim nobis qui mirum fiant sollemnes ", Store.NO));
+ doc.add(new TextField("f0","euismod ut ad nisl dolor eu blandit te eleifend nihil typi qui lectores claritas consuetudium gothica, claram, decima sollemnes ", Store.NO));
+ doc.add(new TextField("f0","ut erat ut nisl ea dolor velit vel eros odio qui feugait facilisi. nihil assum. usus ii legunt littera decima nobis sollemnes ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f5","velit tempor legentis mirum fiant ", Store.NO));
+ doc.add(new TextField("f0","nibh dolore exerci eum esse feugiat facilisis iusto dolore cum quod non facit legunt quam claram, litterarum nunc ", Store.NO));
+ doc.add(new TextField("f4","consectetuer ullamcorper eum dignissim dolore assum. est littera in ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f2","dolore tation duis in eu delenit nam placerat in qui quarta ", Store.NO));
+ doc.add(new TextField("f2","ut suscipit duis at dignissim delenit soluta insitam; me quam qui futurum. ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f0","diam euismod quis autem consequat, eros iusto delenit feugait option quod habent claritatem claritatem. lectores consuetudium nunc per qui sollemnes ", Store.NO));
+ doc.add(new TextField("f4","nibh tincidunt hendrerit nulla usus est quam qui ", Store.NO));
+ doc.add(new TextField("f1","adipiscing diam nostrud duis at zzril te nobis congue est demonstraverunt lius consuetudium est claram, qui in ", Store.NO));
+ doc.add(new StoredField("f0","nibh euismod magna erat suscipit duis dolor esse et delenit tempor quod typi in legunt littera nunc decima. in "));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f1","diam nonummy tincidunt lobortis dolor et luptatum liber cum doming quod assum. habent insitam; est ii littera per et qui ", Store.NO));
+ doc.add(new TextField("f4","diam ullamcorper dignissim assum. claritatem. me etiam qui clari, ", Store.NO));
+ doc.add(new TextField("f0","ipsum ut volutpat. minim autem dolor vulputate vel dolore odio blandit cum nobis mazim placerat facer possim est lectores sequitur consuetudium claram, modo qui in ", Store.NO));
+ doc.add(new TextField("f2","tincidunt nisl duis in zzril placerat habent qui parum litterarum qui ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new StoredField("f3","nibh ea consequat, accumsan tempor est dynamicus, seacula typi, videntur sollemnes "));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f1","volutpat. duis dolor esse at iusto delenit doming est facit est consuetudium humanitatis sollemnes ", Store.NO));
+ doc.add(new TextField("f2","ipsum ut nisl dolor dignissim nam placerat investigationes processus notare nunc in ", Store.NO));
+ doc.add(new TextField("f1","ipsum tincidunt nostrud lobortis in vel nulla dolore placerat facit ii quam littera formas nunc clari, ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f3","adipiscing ea consequat, qui nobis ii mirum et sollemnes ", Store.NO));
+ doc.add(new TextField("f5","velit te legere typi, ", Store.NO));
+ doc.add(new TextField("f3","nisl in dignissim delenit placerat est claritatem. notare anteposuerit et videntur ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f1","dolore nostrud suscipit lobortis duis vel et delenit liber cum habent usus claritatem. qui formas nobis sollemnes futurum. ", Store.NO));
+ doc.add(new TextField("f2","erat tation duis in molestie dignissim liber congue possim me qui litterarum eodem in ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f0","amet, elit, ut minim duis eum esse vel eu iusto blandit nam eleifend nihil typi usus facit legunt notare litterarum per decima clari, ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f2","euismod suscipit in velit delenit facer legunt quam formas parum ", Store.NO));
+ doc.add(new StoredField("f4","ullamcorper accumsan delenit insitam; lius mutationem quarta decima. "));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new StoredField("f3","ipsum adipiscing wisi nisl consequat, praesent placerat qui saepius. dynamicus, seacula videntur "));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new StoredField("f0","consectetuer erat minim suscipit ea esse consequat, feugiat accumsan duis cum nihil typi claritatem facit etiam quam claram, quinta modo futurum. "));
+ doc.add(new TextField("f4","tincidunt illum cum claritatem. mutationem litterarum ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f2","tincidunt erat ad aliquip duis velit dignissim delenit facer insitam; processus qui litterarum formas quarta ", Store.NO));
+ doc.add(new StoredField("f2","erat ut nisl duis at feugait congue in lius anteposuerit nunc "));
+ doc.add(new TextField("f3","adipiscing minim esse luptatum tempor imperdiet est saepius. seacula fiant ", Store.NO));
+ doc.add(new TextField("f0","ipsum elit, magna suscipit dolor eu iusto feugait eleifend quod assum. non est investigationes claritas nunc seacula videntur ", Store.NO));
+ doc.add(new TextField("f4","amet, nibh ullamcorper velit nulla dignissim quod insitam; lius decima. ", Store.NO));
+ directWriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("f5","quis dolore eleifend investigationes mirum per eodem typi, ", Store.NO));
+ directWriter.addDocument(doc);
+
+ directWriter.close();
+ DirectoryReader directReader = DirectoryReader.open(directDir);
+
+ IndexData directIndexData = new IndexData(directReader);
+ System.out.println("Direct index data");
+ System.out.println(directIndexData.toString(true));
+ System.out.println();
+ directReader.close();
+ directDir.close();
+
+ boolean equalsNoOrder = IndexData.equalsNoOrder(directIndexData,
+ updatesIndexData);
+ assertTrue("indexes differ", equalsNoOrder);
+ }
+
+ public void testReplaceAndAddAgain() throws IOException {
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random())));
@@ -264,14 +491,14 @@ public class TestFieldReplacements exten
doc1.add(new StoredField("f1", "c", fieldType));
writer.addDocument(doc1);
- Document doc2 = new Document();
- doc2.add(new StoredField("f1", "b", fieldType));
- writer.addDocument(doc2);
-
Document doc3 = new Document();
doc3.add(new StoredField("f1", "d", fieldType));
writer.updateFields(Operation.REPLACE_FIELDS, new Term("f1", "b"), doc3);
+ Document doc2 = new Document();
+ doc2.add(new StoredField("f1", "b", fieldType));
+ writer.addDocument(doc2);
+
writer.close();
DirectoryReader directoryReader = DirectoryReader.open(dir);
@@ -288,6 +515,8 @@ public class TestFieldReplacements exten
final DocsAndPositionsEnum termPositionsB = atomicReader
.termPositionsEnum(new Term("f1", "b"));
+ assertEquals("wrong doc id", 2, termPositionsB.nextDoc());
+ assertEquals("wrong position", 0, termPositionsB.nextPosition());
assertEquals("wrong doc id", DocIdSetIterator.NO_MORE_DOCS,
termPositionsB.nextDoc());
@@ -302,8 +531,6 @@ public class TestFieldReplacements exten
.termPositionsEnum(new Term("f1", "d"));
assertEquals("wrong doc id", 0, termPositionsD.nextDoc());
assertEquals("wrong position", 0, termPositionsD.nextPosition());
- assertEquals("wrong doc id", 2, termPositionsD.nextDoc());
- assertEquals("wrong position", 0, termPositionsD.nextPosition());
assertEquals("wrong doc id", DocIdSetIterator.NO_MORE_DOCS,
termPositionsD.nextDoc());
@@ -312,7 +539,7 @@ public class TestFieldReplacements exten
final StorableField[] f1_0 = stored0.getFields("f1");
assertEquals("wrong numeber of stored fields", 1, f1_0.length);
assertEquals("wrong field value", "d", f1_0[0].stringValue());
-
+
final StoredDocument stored1 = atomicReader.document(1);
final StorableField[] f1_1 = stored1.getFields("f1");
assertEquals("wrong numeber of stored fields", 2, f1_1.length);
@@ -322,8 +549,8 @@ public class TestFieldReplacements exten
final StoredDocument stored2 = atomicReader.document(2);
final StorableField[] f1_2 = stored2.getFields("f1");
assertEquals("wrong numeber of stored fields", 1, f1_2.length);
- assertEquals("wrong field value", "d", f1_2[0].stringValue());
-
+ assertEquals("wrong field value", "b", f1_2[0].stringValue());
+
directoryReader.close();
}
@@ -351,4 +578,31 @@ public class TestFieldReplacements exten
}
}
}
+
+ public void testprintIndexes() throws IOException {
+ File outDir = new File("D:/temp/ifu/compare/scenario/c");
+ outDir.mkdirs();
+
+ for (int i = 0; i < 42; i++) {
+ //Directory directory = new RAMDirectory();
+ File fsDirFile = new File(outDir, "" + i);
+ fsDirFile.mkdirs();
+ Directory directory = FSDirectory.open(fsDirFile);
+ for (String filename : directory.listAll()) {
+ new File(fsDirFile, filename).delete();
+ }
+ addDocuments(directory, new Random(3), i);
+ DirectoryReader updatesReader = DirectoryReader.open(directory);
+ IndexData updatesIndexData = new IndexData(
+ updatesReader);
+ updatesReader.close();
+
+ File out = new File(outDir, (i < 10 ? "0" : "") + i + ".txt");
+ FileWriter fileWriter = new FileWriter(out);
+ fileWriter.append(updatesIndexData.toString());
+ fileWriter.close();
+ }
+ }
+
+
}
Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldUpdates.java?rev=1458913&r1=1458912&r2=1458913&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldUpdates.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldUpdates.java Wed Mar 20 16:10:06 2013
@@ -115,7 +115,8 @@ public class TestFieldUpdates extends Lu
// test that the norms are not present in the segment if
// omitNorms is true
- for (FieldInfo fi : reader.getFieldInfos()) {
+ FieldInfos fieldInfos = reader.getFieldInfos();
+ for (FieldInfo fi : fieldInfos) {
if (fi.isIndexed()) {
assertTrue("omitNorms=" + fi.omitsNorms() + " normsValue=" + reader.getNormValues(fi.name), fi.omitsNorms() == (reader.getNormValues(fi.name) == null));
}
Added: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/util/IndexData.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/util/IndexData.java?rev=1458913&view=auto
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/util/IndexData.java (added)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/util/IndexData.java Wed Mar 20 16:10:06 2013
@@ -0,0 +1,377 @@
+package org.apache.lucene.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.StorableField;
+import org.apache.lucene.index.StoredDocument;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.DocIdSetIterator;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains terms and stored fields extracted from a reader, to be used to
+ * compare indexes which contain similar documents.
+ */
+public class IndexData {
+
+ ArrayList<DocumentData> nonEmpty;
+
+ public IndexData(IndexReader reader) throws IOException {
+ Bits liveDocs = MultiFields.getLiveDocs(reader);
+
+ ArrayList<DocumentData> docs = new ArrayList<DocumentData>();
+ for (int i = 0; i < reader.maxDoc(); i++) {
+ if (liveDocs == null || liveDocs.get(i)) {
+ docs.add(new DocumentData(reader.document(i)));
+ } else {
+ docs.add(new DocumentData(new StoredDocument()));
+ }
+ }
+
+ Fields fields = MultiFields.getFields(reader);
+ if (fields != null) {
+ Iterator<String> fieldsIterator = fields.iterator();
+ while (fieldsIterator.hasNext()) {
+ String field = fieldsIterator.next();
+ Terms terms = fields.terms(field);
+ TermsEnum termsIterator = terms.iterator(null);
+ BytesRef term;
+ while ((term = termsIterator.next()) != null) {
+ DocsEnum termDocs = MultiFields.getTermDocsEnum(reader, liveDocs,
+ field, term);
+ while (termDocs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+ docs.get(termDocs.docID()).addTerm(field, term.toString(),
+ termDocs.freq());
+ }
+ }
+ }
+ }
+
+ nonEmpty = new ArrayList<DocumentData>();
+ for (DocumentData doc : docs) {
+ if (!doc.isEmpty()) {
+ nonEmpty.add(doc);
+ }
+ }
+ }
+
+ @Override
+ public String toString() {
+ return toString(nonEmpty);
+ }
+
+ public String toString(boolean ordered) {
+ if (!ordered) {
+ return toString();
+ }
+
+ ArrayList<DocumentData> sorted = new ArrayList<IndexData.DocumentData>(
+ nonEmpty);
+ Collections.sort(sorted, new DocumentDataComparator());
+ return toString(sorted);
+ }
+
+ public String toString(ArrayList<DocumentData> documentDatas) {
+ StringBuilder builder = new StringBuilder(this.getClass().getSimpleName());
+ builder.append('\n');
+ int counter = 0;
+ for (DocumentData doc : documentDatas) {
+ builder.append('\t');
+ builder.append("document ");
+ builder.append(counter++);
+ builder.append('\n');
+
+ // print stored fields
+ List<StorableField> storedFields = doc.storedFields;
+ if (!storedFields.isEmpty()) {
+ builder.append('\t');
+ builder.append('\t');
+ builder.append("storedFields");
+ builder.append('\n');
+ for (StorableField storableField : storedFields) {
+ builder.append('\t');
+ builder.append('\t');
+ builder.append('\t');
+ builder.append(storableField.name());
+ builder.append('\t');
+ builder.append(storableField.stringValue());
+ builder.append('\n');
+ }
+ }
+
+ // print terms
+ Map<String,Integer> termsMap = doc.termsMap;
+ if (!termsMap.isEmpty()) {
+ builder.append('\t');
+ builder.append('\t');
+ builder.append("termsMap");
+ builder.append('\n');
+
+ List<String> termsList = new ArrayList<String>(termsMap.keySet());
+ Collections.sort(termsList);
+ for (String term : termsList) {
+ builder.append('\t');
+ builder.append('\t');
+ builder.append('\t');
+ builder.append(term);
+ builder.append('\t');
+ builder.append(termsMap.get(term));
+ builder.append('\n');
+ }
+ }
+ }
+ return builder.toString();
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((nonEmpty == null) ? 0 : nonEmpty.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ IndexData other = (IndexData) obj;
+ if (nonEmpty == null) {
+ if (other.nonEmpty != null) return false;
+ } else if (!nonEmpty.equals(other.nonEmpty)) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Test if two index datas contain the same documents, ignoring documents
+ * order.
+ */
+ public static boolean equalsNoOrder(IndexData id1, IndexData id2) {
+ if (id1.nonEmpty == null) {
+ if (id2.nonEmpty != null) return false;
+ } else if (id2.nonEmpty == null) {
+ return false;
+ }
+
+ ArrayList<DocumentData> sorted1 = new ArrayList<IndexData.DocumentData>(
+ id1.nonEmpty);
+ ArrayList<DocumentData> sorted2 = new ArrayList<IndexData.DocumentData>(
+ id2.nonEmpty);
+
+ Collections.sort(sorted1, new DocumentDataComparator());
+ Collections.sort(sorted2, new DocumentDataComparator());
+
+ Iterator<DocumentData> iterator1 = sorted1.iterator();
+ Iterator<DocumentData> iterator2 = sorted2.iterator();
+
+ while (iterator1.hasNext()) {
+ if (!iterator2.hasNext()) {
+ return false;
+ }
+ if (!iterator1.next().equals(iterator2.next())) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ public static boolean testEquality(ArrayList<DocumentData> sorted1,
+ ArrayList<DocumentData> sorted2) {
+ Iterator<DocumentData> iterator1 = sorted1.iterator();
+ Iterator<DocumentData> iterator2 = sorted2.iterator();
+
+ while (iterator1.hasNext()) {
+ if (!iterator2.hasNext()) {
+ return false;
+ }
+ if (!iterator1.next().equals(iterator2.next())) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ private class DocumentData {
+
+ List<StorableField> storedFields;
+
+ TreeMap<String,Integer> termsMap = new TreeMap<String,Integer>();
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result
+ + ((storedFields == null) ? 0 : storedFields.hashCode());
+ result = prime * result + ((termsMap == null) ? 0 : termsMap.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ DocumentData other = (DocumentData) obj;
+ if (storedFields == null) {
+ if (other.storedFields != null) {
+ return false;
+ }
+ } else {
+ // compare stored fields on by one
+ if (storedFields.size() != other.storedFields.size()) {
+ return false;
+ }
+ Iterator<StorableField> fieldsThis = storedFields.iterator();
+ Iterator<StorableField> fieldsOther = other.storedFields.iterator();
+ while (fieldsThis.hasNext()) {
+ StorableField fieldThis = fieldsThis.next();
+ StorableField fieldOther = fieldsOther.next();
+ if (!fieldThis.name().equals(fieldOther.name())) {
+ return false;
+ }
+ if (!fieldThis.stringValue().equals(fieldOther.stringValue())) {
+ return false;
+ }
+ }
+ }
+ if (termsMap == null) {
+ if (other.termsMap != null) {
+ return false;
+ }
+ } else if (!termsMap.equals(other.termsMap)) {
+ return false;
+ }
+ return true;
+ }
+
+ public DocumentData(StoredDocument document) {
+ storedFields = document.getFields();
+ }
+
+ public boolean isEmpty() {
+ return storedFields.isEmpty() && termsMap.isEmpty();
+ }
+
+ public void addTerm(String field, String term, int freq) {
+ termsMap.put(field + ":" + term, freq);
+ }
+
+ }
+
+ private static class DocumentDataComparator implements Comparator<DocumentData> {
+
+ @Override
+ public int compare(DocumentData doc1, DocumentData doc2) {
+ // start with index terms
+ Map<String,Integer> terms1 = doc1.termsMap;
+ Map<String,Integer> terms2 = doc2.termsMap;
+ if (terms1.isEmpty() && !terms2.isEmpty()) {
+ return -1;
+ } else if (!terms1.isEmpty() && terms2.isEmpty()) {
+ return 1;
+ } else {
+ Iterator<String> iter1 = terms1.keySet().iterator();
+ Iterator<String> iter2 = terms2.keySet().iterator();
+ while (iter1.hasNext()) {
+ if (!iter2.hasNext()) {
+ return -1;
+ } else {
+ String t1 = iter1.next();
+ String t2 = iter2.next();
+ int compTerm = t1.compareTo(t2);
+ if (compTerm != 0) {
+ return compTerm;
+ }
+ int compFreq = terms1.get(t1) - terms2.get(t2);
+ if (compFreq != 0) {
+ return compFreq;
+ }
+ }
+ }
+ if (iter2.hasNext()) {
+ return 1;
+ }
+ }
+
+ // now check stored fields
+ List<StorableField> stored1 = doc1.storedFields;
+ List<StorableField> stored2 = doc2.storedFields;
+ if (stored1.isEmpty() && !stored2.isEmpty()) {
+ return -1;
+ } else if (!stored1.isEmpty() && stored2.isEmpty()) {
+ return 1;
+ } else {
+ Iterator<StorableField> iter1 = stored1.iterator();
+ Iterator<StorableField> iter2 = stored2.iterator();
+ while (iter1.hasNext()) {
+ if (!iter2.hasNext()) {
+ return -1;
+ } else {
+ StorableField f1 = iter1.next();
+ StorableField f2 = iter2.next();
+ int compFieldName = f1.name().compareTo(f2.name());
+ if (compFieldName != 0) {
+ return compFieldName;
+ }
+ compFieldName = f1.stringValue().compareTo(f2.stringValue());
+ if (compFieldName != 0) {
+ return compFieldName;
+ }
+ }
+ }
+ if (iter2.hasNext()) {
+ return 1;
+ }
+ }
+ return 0;
+
+ }
+ }
+}