You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2014/03/18 09:56:13 UTC
svn commit: r1578784 [1/2] - in /lucene/dev/trunk/lucene: ./
core/src/java/org/apache/lucene/index/ core/src/test/org/apache/lucene/index/
test-framework/src/java/org/apache/lucene/index/
Author: shaie
Date: Tue Mar 18 08:56:12 2014
New Revision: 1578784
URL: http://svn.apache.org/r1578784
Log:
LUCENE-5513: add IndexWriter.updateBinaryDocValue
Added:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java (with props)
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java (with props)
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java (with props)
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java (with props)
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java (with props)
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java (with props)
Removed:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericFieldUpdates.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericUpdate.java
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedUpdates.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CoalescedUpdates.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1578784&r1=1578783&r2=1578784&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Tue Mar 18 08:56:12 2014
@@ -108,6 +108,10 @@ New Features
* LUCENE-5530: ComplexPhraseQueryParser throws ParseException for fielded queries.
(Erick Erickson via Tomas Fernandez Lobbe and Ahmet Arslan)
+* LUCENE-5513: Add IndexWriter.updateBinaryDocValue which lets
+ you update the value of a BinaryDocValuesField without reindexing the
+ document(s). (Shai Erera)
+
API Changes
* LUCENE-5454: Add RandomAccessOrds, an optional extension of SortedSetDocValues
Added: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java?rev=1578784&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java (added)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java Tue Mar 18 08:56:12 2014
@@ -0,0 +1,233 @@
+package org.apache.lucene.index;
+
+import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.index.DocValuesUpdate.BinaryDocValuesUpdate;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.InPlaceMergeSorter;
+import org.apache.lucene.util.packed.PackedInts;
+import org.apache.lucene.util.packed.PagedGrowableWriter;
+import org.apache.lucene.util.packed.PagedMutable;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A {@link DocValuesFieldUpdates} which holds updates of documents, of a single
+ * {@link BinaryDocValuesField}.
+ *
+ * @lucene.experimental
+ */
+class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
+
+ final static class Iterator extends DocValuesFieldUpdates.Iterator {
+ private final PagedGrowableWriter offsets;
+ private final int size;
+ private final PagedGrowableWriter lengths;
+ private final PagedMutable docs;
+ private final FixedBitSet docsWithField;
+ private long idx = 0; // long so we don't overflow if size == Integer.MAX_VALUE
+ private int doc = -1;
+ private final BytesRef value;
+ private int offset, length;
+
+ Iterator(int size, PagedGrowableWriter offsets, PagedGrowableWriter lengths,
+ PagedMutable docs, BytesRef values, FixedBitSet docsWithField) {
+ this.offsets = offsets;
+ this.size = size;
+ this.lengths = lengths;
+ this.docs = docs;
+ this.docsWithField = docsWithField;
+ value = values.clone();
+ }
+
+ @Override
+ BytesRef value() {
+ if (offset == -1) {
+ return null;
+ } else {
+ value.offset = offset;
+ value.length = length;
+ return value;
+ }
+ }
+
+ @Override
+ int nextDoc() {
+ if (idx >= size) {
+ offset = -1;
+ return doc = DocIdSetIterator.NO_MORE_DOCS;
+ }
+ doc = (int) docs.get(idx);
+ ++idx;
+ while (idx < size && docs.get(idx) == doc) {
+ ++idx;
+ }
+ // idx points to the "next" element
+ long prevIdx = idx - 1;
+ if (!docsWithField.get((int) prevIdx)) {
+ offset = -1;
+ } else {
+ // cannot change 'value' here because nextDoc is called before the
+ // value is used, and it's a waste to clone the BytesRef when we
+ // obtain the value
+ offset = (int) offsets.get(prevIdx);
+ length = (int) lengths.get(prevIdx);
+ }
+ return doc;
+ }
+
+ @Override
+ int doc() {
+ return doc;
+ }
+
+ @Override
+ void reset() {
+ doc = -1;
+ offset = -1;
+ idx = 0;
+ }
+ }
+
+ private FixedBitSet docsWithField;
+ private PagedMutable docs;
+ private PagedGrowableWriter offsets, lengths;
+ private BytesRef values;
+ private int size;
+
+ public BinaryDocValuesFieldUpdates(String field, int maxDoc) {
+ super(field, Type.BINARY);
+ docsWithField = new FixedBitSet(64);
+ docs = new PagedMutable(1, 1024, PackedInts.bitsRequired(maxDoc - 1), PackedInts.COMPACT);
+ offsets = new PagedGrowableWriter(1, 1024, 1, PackedInts.FAST);
+ lengths = new PagedGrowableWriter(1, 1024, 1, PackedInts.FAST);
+ values = new BytesRef(16); // start small
+ size = 0;
+ }
+
+ @Override
+ public void add(int doc, Object value) {
+ // TODO: if the Sorter interface changes to take long indexes, we can remove that limitation
+ if (size == Integer.MAX_VALUE) {
+ throw new IllegalStateException("cannot support more than Integer.MAX_VALUE doc/value entries");
+ }
+
+ BytesRef val = (BytesRef) value;
+ if (val == null) {
+ val = BinaryDocValuesUpdate.MISSING;
+ }
+
+ // grow the structures to have room for more elements
+ if (docs.size() == size) {
+ docs = docs.grow(size + 1);
+ offsets = offsets.grow(size + 1);
+ lengths = lengths.grow(size + 1);
+ docsWithField = FixedBitSet.ensureCapacity(docsWithField, (int) docs.size());
+ }
+
+ if (val != BinaryDocValuesUpdate.MISSING) {
+ // only mark the document as having a value in that field if the value wasn't set to null (MISSING)
+ docsWithField.set(size);
+ }
+
+ docs.set(size, doc);
+ offsets.set(size, values.length);
+ lengths.set(size, val.length);
+ values.append(val);
+ ++size;
+ }
+
+ @Override
+ public Iterator iterator() {
+ final PagedMutable docs = this.docs;
+ final PagedGrowableWriter offsets = this.offsets;
+ final PagedGrowableWriter lengths = this.lengths;
+ final BytesRef values = this.values;
+ final FixedBitSet docsWithField = this.docsWithField;
+ new InPlaceMergeSorter() {
+ @Override
+ protected void swap(int i, int j) {
+ long tmpDoc = docs.get(j);
+ docs.set(j, docs.get(i));
+ docs.set(i, tmpDoc);
+
+ long tmpOffset = offsets.get(j);
+ offsets.set(j, offsets.get(i));
+ offsets.set(i, tmpOffset);
+
+ long tmpLength = lengths.get(j);
+ lengths.set(j, lengths.get(i));
+ lengths.set(i, tmpLength);
+
+ boolean tmpBool = docsWithField.get(j);
+ if (docsWithField.get(i)) {
+ docsWithField.set(j);
+ } else {
+ docsWithField.clear(j);
+ }
+ if (tmpBool) {
+ docsWithField.set(i);
+ } else {
+ docsWithField.clear(i);
+ }
+ }
+
+ @Override
+ protected int compare(int i, int j) {
+ int x = (int) docs.get(i);
+ int y = (int) docs.get(j);
+ return (x < y) ? -1 : ((x == y) ? 0 : 1);
+ }
+ }.sort(0, size);
+
+ return new Iterator(size, offsets, lengths, docs, values, docsWithField);
+ }
+
+ @Override
+ public void merge(DocValuesFieldUpdates other) {
+ BinaryDocValuesFieldUpdates otherUpdates = (BinaryDocValuesFieldUpdates) other;
+ int newSize = size + otherUpdates.size;
+ if (newSize > Integer.MAX_VALUE) {
+ throw new IllegalStateException(
+ "cannot support more than Integer.MAX_VALUE doc/value entries; size="
+ + size + " other.size=" + otherUpdates.size);
+ }
+ docs = docs.grow(newSize);
+ offsets = offsets.grow(newSize);
+ lengths = lengths.grow(newSize);
+ docsWithField = FixedBitSet.ensureCapacity(docsWithField, (int) docs.size());
+ for (int i = 0; i < otherUpdates.size; i++) {
+ int doc = (int) otherUpdates.docs.get(i);
+ if (otherUpdates.docsWithField.get(i)) {
+ docsWithField.set(size);
+ }
+ docs.set(size, doc);
+ offsets.set(size, values.length + otherUpdates.offsets.get(i)); // correct relative offset
+ lengths.set(size, otherUpdates.lengths.get(i));
+ ++size;
+ }
+ values.append(otherUpdates.values);
+ }
+
+ @Override
+ public boolean any() {
+ return size > 0;
+ }
+
+}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedUpdates.java?rev=1578784&r1=1578783&r2=1578784&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedUpdates.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedUpdates.java Tue Mar 18 08:56:12 2014
@@ -25,6 +25,8 @@ import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.lucene.index.DocValuesUpdate.BinaryDocValuesUpdate;
+import org.apache.lucene.index.DocValuesUpdate.NumericDocValuesUpdate;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.RamUsageEstimator;
@@ -92,9 +94,39 @@ class BufferedUpdates {
* NumericUpdate (val) counts its own size and isn't accounted for here.
*/
final static int BYTES_PER_NUMERIC_UPDATE_ENTRY = 7*RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_INT;
+
+ /* Rough logic: BinaryUpdate calculates its actual size,
+ * including the update Term and DV field (String). The
+ * per-field map holds a reference to the updated field, and
+ * therefore we only account for the object reference and
+ * map space itself. This is incremented when we first see
+ * an updated field.
+ *
+ * HashMap has an array[Entry] w/ varying load
+ * factor (say 2*POINTER). Entry is an object w/ String key,
+ * LinkedHashMap val, int hash, Entry next (OBJ_HEADER + 3*POINTER + INT).
+ *
+ * LinkedHashMap (val) is counted as OBJ_HEADER, array[Entry] ref + header, 4*INT, 1*FLOAT,
+ * Set (entrySet) (2*OBJ_HEADER + ARRAY_HEADER + 2*POINTER + 4*INT + FLOAT)
+ */
+ final static int BYTES_PER_BINARY_FIELD_ENTRY =
+ 7*RamUsageEstimator.NUM_BYTES_OBJECT_REF + 3*RamUsageEstimator.NUM_BYTES_OBJECT_HEADER +
+ RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + 5*RamUsageEstimator.NUM_BYTES_INT + RamUsageEstimator.NUM_BYTES_FLOAT;
+
+ /* Rough logic: Incremented when we see another Term for an already updated
+ * field.
+ * LinkedHashMap has an array[Entry] w/ varying load factor
+ * (say 2*POINTER). Entry is an object w/ Term key, BinaryUpdate val,
+ * int hash, Entry next, Entry before, Entry after (OBJ_HEADER + 5*POINTER + INT).
+ *
+ * Term (key) is counted only as POINTER.
+ * BinaryUpdate (val) counts its own size and isn't accounted for here.
+ */
+ final static int BYTES_PER_BINARY_UPDATE_ENTRY = 7*RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_INT;
final AtomicInteger numTermDeletes = new AtomicInteger();
final AtomicInteger numNumericUpdates = new AtomicInteger();
+ final AtomicInteger numBinaryUpdates = new AtomicInteger();
final Map<Term,Integer> terms = new HashMap<>();
final Map<Query,Integer> queries = new HashMap<>();
final List<Integer> docIDs = new ArrayList<>();
@@ -106,7 +138,16 @@ class BufferedUpdates {
// one that came in wins), and helps us detect faster if the same Term is
// used to update the same field multiple times (so we later traverse it
// only once).
- final Map<String,LinkedHashMap<Term,NumericUpdate>> numericUpdates = new HashMap<>();
+ final Map<String,LinkedHashMap<Term,NumericDocValuesUpdate>> numericUpdates = new HashMap<>();
+
+ // Map<dvField,Map<updateTerm,BinaryUpdate>>
+ // For each field we keep an ordered list of BinaryUpdates, key'd by the
+ // update Term. LinkedHashMap guarantees we will later traverse the map in
+ // insertion order (so that if two terms affect the same document, the last
+ // one that came in wins), and helps us detect faster if the same Term is
+ // used to update the same field multiple times (so we later traverse it
+ // only once).
+ final Map<String,LinkedHashMap<Term,BinaryDocValuesUpdate>> binaryUpdates = new HashMap<>();
public static final Integer MAX_INT = Integer.valueOf(Integer.MAX_VALUE);
@@ -125,7 +166,7 @@ class BufferedUpdates {
if (VERBOSE_DELETES) {
return "gen=" + gen + " numTerms=" + numTermDeletes + ", terms=" + terms
+ ", queries=" + queries + ", docIDs=" + docIDs + ", numericUpdates=" + numericUpdates
- + ", bytesUsed=" + bytesUsed;
+ + ", binaryUpdates=" + binaryUpdates + ", bytesUsed=" + bytesUsed;
} else {
String s = "gen=" + gen;
if (numTermDeletes.get() != 0) {
@@ -140,6 +181,9 @@ class BufferedUpdates {
if (numNumericUpdates.get() != 0) {
s += " " + numNumericUpdates.get() + " numeric updates (unique count=" + numericUpdates.size() + ")";
}
+ if (numBinaryUpdates.get() != 0) {
+ s += " " + numBinaryUpdates.get() + " binary updates (unique count=" + binaryUpdates.size() + ")";
+ }
if (bytesUsed.get() != 0) {
s += " bytesUsed=" + bytesUsed.get();
}
@@ -184,14 +228,14 @@ class BufferedUpdates {
}
}
- public void addNumericUpdate(NumericUpdate update, int docIDUpto) {
- LinkedHashMap<Term,NumericUpdate> fieldUpdates = numericUpdates.get(update.field);
+ public void addNumericUpdate(NumericDocValuesUpdate update, int docIDUpto) {
+ LinkedHashMap<Term,NumericDocValuesUpdate> fieldUpdates = numericUpdates.get(update.field);
if (fieldUpdates == null) {
fieldUpdates = new LinkedHashMap<>();
numericUpdates.put(update.field, fieldUpdates);
bytesUsed.addAndGet(BYTES_PER_NUMERIC_FIELD_ENTRY);
}
- final NumericUpdate current = fieldUpdates.get(update.term);
+ final NumericDocValuesUpdate current = fieldUpdates.get(update.term);
if (current != null && docIDUpto < current.docIDUpto) {
// Only record the new number if it's greater than or equal to the current
// one. This is important because if multiple threads are replacing the
@@ -213,17 +257,48 @@ class BufferedUpdates {
}
}
+ public void addBinaryUpdate(BinaryDocValuesUpdate update, int docIDUpto) {
+ LinkedHashMap<Term,BinaryDocValuesUpdate> fieldUpdates = binaryUpdates.get(update.field);
+ if (fieldUpdates == null) {
+ fieldUpdates = new LinkedHashMap<>();
+ binaryUpdates.put(update.field, fieldUpdates);
+ bytesUsed.addAndGet(BYTES_PER_BINARY_FIELD_ENTRY);
+ }
+ final BinaryDocValuesUpdate current = fieldUpdates.get(update.term);
+ if (current != null && docIDUpto < current.docIDUpto) {
+ // Only record the new number if it's greater than or equal to the current
+ // one. This is important because if multiple threads are replacing the
+ // same doc at nearly the same time, it's possible that one thread that
+ // got a higher docID is scheduled before the other threads.
+ return;
+ }
+
+ update.docIDUpto = docIDUpto;
+ // since it's a LinkedHashMap, we must first remove the Term entry so that
+ // it's added last (we're interested in insertion-order).
+ if (current != null) {
+ fieldUpdates.remove(update.term);
+ }
+ fieldUpdates.put(update.term, update);
+ numBinaryUpdates.incrementAndGet();
+ if (current == null) {
+ bytesUsed.addAndGet(BYTES_PER_BINARY_UPDATE_ENTRY + update.sizeInBytes());
+ }
+ }
+
void clear() {
terms.clear();
queries.clear();
docIDs.clear();
numericUpdates.clear();
+ binaryUpdates.clear();
numTermDeletes.set(0);
numNumericUpdates.set(0);
+ numBinaryUpdates.set(0);
bytesUsed.set(0);
}
boolean any() {
- return terms.size() > 0 || docIDs.size() > 0 || queries.size() > 0 || numericUpdates.size() > 0;
+ return terms.size() > 0 || docIDs.size() > 0 || queries.size() > 0 || numericUpdates.size() > 0 || binaryUpdates.size() > 0;
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java?rev=1578784&r1=1578783&r2=1578784&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java Tue Mar 18 08:56:12 2014
@@ -22,9 +22,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
@@ -214,20 +212,22 @@ class BufferedUpdatesStream {
int delCount = 0;
final boolean segAllDeletes;
try {
- Map<String,NumericFieldUpdates> fieldUpdates = null;
+ final DocValuesFieldUpdates.Container dvUpdates = new DocValuesFieldUpdates.Container();
if (coalescedDeletes != null) {
//System.out.println(" del coalesced");
delCount += applyTermDeletes(coalescedDeletes.termsIterable(), rld, reader);
delCount += applyQueryDeletes(coalescedDeletes.queriesIterable(), rld, reader);
- fieldUpdates = applyNumericDocValuesUpdates(coalescedDeletes.numericDVUpdates, rld, reader, fieldUpdates);
+ applyDocValuesUpdates(coalescedDeletes.numericDVUpdates, rld, reader, dvUpdates);
+ applyDocValuesUpdates(coalescedDeletes.binaryDVUpdates, rld, reader, dvUpdates);
}
//System.out.println(" del exact");
// Don't delete by Term here; DocumentsWriterPerThread
// already did that on flush:
delCount += applyQueryDeletes(packet.queriesIterable(), rld, reader);
- fieldUpdates = applyNumericDocValuesUpdates(Arrays.asList(packet.updates), rld, reader, fieldUpdates);
- if (!fieldUpdates.isEmpty()) {
- rld.writeFieldUpdates(info.info.dir, fieldUpdates);
+ applyDocValuesUpdates(Arrays.asList(packet.numericDVUpdates), rld, reader, dvUpdates);
+ applyDocValuesUpdates(Arrays.asList(packet.binaryDVUpdates), rld, reader, dvUpdates);
+ if (dvUpdates.any()) {
+ rld.writeFieldUpdates(info.info.dir, dvUpdates);
}
final int fullDelCount = rld.info.getDelCount() + rld.getPendingDeleteCount();
assert fullDelCount <= rld.info.info.getDocCount();
@@ -275,9 +275,11 @@ class BufferedUpdatesStream {
try {
delCount += applyTermDeletes(coalescedDeletes.termsIterable(), rld, reader);
delCount += applyQueryDeletes(coalescedDeletes.queriesIterable(), rld, reader);
- Map<String,NumericFieldUpdates> fieldUpdates = applyNumericDocValuesUpdates(coalescedDeletes.numericDVUpdates, rld, reader, null);
- if (!fieldUpdates.isEmpty()) {
- rld.writeFieldUpdates(info.info.dir, fieldUpdates);
+ DocValuesFieldUpdates.Container dvUpdates = new DocValuesFieldUpdates.Container();
+ applyDocValuesUpdates(coalescedDeletes.numericDVUpdates, rld, reader, dvUpdates);
+ applyDocValuesUpdates(coalescedDeletes.binaryDVUpdates, rld, reader, dvUpdates);
+ if (dvUpdates.any()) {
+ rld.writeFieldUpdates(info.info.dir, dvUpdates);
}
final int fullDelCount = rld.info.getDelCount() + rld.getPendingDeleteCount();
assert fullDelCount <= rld.info.info.getDocCount();
@@ -436,14 +438,13 @@ class BufferedUpdatesStream {
return delCount;
}
- // NumericDocValues Updates
- // If otherFieldUpdates != null, we need to merge the updates into them
- private synchronized Map<String,NumericFieldUpdates> applyNumericDocValuesUpdates(Iterable<NumericUpdate> updates,
- ReadersAndUpdates rld, SegmentReader reader, Map<String,NumericFieldUpdates> otherFieldUpdates) throws IOException {
+ // DocValues updates
+ private synchronized void applyDocValuesUpdates(Iterable<? extends DocValuesUpdate> updates,
+ ReadersAndUpdates rld, SegmentReader reader, DocValuesFieldUpdates.Container dvUpdatesContainer) throws IOException {
Fields fields = reader.fields();
if (fields == null) {
// This reader has no postings
- return Collections.emptyMap();
+ return;
}
// TODO: we can process the updates per DV field, from last to first so that
@@ -459,9 +460,9 @@ class BufferedUpdatesStream {
String currentField = null;
TermsEnum termsEnum = null;
DocsEnum docs = null;
- final Map<String,NumericFieldUpdates> result = otherFieldUpdates == null ? new HashMap<String,NumericFieldUpdates>() : otherFieldUpdates;
+
//System.out.println(Thread.currentThread().getName() + " numericDVUpdate reader=" + reader);
- for (NumericUpdate update : updates) {
+ for (DocValuesUpdate update : updates) {
Term term = update.term;
int limit = update.docIDUpto;
@@ -499,10 +500,9 @@ class BufferedUpdatesStream {
//System.out.println("BDS: got docsEnum=" + docsEnum);
- NumericFieldUpdates fieldUpdates = result.get(update.field);
- if (fieldUpdates == null) {
- fieldUpdates = new NumericFieldUpdates.PackedNumericFieldUpdates(reader.maxDoc());
- result.put(update.field, fieldUpdates);
+ DocValuesFieldUpdates dvUpdates = dvUpdatesContainer.getUpdates(update.field, update.type);
+ if (dvUpdates == null) {
+ dvUpdates = dvUpdatesContainer.newUpdates(update.field, update.type, reader.maxDoc());
}
int doc;
while ((doc = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
@@ -510,13 +510,12 @@ class BufferedUpdatesStream {
if (doc >= limit) {
break; // no more docs that can be updated for this term
}
- fieldUpdates.add(doc, update.value);
+ dvUpdates.add(doc, update.value);
}
}
}
- return result;
}
-
+
public static class QueryAndLimit {
public final Query query;
public final int limit;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CoalescedUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CoalescedUpdates.java?rev=1578784&r1=1578783&r2=1578784&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CoalescedUpdates.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CoalescedUpdates.java Tue Mar 18 08:56:12 2014
@@ -25,17 +25,23 @@ import java.util.Map;
import org.apache.lucene.search.Query;
import org.apache.lucene.index.BufferedUpdatesStream.QueryAndLimit;
+import org.apache.lucene.index.DocValuesUpdate.BinaryDocValuesUpdate;
+import org.apache.lucene.index.DocValuesUpdate.NumericDocValuesUpdate;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.MergedIterator;
class CoalescedUpdates {
final Map<Query,Integer> queries = new HashMap<>();
final List<Iterable<Term>> iterables = new ArrayList<>();
- final List<NumericUpdate> numericDVUpdates = new ArrayList<>();
+ final List<NumericDocValuesUpdate> numericDVUpdates = new ArrayList<>();
+ final List<BinaryDocValuesUpdate> binaryDVUpdates = new ArrayList<>();
@Override
public String toString() {
// note: we could add/collect more debugging information
- return "CoalescedUpdates(termSets=" + iterables.size() + ",queries=" + queries.size() + ",numericUpdates=" + numericDVUpdates.size() + ")";
+ return "CoalescedUpdates(termSets=" + iterables.size() + ",queries="
+ + queries.size() + ",numericDVUpdates=" + numericDVUpdates.size()
+ + ",binaryDVUpdates=" + binaryDVUpdates.size() + ")";
}
void update(FrozenBufferedUpdates in) {
@@ -46,11 +52,17 @@ class CoalescedUpdates {
queries.put(query, BufferedUpdates.MAX_INT);
}
- for (NumericUpdate nu : in.updates) {
- NumericUpdate clone = new NumericUpdate(nu.term, nu.field, nu.value);
+ for (NumericDocValuesUpdate nu : in.numericDVUpdates) {
+ NumericDocValuesUpdate clone = new NumericDocValuesUpdate(nu.term, nu.field, (Long) nu.value);
clone.docIDUpto = Integer.MAX_VALUE;
numericDVUpdates.add(clone);
}
+
+ for (BinaryDocValuesUpdate bu : in.binaryDVUpdates) {
+ BinaryDocValuesUpdate clone = new BinaryDocValuesUpdate(bu.term, bu.field, (BytesRef) bu.value);
+ clone.docIDUpto = Integer.MAX_VALUE;
+ binaryDVUpdates.add(clone);
+ }
}
public Iterable<Term> termsIterable() {
Added: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java?rev=1578784&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java (added)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java Tue Mar 18 08:56:12 2014
@@ -0,0 +1,154 @@
+package org.apache.lucene.index;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.index.NumericDocValuesFieldUpdates;
+import org.apache.lucene.search.DocIdSetIterator;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Holds updates of a single DocValues field, for a set of documents.
+ *
+ * @lucene.experimental
+ */
+abstract class DocValuesFieldUpdates {
+
+ static enum Type { NUMERIC, BINARY }
+
+ /**
+ * An iterator over documents and their updated values. Only documents with
+ * updates are returned by this iterator, and the documents are returned in
+ * increasing order.
+ */
+ static abstract class Iterator {
+
+ /**
+ * Returns the next document which has an update, or
+ * {@link DocIdSetIterator#NO_MORE_DOCS} if there are no more documents to
+ * return.
+ */
+ abstract int nextDoc();
+
+ /** Returns the current document this iterator is on. */
+ abstract int doc();
+
+ /**
+ * Returns the value of the document returned from {@link #nextDoc()}. A
+ * {@code null} value means that it was unset for this document.
+ */
+ abstract Object value();
+
+ /**
+ * Reset the iterator's state. Should be called before {@link #nextDoc()}
+ * and {@link #value()}.
+ */
+ abstract void reset();
+
+ }
+
+ static class Container {
+
+ final Map<String,NumericDocValuesFieldUpdates> numericDVUpdates = new HashMap<>();
+ final Map<String,BinaryDocValuesFieldUpdates> binaryDVUpdates = new HashMap<>();
+
+ boolean any() {
+ for (NumericDocValuesFieldUpdates updates : numericDVUpdates.values()) {
+ if (updates.any()) {
+ return true;
+ }
+ }
+ for (BinaryDocValuesFieldUpdates updates : binaryDVUpdates.values()) {
+ if (updates.any()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ int size() {
+ return numericDVUpdates.size() + binaryDVUpdates.size();
+ }
+
+ DocValuesFieldUpdates getUpdates(String field, Type type) {
+ switch (type) {
+ case NUMERIC:
+ return numericDVUpdates.get(field);
+ case BINARY:
+ return binaryDVUpdates.get(field);
+ default:
+ throw new IllegalArgumentException("unsupported type: " + type);
+ }
+ }
+
+ DocValuesFieldUpdates newUpdates(String field, Type type, int maxDoc) {
+ switch (type) {
+ case NUMERIC:
+ assert numericDVUpdates.get(field) == null;
+ NumericDocValuesFieldUpdates numericUpdates = new NumericDocValuesFieldUpdates(field, maxDoc);
+ numericDVUpdates.put(field, numericUpdates);
+ return numericUpdates;
+ case BINARY:
+ assert binaryDVUpdates.get(field) == null;
+ BinaryDocValuesFieldUpdates binaryUpdates = new BinaryDocValuesFieldUpdates(field, maxDoc);
+ binaryDVUpdates.put(field, binaryUpdates);
+ return binaryUpdates;
+ default:
+ throw new IllegalArgumentException("unsupported type: " + type);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "numericDVUpdates=" + numericDVUpdates + " binaryDVUpdates=" + binaryDVUpdates;
+ }
+ }
+
+ final String field;
+ final Type type;
+
+ protected DocValuesFieldUpdates(String field, Type type) {
+ this.field = field;
+ this.type = type;
+ }
+
+ /**
+ * Add an update to a document. For unsetting a value you should pass
+ * {@code null}.
+ */
+ public abstract void add(int doc, Object value);
+
+ /**
+ * Returns an {@link Iterator} over the updated documents and their
+ * values.
+ */
+ public abstract Iterator iterator();
+
+ /**
+ * Merge with another {@link DocValuesFieldUpdates}. This is called for a
+ * segment which received updates while it was being merged. The given updates
+ * should override whatever updates are in that instance.
+ */
+ public abstract void merge(DocValuesFieldUpdates other);
+
+ /** Returns true if this instance contains any updates.
+ * @return TODO*/
+ public abstract boolean any();
+
+}
Added: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java?rev=1578784&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java (added)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java Tue Mar 18 08:56:12 2014
@@ -0,0 +1,113 @@
+package org.apache.lucene.index;
+
+import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
+import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_CHAR;
+import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_INT;
+import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_HEADER;
+import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** An in-place update to a DocValues field. */
+abstract class DocValuesUpdate {
+
+ /* Rough logic: OBJ_HEADER + 3*PTR + INT
+ * Term: OBJ_HEADER + 2*PTR
+ * Term.field: 2*OBJ_HEADER + 4*INT + PTR + string.length*CHAR
+ * Term.bytes: 2*OBJ_HEADER + 2*INT + PTR + bytes.length
+ * String: 2*OBJ_HEADER + 4*INT + PTR + string.length*CHAR
+ * T: OBJ_HEADER
+ */
+ private static final int RAW_SIZE_IN_BYTES = 8*NUM_BYTES_OBJECT_HEADER + 8*NUM_BYTES_OBJECT_REF + 8*NUM_BYTES_INT;
+
+ final DocValuesFieldUpdates.Type type;
+ final Term term;
+ final String field;
+ final Object value;
+ int docIDUpto = -1; // unassigned until applied, and confusing that it's here, when it's just used in BufferedDeletes...
+
+ /**
+ * Constructor.
+ *
+ * @param term the {@link Term} which determines the documents that will be updated
+ * @param field the {@link NumericDocValuesField} to update
+ * @param value the updated value
+ */
+ protected DocValuesUpdate(DocValuesFieldUpdates.Type type, Term term, String field, Object value) {
+ this.type = type;
+ this.term = term;
+ this.field = field;
+ this.value = value;
+ }
+
+ abstract long valueSizeInBytes();
+
+ final int sizeInBytes() {
+ int sizeInBytes = RAW_SIZE_IN_BYTES;
+ sizeInBytes += term.field.length() * NUM_BYTES_CHAR;
+ sizeInBytes += term.bytes.bytes.length;
+ sizeInBytes += field.length() * NUM_BYTES_CHAR;
+ sizeInBytes += valueSizeInBytes();
+ return sizeInBytes;
+ }
+
+ @Override
+ public String toString() {
+ return "term=" + term + ",field=" + field + ",value=" + value;
+ }
+
+ /** An in-place update to a binary DocValues field */
+ static final class BinaryDocValuesUpdate extends DocValuesUpdate {
+
+ /* Size of BytesRef: 2*INT + ARRAY_HEADER + PTR */
+ private static final long RAW_VALUE_SIZE_IN_BYTES = NUM_BYTES_ARRAY_HEADER + 2*NUM_BYTES_INT + NUM_BYTES_OBJECT_REF;
+
+ static final BytesRef MISSING = new BytesRef();
+
+ BinaryDocValuesUpdate(Term term, String field, BytesRef value) {
+ super(DocValuesFieldUpdates.Type.BINARY, term, field, value == null ? MISSING : value);
+ }
+
+ @Override
+ long valueSizeInBytes() {
+ return RAW_VALUE_SIZE_IN_BYTES + ((BytesRef) value).bytes.length;
+ }
+
+ }
+
+ /** An in-place update to a numeric DocValues field */
+ static final class NumericDocValuesUpdate extends DocValuesUpdate {
+
+ static final Long MISSING = new Long(0);
+
+ NumericDocValuesUpdate(Term term, String field, Long value) {
+ super(DocValuesFieldUpdates.Type.NUMERIC, term, field, value == null ? MISSING : value);
+ }
+
+ @Override
+ long valueSizeInBytes() {
+ return RamUsageEstimator.NUM_BYTES_LONG;
+ }
+
+ }
+
+}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1578784&r1=1578783&r2=1578784&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java Tue Mar 18 08:56:12 2014
@@ -29,10 +29,13 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.index.DocumentsWriterFlushQueue.SegmentFlushTicket;
import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment;
import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState;
+import org.apache.lucene.index.DocValuesUpdate.NumericDocValuesUpdate;
+import org.apache.lucene.index.DocValuesUpdate.BinaryDocValuesUpdate;
import org.apache.lucene.index.IndexWriter.Event;
import org.apache.lucene.search.Query;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.InfoStream;
/**
@@ -160,7 +163,14 @@ final class DocumentsWriter {
synchronized boolean updateNumericDocValue(Term term, String field, Long value) throws IOException {
final DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
- deleteQueue.addNumericUpdate(new NumericUpdate(term, field, value));
+ deleteQueue.addNumericUpdate(new NumericDocValuesUpdate(term, field, value));
+ flushControl.doOnDelete();
+ return applyAllDeletes(deleteQueue);
+ }
+
+ synchronized boolean updateBinaryDocValue(Term term, String field, BytesRef value) throws IOException {
+ final DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
+ deleteQueue.addBinaryUpdate(new BinaryDocValuesUpdate(term, field, value));
flushControl.doOnDelete();
return applyAllDeletes(deleteQueue);
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java?rev=1578784&r1=1578783&r2=1578784&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java Tue Mar 18 08:56:12 2014
@@ -21,6 +21,8 @@ import java.util.Arrays;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
import java.util.concurrent.locks.ReentrantLock;
+import org.apache.lucene.index.DocValuesUpdate.BinaryDocValuesUpdate;
+import org.apache.lucene.index.DocValuesUpdate.NumericDocValuesUpdate;
import org.apache.lucene.search.Query;
/**
@@ -107,10 +109,15 @@ final class DocumentsWriterDeleteQueue {
tryApplyGlobalSlice();
}
- void addNumericUpdate(NumericUpdate update) {
+ void addNumericUpdate(NumericDocValuesUpdate update) {
add(new NumericUpdateNode(update));
tryApplyGlobalSlice();
}
+
+ void addBinaryUpdate(BinaryDocValuesUpdate update) {
+ add(new BinaryUpdateNode(update));
+ tryApplyGlobalSlice();
+ }
/**
* invariant for document update
@@ -385,9 +392,9 @@ final class DocumentsWriterDeleteQueue {
}
}
- private static final class NumericUpdateNode extends Node<NumericUpdate> {
+ private static final class NumericUpdateNode extends Node<NumericDocValuesUpdate> {
- NumericUpdateNode(NumericUpdate update) {
+ NumericUpdateNode(NumericDocValuesUpdate update) {
super(update);
}
@@ -401,6 +408,23 @@ final class DocumentsWriterDeleteQueue {
return "update=" + item;
}
}
+
+ private static final class BinaryUpdateNode extends Node<BinaryDocValuesUpdate> {
+
+ BinaryUpdateNode(BinaryDocValuesUpdate update) {
+ super(update);
+ }
+
+ @Override
+ void apply(BufferedUpdates bufferedUpdates, int docIDUpto) {
+ bufferedUpdates.addBinaryUpdate(item, docIDUpto);
+ }
+
+ @Override
+ public String toString() {
+ return "update=" + item;
+ }
+ }
private boolean forceApplyGlobalSlice() {
globalBufferLock.lock();
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1578784&r1=1578783&r2=1578784&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Tue Mar 18 08:56:12 2014
@@ -480,7 +480,7 @@ class DocumentsWriterPerThread {
}
final BufferedUpdates segmentDeletes;
- if (pendingUpdates.queries.isEmpty() && pendingUpdates.numericUpdates.isEmpty()) {
+ if (pendingUpdates.queries.isEmpty() && pendingUpdates.numericUpdates.isEmpty() && pendingUpdates.binaryUpdates.isEmpty()) {
pendingUpdates.clear();
segmentDeletes = null;
} else {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java?rev=1578784&r1=1578783&r2=1578784&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java Tue Mar 18 08:56:12 2014
@@ -24,6 +24,8 @@ import java.util.List;
import java.util.Map;
import org.apache.lucene.index.BufferedUpdatesStream.QueryAndLimit;
+import org.apache.lucene.index.DocValuesUpdate.BinaryDocValuesUpdate;
+import org.apache.lucene.index.DocValuesUpdate.NumericDocValuesUpdate;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.RamUsageEstimator;
@@ -48,7 +50,10 @@ class FrozenBufferedUpdates {
final int[] queryLimits;
// numeric DV update term and their updates
- final NumericUpdate[] updates;
+ final NumericDocValuesUpdate[] numericDVUpdates;
+
+ // binary DV update term and their updates
+ final BinaryDocValuesUpdate[] binaryDVUpdates;
final int bytesUsed;
final int numTermDeletes;
@@ -83,17 +88,34 @@ class FrozenBufferedUpdates {
// so that it maps to all fields it affects, sorted by their docUpto, and traverse
// that Term only once, applying the update to all fields that still need to be
// updated.
- List<NumericUpdate> allUpdates = new ArrayList<>();
+ List<NumericDocValuesUpdate> allNumericUpdates = new ArrayList<>();
int numericUpdatesSize = 0;
- for (LinkedHashMap<Term,NumericUpdate> fieldUpdates : deletes.numericUpdates.values()) {
- for (NumericUpdate update : fieldUpdates.values()) {
- allUpdates.add(update);
+ for (LinkedHashMap<Term,NumericDocValuesUpdate> numericUpdates : deletes.numericUpdates.values()) {
+ for (NumericDocValuesUpdate update : numericUpdates.values()) {
+ allNumericUpdates.add(update);
numericUpdatesSize += update.sizeInBytes();
}
}
- updates = allUpdates.toArray(new NumericUpdate[allUpdates.size()]);
+ numericDVUpdates = allNumericUpdates.toArray(new NumericDocValuesUpdate[allNumericUpdates.size()]);
+
+ // TODO if a Term affects multiple fields, we could keep the updates key'd by Term
+ // so that it maps to all fields it affects, sorted by their docUpto, and traverse
+ // that Term only once, applying the update to all fields that still need to be
+ // updated.
+ List<BinaryDocValuesUpdate> allBinaryUpdates = new ArrayList<>();
+ int binaryUpdatesSize = 0;
+ for (LinkedHashMap<Term,BinaryDocValuesUpdate> binaryUpdates : deletes.binaryUpdates.values()) {
+ for (BinaryDocValuesUpdate update : binaryUpdates.values()) {
+ allBinaryUpdates.add(update);
+ binaryUpdatesSize += update.sizeInBytes();
+ }
+ }
+ binaryDVUpdates = allBinaryUpdates.toArray(new BinaryDocValuesUpdate[allBinaryUpdates.size()]);
+
+ bytesUsed = (int) terms.getSizeInBytes() + queries.length * BYTES_PER_DEL_QUERY
+ + numericUpdatesSize + numericDVUpdates.length * RamUsageEstimator.NUM_BYTES_OBJECT_REF
+ + binaryUpdatesSize + binaryDVUpdates.length * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
- bytesUsed = (int) terms.getSizeInBytes() + queries.length * BYTES_PER_DEL_QUERY + numericUpdatesSize + updates.length * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
numTermDeletes = deletes.numTermDeletes.get();
}
@@ -161,6 +183,6 @@ class FrozenBufferedUpdates {
}
boolean any() {
- return termCount > 0 || queries.length > 0 || updates.length > 0;
+ return termCount > 0 || queries.length > 0 || numericDVUpdates.length > 0 || binaryDVUpdates.length > 0;
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1578784&r1=1578783&r2=1578784&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Tue Mar 18 08:56:12 2014
@@ -17,13 +17,30 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfos.FieldNumbers;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.MergeState.CheckAbort;
-import org.apache.lucene.index.NumericFieldUpdates.UpdatesIterator;
import org.apache.lucene.search.Query;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.CompoundFileDirectory;
@@ -34,29 +51,12 @@ import org.apache.lucene.store.LockObtai
import org.apache.lucene.store.MergeInfo;
import org.apache.lucene.store.TrackingDirectoryWrapper;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.ThreadInterruptedException;
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
/**
An <code>IndexWriter</code> creates and maintains an index.
@@ -1542,10 +1542,11 @@ public class IndexWriter implements Clos
}
/**
- * Updates a document's NumericDocValue for <code>field</code> to the given
- * <code>value</code>. This method can be used to 'unset' a document's value
- * by passing {@code null} as the new value. Also, you can only update fields
- * that already exist in the index, not add new fields through this method.
+ * Updates a document's {@link NumericDocValues} for <code>field</code> to the
+ * given <code>value</code>. This method can be used to 'unset' a document's
+ * value by passing {@code null} as the new value. Also, you can only update
+ * fields that already exist in the index, not add new fields through this
+ * method.
*
* <p>
* <b>NOTE</b>: if this method hits an OutOfMemoryError you should immediately
@@ -1555,7 +1556,7 @@ public class IndexWriter implements Clos
* @param term
* the term to identify the document(s) to be updated
* @param field
- * field name of the NumericDocValues field
+ * field name of the {@link NumericDocValues} field
* @param value
* new value for the field
* @throws CorruptIndexException
@@ -1577,6 +1578,47 @@ public class IndexWriter implements Clos
}
}
+ /**
+ * Updates a document's {@link BinaryDocValues} for <code>field</code> to the
+ * given <code>value</code>. This method can be used to 'unset' a document's
+ * value by passing {@code null} as the new value. Also, you can only update
+ * fields that already exist in the index, not add new fields through this
+ * method.
+ *
+ * <p>
+ * <b>NOTE:</b> this method currently replaces the existing value of all
+ * affected documents with the new value.
+ *
+ * <p>
+ * <b>NOTE:</b> if this method hits an OutOfMemoryError you should immediately
+ * close the writer. See <a href="#OOME">above</a> for details.
+ * </p>
+ *
+ * @param term
+ * the term to identify the document(s) to be updated
+ * @param field
+ * field name of the {@link BinaryDocValues} field
+ * @param value
+ * new value for the field
+ * @throws CorruptIndexException
+ * if the index is corrupt
+ * @throws IOException
+ * if there is a low-level IO error
+ */
+ public void updateBinaryDocValue(Term term, String field, BytesRef value) throws IOException {
+ ensureOpen();
+ if (!globalFieldNumberMap.contains(field, DocValuesType.BINARY)) {
+ throw new IllegalArgumentException("can only update existing binary-docvalues fields!");
+ }
+ try {
+ if (docWriter.updateBinaryDocValue(term, field, value)) {
+ processEvents(true, false);
+ }
+ } catch (OutOfMemoryError oom) {
+ handleOOM(oom, "updateBinaryDocValue");
+ }
+ }
+
// for test purpose
final synchronized int getSegmentCount(){
return segmentInfos.size();
@@ -3162,13 +3204,13 @@ public class IndexWriter implements Clos
return docMap;
}
- private void skipDeletedDoc(UpdatesIterator[] updatesIters, int deletedDoc) {
- for (UpdatesIterator iter : updatesIters) {
+ private void skipDeletedDoc(DocValuesFieldUpdates.Iterator[] updatesIters, int deletedDoc) {
+ for (DocValuesFieldUpdates.Iterator iter : updatesIters) {
if (iter.doc() == deletedDoc) {
iter.nextDoc();
}
// when entering the method, all iterators must already be beyond the
- // deleted document, or right on it, in which case we advance them above
+ // deleted document, or right on it, in which case we advance them over
// and they must be beyond it now.
assert iter.doc() > deletedDoc : "updateDoc=" + iter.doc() + " deletedDoc=" + deletedDoc;
}
@@ -3203,7 +3245,7 @@ public class IndexWriter implements Clos
ReadersAndUpdates mergedDeletesAndUpdates = null;
boolean initWritableLiveDocs = false;
MergePolicy.DocMap docMap = null;
- final Map<String,NumericFieldUpdates> mergedFieldUpdates = new HashMap<>();
+ final DocValuesFieldUpdates.Container mergedDVUpdates = new DocValuesFieldUpdates.Container();
for (int i = 0; i < sourceSegments.size(); i++) {
SegmentCommitInfo info = sourceSegments.get(i);
@@ -3214,19 +3256,28 @@ public class IndexWriter implements Clos
// We hold a ref so it should still be in the pool:
assert rld != null: "seg=" + info.info.name;
final Bits currentLiveDocs = rld.getLiveDocs();
- final Map<String,NumericFieldUpdates> mergingFieldUpdates = rld.getMergingFieldUpdates();
+ final Map<String,DocValuesFieldUpdates> mergingFieldUpdates = rld.getMergingFieldUpdates();
final String[] mergingFields;
- final UpdatesIterator[] updatesIters;
+ final DocValuesFieldUpdates[] dvFieldUpdates;
+ final DocValuesFieldUpdates.Iterator[] updatesIters;
if (mergingFieldUpdates.isEmpty()) {
mergingFields = null;
updatesIters = null;
+ dvFieldUpdates = null;
} else {
mergingFields = new String[mergingFieldUpdates.size()];
- updatesIters = new UpdatesIterator[mergingFieldUpdates.size()];
+ dvFieldUpdates = new DocValuesFieldUpdates[mergingFieldUpdates.size()];
+ updatesIters = new DocValuesFieldUpdates.Iterator[mergingFieldUpdates.size()];
int idx = 0;
- for (Entry<String,NumericFieldUpdates> e : mergingFieldUpdates.entrySet()) {
- mergingFields[idx] = e.getKey();
- updatesIters[idx] = e.getValue().getUpdates();
+ for (Entry<String,DocValuesFieldUpdates> e : mergingFieldUpdates.entrySet()) {
+ String field = e.getKey();
+ DocValuesFieldUpdates updates = e.getValue();
+ mergingFields[idx] = field;
+ dvFieldUpdates[idx] = mergedDVUpdates.getUpdates(field, updates.type);
+ if (dvFieldUpdates[idx] == null) {
+ dvFieldUpdates[idx] = mergedDVUpdates.newUpdates(field, updates.type, mergeState.segmentInfo.getDocCount());
+ }
+ updatesIters[idx] = updates.iterator();
updatesIters[idx].nextDoc(); // advance to first update doc
++idx;
}
@@ -3279,7 +3330,7 @@ public class IndexWriter implements Clos
// document isn't deleted, check if any of the fields have an update to it
int newDoc = -1;
for (int idx = 0; idx < mergingFields.length; idx++) {
- UpdatesIterator updatesIter = updatesIters[idx];
+ DocValuesFieldUpdates.Iterator updatesIter = updatesIters[idx];
if (updatesIter.doc() == j) { // document has an update
if (mergedDeletesAndUpdates == null) {
mergedDeletesAndUpdates = readerPool.get(merge.info, true);
@@ -3288,14 +3339,8 @@ public class IndexWriter implements Clos
if (newDoc == -1) { // map once per all field updates, but only if there are any updates
newDoc = docMap.map(docUpto);
}
- String field = mergingFields[idx];
- NumericFieldUpdates fieldUpdates = mergedFieldUpdates.get(field);
- if (fieldUpdates == null) {
- // an approximantion of maxDoc, used to compute best bitsPerValue
- fieldUpdates = new NumericFieldUpdates.PackedNumericFieldUpdates(mergeState.segmentInfo.getDocCount());
- mergedFieldUpdates.put(field, fieldUpdates);
- }
- fieldUpdates.add(newDoc, updatesIter.value() == null ? NumericUpdate.MISSING : updatesIter.value());
+ DocValuesFieldUpdates dvUpdates = dvFieldUpdates[idx];
+ dvUpdates.add(newDoc, updatesIter.value());
updatesIter.nextDoc(); // advance to next document
} else {
assert updatesIter.doc() > j : "updateDoc=" + updatesIter.doc() + " curDoc=" + j;
@@ -3312,7 +3357,7 @@ public class IndexWriter implements Clos
// document isn't deleted, check if any of the fields have an update to it
int newDoc = -1;
for (int idx = 0; idx < mergingFields.length; idx++) {
- UpdatesIterator updatesIter = updatesIters[idx];
+ DocValuesFieldUpdates.Iterator updatesIter = updatesIters[idx];
if (updatesIter.doc() == j) { // document has an update
if (mergedDeletesAndUpdates == null) {
mergedDeletesAndUpdates = readerPool.get(merge.info, true);
@@ -3321,14 +3366,8 @@ public class IndexWriter implements Clos
if (newDoc == -1) { // map once per all field updates, but only if there are any updates
newDoc = docMap.map(docUpto);
}
- String field = mergingFields[idx];
- NumericFieldUpdates fieldUpdates = mergedFieldUpdates.get(field);
- if (fieldUpdates == null) {
- // an approximantion of maxDoc, used to compute best bitsPerValue
- fieldUpdates = new NumericFieldUpdates.PackedNumericFieldUpdates(mergeState.segmentInfo.getDocCount());
- mergedFieldUpdates.put(field, fieldUpdates);
- }
- fieldUpdates.add(newDoc, updatesIter.value() == null ? NumericUpdate.MISSING : updatesIter.value());
+ DocValuesFieldUpdates dvUpdates = dvFieldUpdates[idx];
+ dvUpdates.add(newDoc, updatesIter.value());
updatesIter.nextDoc(); // advance to next document
} else {
assert updatesIter.doc() > j : "updateDoc=" + updatesIter.doc() + " curDoc=" + j;
@@ -3367,7 +3406,7 @@ public class IndexWriter implements Clos
// document isn't deleted, check if any of the fields have an update to it
int newDoc = -1;
for (int idx = 0; idx < mergingFields.length; idx++) {
- UpdatesIterator updatesIter = updatesIters[idx];
+ DocValuesFieldUpdates.Iterator updatesIter = updatesIters[idx];
if (updatesIter.doc() == j) { // document has an update
if (mergedDeletesAndUpdates == null) {
mergedDeletesAndUpdates = readerPool.get(merge.info, true);
@@ -3376,14 +3415,8 @@ public class IndexWriter implements Clos
if (newDoc == -1) { // map once per all field updates, but only if there are any updates
newDoc = docMap.map(docUpto);
}
- String field = mergingFields[idx];
- NumericFieldUpdates fieldUpdates = mergedFieldUpdates.get(field);
- if (fieldUpdates == null) {
- // an approximantion of maxDoc, used to compute best bitsPerValue
- fieldUpdates = new NumericFieldUpdates.PackedNumericFieldUpdates(mergeState.segmentInfo.getDocCount());
- mergedFieldUpdates.put(field, fieldUpdates);
- }
- fieldUpdates.add(newDoc, updatesIter.value() == null ? NumericUpdate.MISSING : updatesIter.value());
+ DocValuesFieldUpdates dvUpdates = dvFieldUpdates[idx];
+ dvUpdates.add(newDoc, updatesIter.value());
updatesIter.nextDoc(); // advance to next document
} else {
assert updatesIter.doc() > j : "field=" + mergingFields[idx] + " updateDoc=" + updatesIter.doc() + " curDoc=" + j;
@@ -3397,7 +3430,7 @@ public class IndexWriter implements Clos
for (int j = 0; j < docCount; j++) {
int newDoc = -1;
for (int idx = 0; idx < mergingFields.length; idx++) {
- UpdatesIterator updatesIter = updatesIters[idx];
+ DocValuesFieldUpdates.Iterator updatesIter = updatesIters[idx];
if (updatesIter.doc() == j) { // document has an update
if (mergedDeletesAndUpdates == null) {
mergedDeletesAndUpdates = readerPool.get(merge.info, true);
@@ -3406,14 +3439,8 @@ public class IndexWriter implements Clos
if (newDoc == -1) { // map once per all field updates, but only if there are any updates
newDoc = docMap.map(docUpto);
}
- String field = mergingFields[idx];
- NumericFieldUpdates fieldUpdates = mergedFieldUpdates.get(field);
- if (fieldUpdates == null) {
- // an approximantion of maxDoc, used to compute best bitsPerValue
- fieldUpdates = new NumericFieldUpdates.PackedNumericFieldUpdates(mergeState.segmentInfo.getDocCount());
- mergedFieldUpdates.put(field, fieldUpdates);
- }
- fieldUpdates.add(newDoc, updatesIter.value() == null ? NumericUpdate.MISSING : updatesIter.value());
+ DocValuesFieldUpdates dvUpdates = dvFieldUpdates[idx];
+ dvUpdates.add(newDoc, updatesIter.value());
updatesIter.nextDoc(); // advance to next document
} else {
assert updatesIter.doc() > j : "updateDoc=" + updatesIter.doc() + " curDoc=" + j;
@@ -3430,7 +3457,7 @@ public class IndexWriter implements Clos
assert docUpto == merge.info.info.getDocCount();
- if (!mergedFieldUpdates.isEmpty()) {
+ if (mergedDVUpdates.any()) {
// System.out.println("[" + Thread.currentThread().getName() + "] IW.commitMergedDeletes: mergedDeletes.info=" + mergedDeletes.info + ", mergedFieldUpdates=" + mergedFieldUpdates);
boolean success = false;
try {
@@ -3440,7 +3467,7 @@ public class IndexWriter implements Clos
// NOTE: currently this is the only place which throws a true
// IOException. If this ever changes, we need to extend that try/finally
// block to the rest of the method too.
- mergedDeletesAndUpdates.writeFieldUpdates(directory, mergedFieldUpdates);
+ mergedDeletesAndUpdates.writeFieldUpdates(directory, mergedDVUpdates);
success = true;
} finally {
if (!success) {
@@ -3455,8 +3482,8 @@ public class IndexWriter implements Clos
infoStream.message("IW", "no new deletes or field updates since merge started");
} else {
String msg = mergedDeletesAndUpdates.getPendingDeleteCount() + " new deletes";
- if (!mergedFieldUpdates.isEmpty()) {
- msg += " and " + mergedFieldUpdates.size() + " new field updates";
+ if (mergedDVUpdates.any()) {
+ msg += " and " + mergedDVUpdates.size() + " new field updates";
}
msg += " since merge started";
infoStream.message("IW", msg);
Added: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java?rev=1578784&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java (added)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java Tue Mar 18 08:56:12 2014
@@ -0,0 +1,201 @@
+package org.apache.lucene.index;
+
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.index.DocValuesUpdate.NumericDocValuesUpdate;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.InPlaceMergeSorter;
+import org.apache.lucene.util.packed.PackedInts;
+import org.apache.lucene.util.packed.PagedGrowableWriter;
+import org.apache.lucene.util.packed.PagedMutable;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A {@link DocValuesFieldUpdates} which holds updates of documents, of a single
+ * {@link NumericDocValuesField}.
+ *
+ * @lucene.experimental
+ */
+class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
+
+ final static class Iterator extends DocValuesFieldUpdates.Iterator {
+ private final int size;
+ private final PagedGrowableWriter values;
+ private final FixedBitSet docsWithField;
+ private final PagedMutable docs;
+ private long idx = 0; // long so we don't overflow if size == Integer.MAX_VALUE
+ private int doc = -1;
+ private Long value = null;
+
+ Iterator(int size, PagedGrowableWriter values, FixedBitSet docsWithField, PagedMutable docs) {
+ this.size = size;
+ this.values = values;
+ this.docsWithField = docsWithField;
+ this.docs = docs;
+ }
+
+ @Override
+ Long value() {
+ return value;
+ }
+
+ @Override
+ int nextDoc() {
+ if (idx >= size) {
+ value = null;
+ return doc = DocIdSetIterator.NO_MORE_DOCS;
+ }
+ doc = (int) docs.get(idx);
+ ++idx;
+ while (idx < size && docs.get(idx) == doc) {
+ ++idx;
+ }
+ if (!docsWithField.get((int) (idx - 1))) {
+ value = null;
+ } else {
+ // idx points to the "next" element
+ value = Long.valueOf(values.get(idx - 1));
+ }
+ return doc;
+ }
+
+ @Override
+ int doc() {
+ return doc;
+ }
+
+ @Override
+ void reset() {
+ doc = -1;
+ value = null;
+ idx = 0;
+ }
+ }
+
+ private FixedBitSet docsWithField;
+ private PagedMutable docs;
+ private PagedGrowableWriter values;
+ private int size;
+
+ public NumericDocValuesFieldUpdates(String field, int maxDoc) {
+ super(field, Type.NUMERIC);
+ docsWithField = new FixedBitSet(64);
+ docs = new PagedMutable(1, 1024, PackedInts.bitsRequired(maxDoc - 1), PackedInts.COMPACT);
+ values = new PagedGrowableWriter(1, 1024, 1, PackedInts.FAST);
+ size = 0;
+ }
+
+ @Override
+ public void add(int doc, Object value) {
+ // TODO: if the Sorter interface changes to take long indexes, we can remove that limitation
+ if (size == Integer.MAX_VALUE) {
+ throw new IllegalStateException("cannot support more than Integer.MAX_VALUE doc/value entries");
+ }
+
+ Long val = (Long) value;
+ if (val == null) {
+ val = NumericDocValuesUpdate.MISSING;
+ }
+
+ // grow the structures to have room for more elements
+ if (docs.size() == size) {
+ docs = docs.grow(size + 1);
+ values = values.grow(size + 1);
+ docsWithField = FixedBitSet.ensureCapacity(docsWithField, (int) docs.size());
+ }
+
+ if (val != NumericDocValuesUpdate.MISSING) {
+ // only mark the document as having a value in that field if the value wasn't set to null (MISSING)
+ docsWithField.set(size);
+ }
+
+ docs.set(size, doc);
+ values.set(size, val.longValue());
+ ++size;
+ }
+
+ @Override
+ public Iterator iterator() {
+ final PagedMutable docs = this.docs;
+ final PagedGrowableWriter values = this.values;
+ final FixedBitSet docsWithField = this.docsWithField;
+ new InPlaceMergeSorter() {
+ @Override
+ protected void swap(int i, int j) {
+ long tmpDoc = docs.get(j);
+ docs.set(j, docs.get(i));
+ docs.set(i, tmpDoc);
+
+ long tmpVal = values.get(j);
+ values.set(j, values.get(i));
+ values.set(i, tmpVal);
+
+ boolean tmpBool = docsWithField.get(j);
+ if (docsWithField.get(i)) {
+ docsWithField.set(j);
+ } else {
+ docsWithField.clear(j);
+ }
+ if (tmpBool) {
+ docsWithField.set(i);
+ } else {
+ docsWithField.clear(i);
+ }
+ }
+
+ @Override
+ protected int compare(int i, int j) {
+ int x = (int) docs.get(i);
+ int y = (int) docs.get(j);
+ return (x < y) ? -1 : ((x == y) ? 0 : 1);
+ }
+ }.sort(0, size);
+
+ return new Iterator(size, values, docsWithField, docs);
+ }
+
+ @Override
+ public void merge(DocValuesFieldUpdates other) {
+ assert other instanceof NumericDocValuesFieldUpdates;
+ NumericDocValuesFieldUpdates otherUpdates = (NumericDocValuesFieldUpdates) other;
+ if (size + otherUpdates.size > Integer.MAX_VALUE) {
+ throw new IllegalStateException(
+ "cannot support more than Integer.MAX_VALUE doc/value entries; size="
+ + size + " other.size=" + otherUpdates.size);
+ }
+ docs = docs.grow(size + otherUpdates.size);
+ values = values.grow(size + otherUpdates.size);
+ docsWithField = FixedBitSet.ensureCapacity(docsWithField, (int) docs.size());
+ for (int i = 0; i < otherUpdates.size; i++) {
+ int doc = (int) otherUpdates.docs.get(i);
+ if (otherUpdates.docsWithField.get(i)) {
+ docsWithField.set(size);
+ }
+ docs.set(size, doc);
+ values.set(size, otherUpdates.values.get(i));
+ ++size;
+ }
+ }
+
+ @Override
+ public boolean any() {
+ return size > 0;
+ }
+
+}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java?rev=1578784&r1=1578783&r2=1578784&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java Tue Mar 18 08:56:12 2014
@@ -30,12 +30,13 @@ import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.LiveDocsFormat;
+import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.index.NumericFieldUpdates.UpdatesIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.TrackingDirectoryWrapper;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.MutableBits;
@@ -78,7 +79,7 @@ class ReadersAndUpdates {
// updates on the merged segment too.
private boolean isMerging = false;
- private final Map<String,NumericFieldUpdates> mergingNumericUpdates = new HashMap<>();
+ private final Map<String,DocValuesFieldUpdates> mergingDVUpdates = new HashMap<>();
public ReadersAndUpdates(IndexWriter writer, SegmentCommitInfo info) {
this.info = info;
@@ -294,11 +295,11 @@ class ReadersAndUpdates {
}
// Writes field updates (new _X_N updates files) to the directory
- public synchronized void writeFieldUpdates(Directory dir, Map<String,NumericFieldUpdates> numericFieldUpdates) throws IOException {
+ public synchronized void writeFieldUpdates(Directory dir, DocValuesFieldUpdates.Container dvUpdates) throws IOException {
assert Thread.holdsLock(writer);
//System.out.println("rld.writeFieldUpdates: seg=" + info + " numericFieldUpdates=" + numericFieldUpdates);
- assert numericFieldUpdates != null && !numericFieldUpdates.isEmpty();
+ assert dvUpdates.any();
// Do this so we can delete any created files on
// exception; this saves all codecs from having to do
@@ -330,9 +331,13 @@ class ReadersAndUpdates {
clone.setDocValuesGen(fi.getDocValuesGen());
}
// create new fields or update existing ones to have NumericDV type
- for (String f : numericFieldUpdates.keySet()) {
+ for (String f : dvUpdates.numericDVUpdates.keySet()) {
builder.addOrUpdate(f, NumericDocValuesField.TYPE);
}
+ // create new fields or update existing ones to have BinaryDV type
+ for (String f : dvUpdates.binaryDVUpdates.keySet()) {
+ builder.addOrUpdate(f, BinaryDocValuesField.TYPE);
+ }
fieldInfos = builder.finish();
final long nextFieldInfosGen = info.getNextFieldInfosGen();
@@ -342,10 +347,10 @@ class ReadersAndUpdates {
final DocValuesConsumer fieldsConsumer = docValuesFormat.fieldsConsumer(state);
boolean fieldsConsumerSuccess = false;
try {
-// System.out.println("[" + Thread.currentThread().getName() + "] RLD.writeLiveDocs: applying updates; seg=" + info + " updates=" + numericUpdates);
- for (Entry<String,NumericFieldUpdates> e : numericFieldUpdates.entrySet()) {
+// System.out.println("[" + Thread.currentThread().getName() + "] RLD.writeFieldUpdates: applying numeric updates; seg=" + info + " updates=" + numericFieldUpdates);
+ for (Entry<String,NumericDocValuesFieldUpdates> e : dvUpdates.numericDVUpdates.entrySet()) {
final String field = e.getKey();
- final NumericFieldUpdates fieldUpdates = e.getValue();
+ final NumericDocValuesFieldUpdates fieldUpdates = e.getValue();
final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
assert fieldInfo != null;
@@ -355,7 +360,7 @@ class ReadersAndUpdates {
final NumericDocValues currentValues = reader.getNumericDocValues(field);
final Bits docsWithField = reader.getDocsWithField(field);
final int maxDoc = reader.maxDoc();
- final UpdatesIterator updatesIter = fieldUpdates.getUpdates();
+ final NumericDocValuesFieldUpdates.Iterator updatesIter = fieldUpdates.iterator();
@Override
public Iterator<Number> iterator() {
updatesIter.reset();
@@ -398,7 +403,68 @@ class ReadersAndUpdates {
}
});
}
-
+
+// System.out.println("[" + Thread.currentThread().getName() + "] RAU.writeFieldUpdates: applying binary updates; seg=" + info + " updates=" + dvUpdates.binaryDVUpdates);
+ for (Entry<String,BinaryDocValuesFieldUpdates> e : dvUpdates.binaryDVUpdates.entrySet()) {
+ final String field = e.getKey();
+ final BinaryDocValuesFieldUpdates dvFieldUpdates = e.getValue();
+ final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+ assert fieldInfo != null;
+
+// System.out.println("[" + Thread.currentThread().getName() + "] RAU.writeFieldUpdates: applying binary updates; seg=" + info + " f=" + dvFieldUpdates + ", updates=" + dvFieldUpdates);
+
+ fieldInfo.setDocValuesGen(nextFieldInfosGen);
+ // write the numeric updates to a new gen'd docvalues file
+ fieldsConsumer.addBinaryField(fieldInfo, new Iterable<BytesRef>() {
+ final BinaryDocValues currentValues = reader.getBinaryDocValues(field);
+ final Bits docsWithField = reader.getDocsWithField(field);
+ final int maxDoc = reader.maxDoc();
+ final BinaryDocValuesFieldUpdates.Iterator updatesIter = dvFieldUpdates.iterator();
+ @Override
+ public Iterator<BytesRef> iterator() {
+ updatesIter.reset();
+ return new Iterator<BytesRef>() {
+
+ int curDoc = -1;
+ int updateDoc = updatesIter.nextDoc();
+ BytesRef scratch = new BytesRef();
+
+ @Override
+ public boolean hasNext() {
+ return curDoc < maxDoc - 1;
+ }
+
+ @Override
+ public BytesRef next() {
+ if (++curDoc >= maxDoc) {
+ throw new NoSuchElementException("no more documents to return values for");
+ }
+ if (curDoc == updateDoc) { // this document has an updated value
+ BytesRef value = updatesIter.value(); // either null (unset value) or updated value
+ updateDoc = updatesIter.nextDoc(); // prepare for next round
+ return value;
+ } else {
+ // no update for this document
+ assert curDoc < updateDoc;
+ if (currentValues != null && docsWithField.get(curDoc)) {
+ // only read the current value if the document had a value before
+ currentValues.get(curDoc, scratch);
+ return scratch;
+ } else {
+ return null;
+ }
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException("this iterator does not support removing elements");
+ }
+ };
+ }
+ });
+ }
+
codec.fieldInfosFormat().getFieldInfosWriter().write(trackingDir, info.info.name, segmentSuffix, fieldInfos, IOContext.DEFAULT);
fieldsConsumerSuccess = true;
} finally {
@@ -436,12 +502,20 @@ class ReadersAndUpdates {
info.advanceFieldInfosGen();
// copy all the updates to mergingUpdates, so they can later be applied to the merged segment
if (isMerging) {
- for (Entry<String,NumericFieldUpdates> e : numericFieldUpdates.entrySet()) {
- NumericFieldUpdates fieldUpdates = mergingNumericUpdates.get(e.getKey());
- if (fieldUpdates == null) {
- mergingNumericUpdates.put(e.getKey(), e.getValue());
+ for (Entry<String,NumericDocValuesFieldUpdates> e : dvUpdates.numericDVUpdates.entrySet()) {
+ DocValuesFieldUpdates updates = mergingDVUpdates.get(e.getKey());
+ if (updates == null) {
+ mergingDVUpdates.put(e.getKey(), e.getValue());
+ } else {
+ updates.merge(e.getValue());
+ }
+ }
+ for (Entry<String,BinaryDocValuesFieldUpdates> e : dvUpdates.binaryDVUpdates.entrySet()) {
+ DocValuesFieldUpdates updates = mergingDVUpdates.get(e.getKey());
+ if (updates == null) {
+ mergingDVUpdates.put(e.getKey(), e.getValue());
} else {
- fieldUpdates.merge(e.getValue());
+ updates.merge(e.getValue());
}
}
}
@@ -502,13 +576,13 @@ class ReadersAndUpdates {
* finished merging (whether successfully or not).
*/
public synchronized void dropMergingUpdates() {
- mergingNumericUpdates.clear();
+ mergingDVUpdates.clear();
isMerging = false;
}
/** Returns updates that came in while this segment was merging. */
- public synchronized Map<String,NumericFieldUpdates> getMergingFieldUpdates() {
- return mergingNumericUpdates;
+ public synchronized Map<String,DocValuesFieldUpdates> getMergingFieldUpdates() {
+ return mergingDVUpdates;
}
@Override