You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2014/05/29 14:24:49 UTC

svn commit: r1598272 - in /lucene/dev/trunk/lucene: ./ core/src/java/org/apache/lucene/index/ core/src/test/org/apache/lucene/index/ test-framework/src/java/org/apache/lucene/index/

Author: shaie
Date: Thu May 29 12:24:48 2014
New Revision: 1598272

URL: http://svn.apache.org/r1598272
Log:
LUCENE-5680: add aotmic DocValues updates

Modified:
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
    lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1598272&r1=1598271&r2=1598272&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Thu May 29 12:24:48 2014
@@ -118,6 +118,9 @@ New Features
 * LUCENE-5675: Add IDVersionPostingsFormat, a postings format
   optimized for primary-key (ID) fields that also record a version
   (long) for each ID.  (Robert Muir, Mike McCandless)
+  
+* LUCENE-5680: Add ability to atomically update a set of DocValues
+  fields. (Shai Erera)
 
 Changes in Backwards Compatibility Policy
 

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java?rev=1598272&r1=1598271&r2=1598272&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java Thu May 29 12:24:48 2014
@@ -101,7 +101,7 @@ class BinaryDocValuesFieldUpdates extend
   private final int bitsPerValue;
   
   public BinaryDocValuesFieldUpdates(String field, int maxDoc) {
-    super(field, Type.BINARY);
+    super(field, FieldInfo.DocValuesType.BINARY);
     bitsPerValue = PackedInts.bitsRequired(maxDoc - 1);
     docs = new PagedMutable(1, PAGE_SIZE, bitsPerValue, PackedInts.COMPACT);
     offsets = new PagedGrowableWriter(1, PAGE_SIZE, 1, PackedInts.FAST);

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java?rev=1598272&r1=1598271&r2=1598272&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java Thu May 29 12:24:48 2014
@@ -33,8 +33,6 @@ abstract class DocValuesFieldUpdates {
   
   protected static final int PAGE_SIZE = 1024;
 
-  static enum Type { NUMERIC, BINARY }
-  
   /**
    * An iterator over documents and their updated values. Only documents with
    * updates are returned by this iterator, and the documents are returned in
@@ -100,7 +98,7 @@ abstract class DocValuesFieldUpdates {
       return ramBytesPerDoc;
     }
     
-    DocValuesFieldUpdates getUpdates(String field, Type type) {
+    DocValuesFieldUpdates getUpdates(String field, FieldInfo.DocValuesType type) {
       switch (type) {
         case NUMERIC:
           return numericDVUpdates.get(field);
@@ -111,7 +109,7 @@ abstract class DocValuesFieldUpdates {
       }
     }
     
-    DocValuesFieldUpdates newUpdates(String field, Type type, int maxDoc) {
+    DocValuesFieldUpdates newUpdates(String field, FieldInfo.DocValuesType type, int maxDoc) {
       switch (type) {
         case NUMERIC:
           assert numericDVUpdates.get(field) == null;
@@ -135,9 +133,9 @@ abstract class DocValuesFieldUpdates {
   }
   
   final String field;
-  final Type type;
+  final FieldInfo.DocValuesType type;
   
-  protected DocValuesFieldUpdates(String field, Type type) {
+  protected DocValuesFieldUpdates(String field, FieldInfo.DocValuesType type) {
     this.field = field;
     this.type = type;
   }

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java?rev=1598272&r1=1598271&r2=1598272&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java Thu May 29 12:24:48 2014
@@ -39,7 +39,7 @@ abstract class DocValuesUpdate {
    */
   private static final int RAW_SIZE_IN_BYTES = 8*NUM_BYTES_OBJECT_HEADER + 8*NUM_BYTES_OBJECT_REF + 8*NUM_BYTES_INT;
   
-  final DocValuesFieldUpdates.Type type;
+  final FieldInfo.DocValuesType type;
   final Term term;
   final String field;
   final Object value;
@@ -52,7 +52,7 @@ abstract class DocValuesUpdate {
    * @param field the {@link NumericDocValuesField} to update
    * @param value the updated value
    */
-  protected DocValuesUpdate(DocValuesFieldUpdates.Type type, Term term, String field, Object value) {
+  protected DocValuesUpdate(FieldInfo.DocValuesType type, Term term, String field, Object value) {
     this.type = type;
     this.term = term;
     this.field = field;
@@ -82,7 +82,7 @@ abstract class DocValuesUpdate {
     private static final long RAW_VALUE_SIZE_IN_BYTES = NUM_BYTES_ARRAY_HEADER + 2*NUM_BYTES_INT + NUM_BYTES_OBJECT_REF;
     
     BinaryDocValuesUpdate(Term term, String field, BytesRef value) {
-      super(DocValuesFieldUpdates.Type.BINARY, term, field, value);
+      super(FieldInfo.DocValuesType.BINARY, term, field, value);
     }
 
     @Override
@@ -94,9 +94,9 @@ abstract class DocValuesUpdate {
 
   /** An in-place update to a numeric DocValues field */
   static final class NumericDocValuesUpdate extends DocValuesUpdate {
-    
+
     NumericDocValuesUpdate(Term term, String field, Long value) {
-      super(DocValuesFieldUpdates.Type.NUMERIC, term, field, value);
+      super(FieldInfo.DocValuesType.NUMERIC, term, field, value);
     }
 
     @Override

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1598272&r1=1598271&r2=1598272&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java Thu May 29 12:24:48 2014
@@ -30,13 +30,10 @@ import org.apache.lucene.analysis.Analyz
 import org.apache.lucene.index.DocumentsWriterFlushQueue.SegmentFlushTicket;
 import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment;
 import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState;
-import org.apache.lucene.index.DocValuesUpdate.NumericDocValuesUpdate;
-import org.apache.lucene.index.DocValuesUpdate.BinaryDocValuesUpdate;
 import org.apache.lucene.index.IndexWriter.Event;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.InfoStream;
 
 /**
@@ -156,20 +153,13 @@ final class DocumentsWriter implements C
     return applyAllDeletes( deleteQueue);
   }
 
-  synchronized boolean updateNumericDocValue(Term term, String field, long value) throws IOException {
+  synchronized boolean updateDocValues(DocValuesUpdate... updates) throws IOException {
     final DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
-    deleteQueue.addNumericUpdate(new NumericDocValuesUpdate(term, field, Long.valueOf(value)));
+    deleteQueue.addDocValuesUpdates(updates);
     flushControl.doOnDelete();
     return applyAllDeletes(deleteQueue);
   }
   
-  synchronized boolean updateBinaryDocValue(Term term, String field, BytesRef value) throws IOException {
-    final DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
-    deleteQueue.addBinaryUpdate(new BinaryDocValuesUpdate(term, field, value));
-    flushControl.doOnDelete();
-    return applyAllDeletes(deleteQueue);
-  }
-
   DocumentsWriterDeleteQueue currentDeleteSession() {
     return deleteQueue;
   }

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java?rev=1598272&r1=1598271&r2=1598272&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java Thu May 29 12:24:48 2014
@@ -24,6 +24,7 @@ import java.util.concurrent.locks.Reentr
 import org.apache.lucene.index.DocValuesUpdate.BinaryDocValuesUpdate;
 import org.apache.lucene.index.DocValuesUpdate.NumericDocValuesUpdate;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BytesRef;
 
 /**
  * {@link DocumentsWriterDeleteQueue} is a non-blocking linked pending deletes
@@ -109,16 +110,11 @@ final class DocumentsWriterDeleteQueue {
     tryApplyGlobalSlice();
   }
 
-  void addNumericUpdate(NumericDocValuesUpdate update) {
-    add(new NumericUpdateNode(update));
+  void addDocValuesUpdates(DocValuesUpdate... updates) {
+    add(new DocValuesUpdatesNode(updates));
     tryApplyGlobalSlice();
   }
   
-  void addBinaryUpdate(BinaryDocValuesUpdate update) {
-    add(new BinaryUpdateNode(update));
-    tryApplyGlobalSlice();
-  }
-
   /**
    * invariant for document update
    */
@@ -392,40 +388,43 @@ final class DocumentsWriterDeleteQueue {
     }
   }
 
-  private static final class NumericUpdateNode extends Node<NumericDocValuesUpdate> {
+  private static final class DocValuesUpdatesNode extends Node<DocValuesUpdate[]> {
 
-    NumericUpdateNode(NumericDocValuesUpdate update) {
-      super(update);
+    DocValuesUpdatesNode(DocValuesUpdate... updates) {
+      super(updates);
     }
 
     @Override
     void apply(BufferedUpdates bufferedUpdates, int docIDUpto) {
-      bufferedUpdates.addNumericUpdate(item, docIDUpto);
+      for (DocValuesUpdate update : item) {
+        switch (update.type) {
+          case NUMERIC:
+            bufferedUpdates.addNumericUpdate(new NumericDocValuesUpdate(update.term, update.field, (Long) update.value), docIDUpto);
+            break;
+          case BINARY:
+            bufferedUpdates.addBinaryUpdate(new BinaryDocValuesUpdate(update.term, update.field, (BytesRef) update.value), docIDUpto);
+            break;
+          default:
+            throw new IllegalArgumentException(update.type + " DocValues updates not supported yet!");
+        }
+      }
     }
 
     @Override
     public String toString() {
-      return "update=" + item;
+      StringBuilder sb = new StringBuilder();
+      sb.append("docValuesUpdates: ");
+      if (item.length > 0) {
+        sb.append("term=").append(item[0].term).append("; updates: [");
+        for (DocValuesUpdate update : item) {
+          sb.append(update.field).append(':').append(update.value).append(',');
+        }
+        sb.setCharAt(sb.length()-1, ']');
+      }
+      return sb.toString();
     }
   }
   
-  private static final class BinaryUpdateNode extends Node<BinaryDocValuesUpdate> {
-    
-    BinaryUpdateNode(BinaryDocValuesUpdate update) {
-      super(update);
-    }
-    
-    @Override
-    void apply(BufferedUpdates bufferedUpdates, int docIDUpto) {
-      bufferedUpdates.addBinaryUpdate(item, docIDUpto);
-    }
-    
-    @Override
-    public String toString() {
-      return "update=" + item;
-    }
-  }
-
   private boolean forceApplyGlobalSlice() {
     globalBufferLock.lock();
     final Node<?> currentTail = tail;

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1598272&r1=1598271&r2=1598272&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Thu May 29 12:24:48 2014
@@ -32,14 +32,17 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map.Entry;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Queue;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.DocValuesUpdate.BinaryDocValuesUpdate;
+import org.apache.lucene.index.DocValuesUpdate.NumericDocValuesUpdate;
 import org.apache.lucene.index.FieldInfo.DocValuesType;
 import org.apache.lucene.index.FieldInfos.FieldNumbers;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@@ -1429,7 +1432,7 @@ public class IndexWriter implements Clos
       throw new IllegalArgumentException("can only update existing numeric-docvalues fields!");
     }
     try {
-      if (docWriter.updateNumericDocValue(term, field, value)) {
+      if (docWriter.updateDocValues(new NumericDocValuesUpdate(term, field, value))) {
         processEvents(true, false);
       }
     } catch (OutOfMemoryError oom) {
@@ -1471,14 +1474,64 @@ public class IndexWriter implements Clos
       throw new IllegalArgumentException("can only update existing binary-docvalues fields!");
     }
     try {
-      if (docWriter.updateBinaryDocValue(term, field, value)) {
+      if (docWriter.updateDocValues(new BinaryDocValuesUpdate(term, field, value))) {
         processEvents(true, false);
       }
     } catch (OutOfMemoryError oom) {
       handleOOM(oom, "updateBinaryDocValue");
     }
   }
-
+  
+  /**
+   * Updates documents' DocValues fields to the given values. Each field update
+   * is applied to the set of documents that are associated with the
+   * {@link Term} to the same value. All updates are atomically applied and
+   * flushed together.
+   * 
+   * <p>
+   * <b>NOTE</b>: if this method hits an OutOfMemoryError you should immediately
+   * close the writer. See <a href="#OOME">above</a> for details.
+   * </p>
+   * 
+   * @param updates
+   *          the updates to apply
+   * @throws CorruptIndexException
+   *           if the index is corrupt
+   * @throws IOException
+   *           if there is a low-level IO error
+   */
+  public void updateDocValues(Term term, Field... updates) throws IOException {
+    ensureOpen();
+    DocValuesUpdate[] dvUpdates = new DocValuesUpdate[updates.length];
+    for (int i = 0; i < updates.length; i++) {
+      final Field f = updates[i];
+      final DocValuesType dvType = f.fieldType().docValueType();
+      if (dvType == null) {
+        throw new IllegalArgumentException("can only update NUMERIC or BINARY fields! field=" + f.name());
+      }
+      if (!globalFieldNumberMap.contains(f.name(), dvType)) {
+        throw new IllegalArgumentException("can only update existing docvalues fields! field=" + f.name() + ", type=" + dvType);
+      }
+      switch (dvType) {
+        case NUMERIC:
+          dvUpdates[i] = new NumericDocValuesUpdate(term, f.name(), (Long) f.numericValue());
+          break;
+        case BINARY:
+          dvUpdates[i] = new BinaryDocValuesUpdate(term, f.name(), f.binaryValue());
+          break;
+        default:
+          throw new IllegalArgumentException("can only update NUMERIC or BINARY fields: field=" + f.name() + ", type=" + dvType);
+      }
+    }
+    try {
+      if (docWriter.updateDocValues(dvUpdates)) {
+        processEvents(true, false);
+      }
+    } catch (OutOfMemoryError oom) {
+      handleOOM(oom, "updateDocValues");
+    }
+  }
+  
   // for test purpose
   final synchronized int getSegmentCount(){
     return segmentInfos.size();

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java?rev=1598272&r1=1598271&r2=1598272&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java Thu May 29 12:24:48 2014
@@ -86,7 +86,7 @@ class NumericDocValuesFieldUpdates exten
   private int size;
   
   public NumericDocValuesFieldUpdates(String field, int maxDoc) {
-    super(field, Type.NUMERIC);
+    super(field, FieldInfo.DocValuesType.NUMERIC);
     bitsPerValue = PackedInts.bitsRequired(maxDoc - 1);
     docs = new PagedMutable(1, PAGE_SIZE, bitsPerValue, PackedInts.COMPACT);
     values = new PagedGrowableWriter(1, PAGE_SIZE, 1, PackedInts.FAST);

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java?rev=1598272&r1=1598271&r2=1598272&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java Thu May 29 12:24:48 2014
@@ -911,7 +911,7 @@ public class TestBinaryDocValuesUpdates 
     final IndexWriter writer = new IndexWriter(dir, conf);
     
     // create index
-    final int numThreads = TestUtil.nextInt(random(), 3, 6);
+    final int numFields = TestUtil.nextInt(random(), 1, 4);
     final int numDocs = atLeast(2000);
     for (int i = 0; i < numDocs; i++) {
       Document doc = new Document();
@@ -923,7 +923,7 @@ public class TestBinaryDocValuesUpdates 
       else if (group < 0.8) g = "g2";
       else g = "g3";
       doc.add(new StringField("updKey", g, Store.NO));
-      for (int j = 0; j < numThreads; j++) {
+      for (int j = 0; j < numFields; j++) {
         long value = random().nextInt();
         doc.add(new BinaryDocValuesField("f" + j, toBytes(value)));
         doc.add(new BinaryDocValuesField("cf" + j, toBytes(value * 2))); // control, always updated to f * 2
@@ -931,14 +931,13 @@ public class TestBinaryDocValuesUpdates 
       writer.addDocument(doc);
     }
     
+    final int numThreads = TestUtil.nextInt(random(), 3, 6);
     final CountDownLatch done = new CountDownLatch(numThreads);
     final AtomicInteger numUpdates = new AtomicInteger(atLeast(100));
     
     // same thread updates a field as well as reopens
     Thread[] threads = new Thread[numThreads];
     for (int i = 0; i < threads.length; i++) {
-      final String f = "f" + i;
-      final String cf = "cf" + i;
       threads[i] = new Thread("UpdateThread-" + i) {
         @Override
         public void run() {
@@ -953,10 +952,13 @@ public class TestBinaryDocValuesUpdates 
               else if (group < 0.5) t = new Term("updKey", "g1");
               else if (group < 0.8) t = new Term("updKey", "g2");
               else t = new Term("updKey", "g3");
-//              System.out.println("[" + Thread.currentThread().getName() + "] numUpdates=" + numUpdates + " updateTerm=" + t);
+
+              final int field = random().nextInt(numFields);
+              final String f = "f" + field;
+              final String cf = "cf" + field;
+//              System.out.println("[" + Thread.currentThread().getName() + "] numUpdates=" + numUpdates + " updateTerm=" + t + " field=" + field);
               long updValue = random.nextInt();
-              writer.updateBinaryDocValue(t, f, toBytes(updValue));
-              writer.updateBinaryDocValue(t, cf, toBytes(updValue * 2));
+              writer.updateDocValues(t, new BinaryDocValuesField(f, toBytes(updValue)), new BinaryDocValuesField(cf, toBytes(updValue*2)));
               
               if (random.nextDouble() < 0.2) {
                 // delete a random document
@@ -1012,7 +1014,7 @@ public class TestBinaryDocValuesUpdates 
     BytesRef scratch = new BytesRef();
     for (AtomicReaderContext context : reader.leaves()) {
       AtomicReader r = context.reader();
-      for (int i = 0; i < numThreads; i++) {
+      for (int i = 0; i < numFields; i++) {
         BinaryDocValues bdv = r.getBinaryDocValues("f" + i);
         BinaryDocValues control = r.getBinaryDocValues("cf" + i);
         Bits docsWithBdv = r.getDocsWithField("f" + i);
@@ -1054,8 +1056,7 @@ public class TestBinaryDocValuesUpdates 
       int doc = random().nextInt(numDocs);
       Term t = new Term("id", "doc" + doc);
       long value = random().nextLong();
-      writer.updateBinaryDocValue(t, "f", toBytes(value));
-      writer.updateBinaryDocValue(t, "cf", toBytes(value * 2));
+      writer.updateDocValues(t, new BinaryDocValuesField("f", toBytes(value)), new BinaryDocValuesField("cf", toBytes(value*2)));
       DirectoryReader reader = DirectoryReader.open(writer, true);
       for (AtomicReaderContext context : reader.leaves()) {
         AtomicReader r = context.reader();
@@ -1146,8 +1147,7 @@ public class TestBinaryDocValuesUpdates 
     // update some docs to a random value
     long value = random().nextInt();
     Term term = new Term("id", RandomPicks.randomFrom(random(), randomTerms));
-    writer.updateBinaryDocValue(term, "bdv", toBytes(value));
-    writer.updateBinaryDocValue(term, "control", toBytes(value * 2));
+    writer.updateDocValues(term, new BinaryDocValuesField("bdv", toBytes(value)), new BinaryDocValuesField("control", toBytes(value * 2)));
     writer.shutdown();
     
     Directory dir2 = newDirectory();
@@ -1252,8 +1252,8 @@ public class TestBinaryDocValuesUpdates 
       int field = random.nextInt(numBinaryFields);
       Term updateTerm = new Term("upd", RandomPicks.randomFrom(random, updateTerms));
       long value = random.nextInt();
-      writer.updateBinaryDocValue(updateTerm, "f" + field, toBytes(value));
-      writer.updateBinaryDocValue(updateTerm, "cf" + field, toBytes(value * 2));
+      writer.updateDocValues(updateTerm, new BinaryDocValuesField("f" + field, toBytes(value)), 
+          new BinaryDocValuesField("cf" + field, toBytes(value * 2)));
     }
 
     writer.shutdown();

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1598272&r1=1598271&r2=1598272&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Thu May 29 12:24:48 2014
@@ -35,7 +35,6 @@ import org.apache.lucene.analysis.MockTo
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
@@ -1979,8 +1978,6 @@ public class TestIndexWriterExceptions e
 
       shouldFail.set(true);
       boolean doClose = false;
-      int updatingDocID = -1;
-      long updatingValue = -1;
       try {
         boolean defaultCodecSupportsFieldUpdates = defaultCodecSupportsFieldUpdates();
         for(int i=0;i<numDocs;i++) {
@@ -1993,28 +1990,18 @@ public class TestIndexWriterExceptions e
                 System.out.println("  update id=" + docid + " to value " + value);
               }
               Term idTerm = new Term("id", Integer.toString(docid));
-              updatingDocID = docid; // record that we're updating that document
-              updatingValue = value; // and its updating value
               if (random().nextBoolean()) { // update only numeric field
-                w.updateNumericDocValue(idTerm, "f", value);
-                w.updateNumericDocValue(idTerm, "cf", value * 2);
+                w.updateDocValues(idTerm, new NumericDocValuesField("f", value), new NumericDocValuesField("cf", value*2));
               } else if (random().nextBoolean()) {
-                w.updateBinaryDocValue(idTerm, "bf", TestBinaryDocValuesUpdates.toBytes(value));
-                w.updateBinaryDocValue(idTerm, "bcf", TestBinaryDocValuesUpdates.toBytes(value * 2));
+                w.updateDocValues(idTerm, new BinaryDocValuesField("bf", TestBinaryDocValuesUpdates.toBytes(value)),
+                    new BinaryDocValuesField("bcf", TestBinaryDocValuesUpdates.toBytes(value*2)));
               } else {
-                w.updateNumericDocValue(idTerm, "f", value);
-                w.updateNumericDocValue(idTerm, "cf", value * 2);
-                w.updateBinaryDocValue(idTerm, "bf", TestBinaryDocValuesUpdates.toBytes(value));
-                w.updateBinaryDocValue(idTerm, "bcf", TestBinaryDocValuesUpdates.toBytes(value * 2));
+                w.updateDocValues(idTerm, 
+                    new NumericDocValuesField("f", value), 
+                    new NumericDocValuesField("cf", value*2),
+                    new BinaryDocValuesField("bf", TestBinaryDocValuesUpdates.toBytes(value)),
+                    new BinaryDocValuesField("bcf", TestBinaryDocValuesUpdates.toBytes(value*2)));
               }
-              // record that we successfully updated the document. this is
-              // important when we later assert the value of the DV fields of
-              // that document - since we update two fields that depend on each
-              // other, could be that one of the fields successfully updates,
-              // while the other fails (since we turn on random exceptions).
-              // while this is supported, it makes the test raise false alarms.
-              updatingDocID = -1;
-              updatingValue = -1;
             }
             
             // sometimes do both deletes and updates
@@ -2062,18 +2049,6 @@ public class TestIndexWriterExceptions e
       }
       shouldFail.set(false);
 
-      if (updatingDocID != -1) {
-        // Updating this document did not succeed. Since the fields we assert on
-        // depend on each other, and the update may have gone through halfway,
-        // replay the update on both numeric and binary DV fields, so later
-        // asserts succeed.
-        Term idTerm = new Term("id", ""+updatingDocID);
-        w.updateNumericDocValue(idTerm, "f", updatingValue);
-        w.updateNumericDocValue(idTerm, "cf", updatingValue * 2);
-        w.updateBinaryDocValue(idTerm, "bf", TestBinaryDocValuesUpdates.toBytes(updatingValue));
-        w.updateBinaryDocValue(idTerm, "bcf", TestBinaryDocValuesUpdates.toBytes(updatingValue * 2));
-      }
-      
       IndexReader r;
 
       if (doClose && w != null) {

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java?rev=1598272&r1=1598271&r2=1598272&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java Thu May 29 12:24:48 2014
@@ -160,6 +160,7 @@ public class TestMixedDocValuesUpdates e
     final IndexWriter writer = new IndexWriter(dir, conf);
     
     // create index
+    final int numFields = TestUtil.nextInt(random(), 2, 4);
     final int numThreads = TestUtil.nextInt(random(), 3, 6);
     final int numDocs = atLeast(2000);
     for (int i = 0; i < numDocs; i++) {
@@ -172,7 +173,7 @@ public class TestMixedDocValuesUpdates e
       else if (group < 0.8) g = "g2";
       else g = "g3";
       doc.add(new StringField("updKey", g, Store.NO));
-      for (int j = 0; j < numThreads; j++) {
+      for (int j = 0; j < numFields; j++) {
         long value = random().nextInt();
         doc.add(new BinaryDocValuesField("f" + j, TestBinaryDocValuesUpdates.toBytes(value)));
         doc.add(new NumericDocValuesField("cf" + j, value * 2)); // control, always updated to f * 2
@@ -186,8 +187,6 @@ public class TestMixedDocValuesUpdates e
     // same thread updates a field as well as reopens
     Thread[] threads = new Thread[numThreads];
     for (int i = 0; i < threads.length; i++) {
-      final String f = "f" + i;
-      final String cf = "cf" + i;
       threads[i] = new Thread("UpdateThread-" + i) {
         @Override
         public void run() {
@@ -203,10 +202,13 @@ public class TestMixedDocValuesUpdates e
               else if (group < 0.8) t = new Term("updKey", "g2");
               else t = new Term("updKey", "g3");
 //              System.out.println("[" + Thread.currentThread().getName() + "] numUpdates=" + numUpdates + " updateTerm=" + t);
+              int field = random().nextInt(numFields);
+              final String f = "f" + field;
+              final String cf = "cf" + field;
               long updValue = random.nextInt();
 //              System.err.println("[" + Thread.currentThread().getName() + "] t=" + t + ", f=" + f + ", updValue=" + updValue);
-              writer.updateBinaryDocValue(t, f, TestBinaryDocValuesUpdates.toBytes(updValue));
-              writer.updateNumericDocValue(t, cf, updValue * 2);
+              writer.updateDocValues(t, new BinaryDocValuesField(f, TestBinaryDocValuesUpdates.toBytes(updValue)),
+                  new NumericDocValuesField(cf, updValue*2));
               
               if (random.nextDouble() < 0.2) {
                 // delete a random document
@@ -262,7 +264,7 @@ public class TestMixedDocValuesUpdates e
     BytesRef scratch = new BytesRef();
     for (AtomicReaderContext context : reader.leaves()) {
       AtomicReader r = context.reader();
-      for (int i = 0; i < numThreads; i++) {
+      for (int i = 0; i < numFields; i++) {
         BinaryDocValues bdv = r.getBinaryDocValues("f" + i);
         NumericDocValues control = r.getNumericDocValues("cf" + i);
         Bits docsWithBdv = r.getDocsWithField("f" + i);
@@ -270,15 +272,14 @@ public class TestMixedDocValuesUpdates e
         Bits liveDocs = r.getLiveDocs();
         for (int j = 0; j < r.maxDoc(); j++) {
           if (liveDocs == null || liveDocs.get(j)) {
-            assertEquals(docsWithBdv.get(j), docsWithControl.get(j));
-            if (docsWithBdv.get(j)) {
-              long ctrlValue = control.get(j);
-              long bdvValue = TestBinaryDocValuesUpdates.getValue(bdv, j, scratch) * 2;
+            assertTrue(docsWithBdv.get(j));
+            assertTrue(docsWithControl.get(j));
+            long ctrlValue = control.get(j);
+            long bdvValue = TestBinaryDocValuesUpdates.getValue(bdv, j, scratch) * 2;
 //              if (ctrlValue != bdvValue) {
 //                System.out.println("seg=" + r + ", f=f" + i + ", doc=" + j + ", group=" + r.document(j).get("updKey") + ", ctrlValue=" + ctrlValue + ", bdvBytes=" + scratch);
 //              }
-              assertEquals(ctrlValue, bdvValue);
-            }
+            assertEquals(ctrlValue, bdvValue);
           }
         }
       }
@@ -310,8 +311,8 @@ public class TestMixedDocValuesUpdates e
       int doc = random().nextInt(numDocs);
       Term t = new Term("id", "doc" + doc);
       long value = random().nextLong();
-      writer.updateBinaryDocValue(t, "f", TestBinaryDocValuesUpdates.toBytes(value));
-      writer.updateNumericDocValue(t, "cf", value * 2);
+      writer.updateDocValues(t, new BinaryDocValuesField("f", TestBinaryDocValuesUpdates.toBytes(value)),
+          new NumericDocValuesField("cf", value*2));
       DirectoryReader reader = DirectoryReader.open(writer, true);
       for (AtomicReaderContext context : reader.leaves()) {
         AtomicReader r = context.reader();
@@ -373,8 +374,8 @@ public class TestMixedDocValuesUpdates e
       int field = random.nextInt(numBinaryFields);
       Term updateTerm = new Term("upd", RandomPicks.randomFrom(random, updateTerms));
       long value = random.nextInt();
-      writer.updateBinaryDocValue(updateTerm, "f" + field, TestBinaryDocValuesUpdates.toBytes(value));
-      writer.updateNumericDocValue(updateTerm, "cf" + field, value * 2);
+      writer.updateDocValues(updateTerm, new BinaryDocValuesField("f"+field, TestBinaryDocValuesUpdates.toBytes(value)),
+          new NumericDocValuesField("cf"+field, value*2));
     }
 
     writer.shutdown();

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java?rev=1598272&r1=1598271&r2=1598272&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java Thu May 29 12:24:48 2014
@@ -57,7 +57,7 @@ import com.carrotsearch.randomizedtestin
 @SuppressCodecs({"Lucene40","Lucene41","Lucene42","Lucene45"})
 @SuppressWarnings("resource")
 public class TestNumericDocValuesUpdates extends LuceneTestCase {
-  
+
   private Document doc(int id) {
     Document doc = new Document();
     doc.add(new StringField("id", "doc-" + id, Store.NO));
@@ -893,7 +893,7 @@ public class TestNumericDocValuesUpdates
     final IndexWriter writer = new IndexWriter(dir, conf);
     
     // create index
-    final int numThreads = TestUtil.nextInt(random(), 3, 6);
+    final int numFields = TestUtil.nextInt(random(), 1, 4);
     final int numDocs = atLeast(2000);
     for (int i = 0; i < numDocs; i++) {
       Document doc = new Document();
@@ -905,7 +905,7 @@ public class TestNumericDocValuesUpdates
       else if (group < 0.8) g = "g2";
       else g = "g3";
       doc.add(new StringField("updKey", g, Store.NO));
-      for (int j = 0; j < numThreads; j++) {
+      for (int j = 0; j < numFields; j++) {
         long value = random().nextInt();
         doc.add(new NumericDocValuesField("f" + j, value));
         doc.add(new NumericDocValuesField("cf" + j, value * 2)); // control, always updated to f * 2
@@ -913,14 +913,13 @@ public class TestNumericDocValuesUpdates
       writer.addDocument(doc);
     }
     
+    final int numThreads = TestUtil.nextInt(random(), 3, 6);
     final CountDownLatch done = new CountDownLatch(numThreads);
     final AtomicInteger numUpdates = new AtomicInteger(atLeast(100));
     
     // same thread updates a field as well as reopens
     Thread[] threads = new Thread[numThreads];
     for (int i = 0; i < threads.length; i++) {
-      final String f = "f" + i;
-      final String cf = "cf" + i;
       threads[i] = new Thread("UpdateThread-" + i) {
         @Override
         public void run() {
@@ -935,10 +934,13 @@ public class TestNumericDocValuesUpdates
               else if (group < 0.5) t = new Term("updKey", "g1");
               else if (group < 0.8) t = new Term("updKey", "g2");
               else t = new Term("updKey", "g3");
-//              System.out.println("[" + Thread.currentThread().getName() + "] numUpdates=" + numUpdates + " updateTerm=" + t);
+
+              final int field = random().nextInt(numFields);
+              final String f = "f" + field;
+              final String cf = "cf" + field;
+//              System.out.println("[" + Thread.currentThread().getName() + "] numUpdates=" + numUpdates + " updateTerm=" + t + " field=" + field);
               long updValue = random.nextInt();
-              writer.updateNumericDocValue(t, f, updValue);
-              writer.updateNumericDocValue(t, cf, updValue * 2);
+              writer.updateDocValues(t, new NumericDocValuesField(f, updValue), new NumericDocValuesField(cf, updValue*2));
               
               if (random.nextDouble() < 0.2) {
                 // delete a random document
@@ -993,7 +995,7 @@ public class TestNumericDocValuesUpdates
     DirectoryReader reader = DirectoryReader.open(dir);
     for (AtomicReaderContext context : reader.leaves()) {
       AtomicReader r = context.reader();
-      for (int i = 0; i < numThreads; i++) {
+      for (int i = 0; i < numFields; i++) {
         NumericDocValues ndv = r.getNumericDocValues("f" + i);
         NumericDocValues control = r.getNumericDocValues("cf" + i);
         Bits docsWithNdv = r.getDocsWithField("f" + i);
@@ -1035,8 +1037,7 @@ public class TestNumericDocValuesUpdates
       int doc = random().nextInt(numDocs);
       Term t = new Term("id", "doc" + doc);
       long value = random().nextLong();
-      writer.updateNumericDocValue(t, "f", value);
-      writer.updateNumericDocValue(t, "cf", value * 2);
+      writer.updateDocValues(t, new NumericDocValuesField("f", value), new NumericDocValuesField("cf", value*2));
       DirectoryReader reader = DirectoryReader.open(writer, true);
       for (AtomicReaderContext context : reader.leaves()) {
         AtomicReader r = context.reader();
@@ -1128,8 +1129,7 @@ public class TestNumericDocValuesUpdates
     // update some docs to a random value
     long value = random().nextInt();
     Term term = new Term("id", RandomPicks.randomFrom(random(), randomTerms));
-    writer.updateNumericDocValue(term, "ndv", value);
-    writer.updateNumericDocValue(term, "control", value * 2);
+    writer.updateDocValues(term, new NumericDocValuesField("ndv", value), new NumericDocValuesField("control", value*2));
     writer.shutdown();
     
     Directory dir2 = newDirectory();
@@ -1235,8 +1235,7 @@ public class TestNumericDocValuesUpdates
       int field = random.nextInt(numNumericFields);
       Term updateTerm = new Term("upd", RandomPicks.randomFrom(random, updateTerms));
       long value = random.nextInt();
-      writer.updateNumericDocValue(updateTerm, "f" + field, value);
-      writer.updateNumericDocValue(updateTerm, "cf" + field, value * 2);
+      writer.updateDocValues(updateTerm, new NumericDocValuesField("f"+field, value), new NumericDocValuesField("cf"+field, value*2));
     }
 
     writer.shutdown();

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1598272&r1=1598271&r2=1598272&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java Thu May 29 12:24:48 2014
@@ -25,6 +25,7 @@ import java.util.Random;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.document.Field;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
@@ -239,6 +240,11 @@ public class RandomIndexWriter implement
     w.updateBinaryDocValue(term, field, value);
   }
   
+  public void updateDocValues(Term term, Field... updates) throws IOException {
+    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
+    w.updateDocValues(term, updates);
+  }
+  
   public void deleteDocuments(Term term) throws IOException {
     LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
     w.deleteDocuments(term);