You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2014/05/26 17:19:21 UTC
svn commit: r1597604 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/index/
lucene/core/src/test/org/apache/lucene/index/
Author: shaie
Date: Mon May 26 15:19:20 2014
New Revision: 1597604
URL: http://svn.apache.org/r1597604
Log:
LUCENE-5706: remove the ability to unset a DocValues field through DV updates
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1597604&r1=1597603&r2=1597604&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Mon May 26 15:19:20 2014
@@ -69,6 +69,9 @@ API Changes
IndexWriter.deleteDocuments(Query) with their varargs counterparts.
(Shai Erera)
+* LUCENE-5706: Removed the option to unset a DocValues field through DocValues
+ updates. (Shai Erera)
+
Optimizations
* LUCENE-5603: hunspell stemmer more efficiently strips prefixes
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java?rev=1597604&r1=1597603&r2=1597604&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java Mon May 26 15:19:20 2014
@@ -1,11 +1,9 @@
package org.apache.lucene.index;
import org.apache.lucene.document.BinaryDocValuesField;
-import org.apache.lucene.index.DocValuesUpdate.BinaryDocValuesUpdate;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.InPlaceMergeSorter;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PagedGrowableWriter;
@@ -41,31 +39,25 @@ class BinaryDocValuesFieldUpdates extend
private final int size;
private final PagedGrowableWriter lengths;
private final PagedMutable docs;
- private final FixedBitSet docsWithField;
private long idx = 0; // long so we don't overflow if size == Integer.MAX_VALUE
private int doc = -1;
private final BytesRef value;
private int offset, length;
Iterator(int size, PagedGrowableWriter offsets, PagedGrowableWriter lengths,
- PagedMutable docs, BytesRef values, FixedBitSet docsWithField) {
+ PagedMutable docs, BytesRef values) {
this.offsets = offsets;
this.size = size;
this.lengths = lengths;
this.docs = docs;
- this.docsWithField = docsWithField;
value = values.clone();
}
@Override
BytesRef value() {
- if (offset == -1) {
- return null;
- } else {
- value.offset = offset;
- value.length = length;
- return value;
- }
+ value.offset = offset;
+ value.length = length;
+ return value;
}
@Override
@@ -81,15 +73,11 @@ class BinaryDocValuesFieldUpdates extend
}
// idx points to the "next" element
long prevIdx = idx - 1;
- if (!docsWithField.get((int) prevIdx)) {
- offset = -1;
- } else {
- // cannot change 'value' here because nextDoc is called before the
- // value is used, and it's a waste to clone the BytesRef when we
- // obtain the value
- offset = (int) offsets.get(prevIdx);
- length = (int) lengths.get(prevIdx);
- }
+ // cannot change 'value' here because nextDoc is called before the
+ // value is used, and it's a waste to clone the BytesRef when we
+ // obtain the value
+ offset = (int) offsets.get(prevIdx);
+ length = (int) lengths.get(prevIdx);
return doc;
}
@@ -106,7 +94,6 @@ class BinaryDocValuesFieldUpdates extend
}
}
- private FixedBitSet docsWithField;
private PagedMutable docs;
private PagedGrowableWriter offsets, lengths;
private BytesRef values;
@@ -115,7 +102,6 @@ class BinaryDocValuesFieldUpdates extend
public BinaryDocValuesFieldUpdates(String field, int maxDoc) {
super(field, Type.BINARY);
- docsWithField = new FixedBitSet(64);
bitsPerValue = PackedInts.bitsRequired(maxDoc - 1);
docs = new PagedMutable(1, PAGE_SIZE, bitsPerValue, PackedInts.COMPACT);
offsets = new PagedGrowableWriter(1, PAGE_SIZE, 1, PackedInts.FAST);
@@ -132,21 +118,12 @@ class BinaryDocValuesFieldUpdates extend
}
BytesRef val = (BytesRef) value;
- if (val == null) {
- val = BinaryDocValuesUpdate.MISSING;
- }
// grow the structures to have room for more elements
if (docs.size() == size) {
docs = docs.grow(size + 1);
offsets = offsets.grow(size + 1);
lengths = lengths.grow(size + 1);
- docsWithField = FixedBitSet.ensureCapacity(docsWithField, (int) docs.size());
- }
-
- if (val != BinaryDocValuesUpdate.MISSING) {
- // only mark the document as having a value in that field if the value wasn't set to null (MISSING)
- docsWithField.set(size);
}
docs.set(size, doc);
@@ -162,7 +139,6 @@ class BinaryDocValuesFieldUpdates extend
final PagedGrowableWriter offsets = this.offsets;
final PagedGrowableWriter lengths = this.lengths;
final BytesRef values = this.values;
- final FixedBitSet docsWithField = this.docsWithField;
new InPlaceMergeSorter() {
@Override
protected void swap(int i, int j) {
@@ -177,18 +153,6 @@ class BinaryDocValuesFieldUpdates extend
long tmpLength = lengths.get(j);
lengths.set(j, lengths.get(i));
lengths.set(i, tmpLength);
-
- boolean tmpBool = docsWithField.get(j);
- if (docsWithField.get(i)) {
- docsWithField.set(j);
- } else {
- docsWithField.clear(j);
- }
- if (tmpBool) {
- docsWithField.set(i);
- } else {
- docsWithField.clear(i);
- }
}
@Override
@@ -199,7 +163,7 @@ class BinaryDocValuesFieldUpdates extend
}
}.sort(0, size);
- return new Iterator(size, offsets, lengths, docs, values, docsWithField);
+ return new Iterator(size, offsets, lengths, docs, values);
}
@Override
@@ -214,12 +178,8 @@ class BinaryDocValuesFieldUpdates extend
docs = docs.grow(newSize);
offsets = offsets.grow(newSize);
lengths = lengths.grow(newSize);
- docsWithField = FixedBitSet.ensureCapacity(docsWithField, (int) docs.size());
for (int i = 0; i < otherUpdates.size; i++) {
int doc = (int) otherUpdates.docs.get(i);
- if (otherUpdates.docsWithField.get(i)) {
- docsWithField.set(size);
- }
docs.set(size, doc);
offsets.set(size, values.length + otherUpdates.offsets.get(i)); // correct relative offset
lengths.set(size, otherUpdates.lengths.get(i));
@@ -240,7 +200,7 @@ class BinaryDocValuesFieldUpdates extend
@Override
public long ramBytesPerDoc() {
- long bytesPerDoc = (long) Math.ceil((double) (bitsPerValue + 1 /* docsWithField */) / 8); // docs
+ long bytesPerDoc = (long) Math.ceil((double) (bitsPerValue) / 8); // docs
final int capacity = estimateCapacity(size);
bytesPerDoc += (long) Math.ceil((double) offsets.ramBytesUsed() / capacity); // offsets
bytesPerDoc += (long) Math.ceil((double) lengths.ramBytesUsed() / capacity); // lengths
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java?rev=1597604&r1=1597603&r2=1597604&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java Mon May 26 15:19:20 2014
@@ -81,10 +81,8 @@ abstract class DocValuesUpdate {
/* Size of BytesRef: 2*INT + ARRAY_HEADER + PTR */
private static final long RAW_VALUE_SIZE_IN_BYTES = NUM_BYTES_ARRAY_HEADER + 2*NUM_BYTES_INT + NUM_BYTES_OBJECT_REF;
- static final BytesRef MISSING = new BytesRef();
-
BinaryDocValuesUpdate(Term term, String field, BytesRef value) {
- super(DocValuesFieldUpdates.Type.BINARY, term, field, value == null ? MISSING : value);
+ super(DocValuesFieldUpdates.Type.BINARY, term, field, value);
}
@Override
@@ -97,10 +95,8 @@ abstract class DocValuesUpdate {
/** An in-place update to a numeric DocValues field */
static final class NumericDocValuesUpdate extends DocValuesUpdate {
- static final Long MISSING = new Long(0);
-
NumericDocValuesUpdate(Term term, String field, Long value) {
- super(DocValuesFieldUpdates.Type.NUMERIC, term, field, value == null ? MISSING : value);
+ super(DocValuesFieldUpdates.Type.NUMERIC, term, field, value);
}
@Override
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1597604&r1=1597603&r2=1597604&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java Mon May 26 15:19:20 2014
@@ -156,9 +156,9 @@ final class DocumentsWriter implements C
return applyAllDeletes( deleteQueue);
}
- synchronized boolean updateNumericDocValue(Term term, String field, Long value) throws IOException {
+ synchronized boolean updateNumericDocValue(Term term, String field, long value) throws IOException {
final DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
- deleteQueue.addNumericUpdate(new NumericDocValuesUpdate(term, field, value));
+ deleteQueue.addNumericUpdate(new NumericDocValuesUpdate(term, field, Long.valueOf(value)));
flushControl.doOnDelete();
return applyAllDeletes(deleteQueue);
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1597604&r1=1597603&r2=1597604&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Mon May 26 15:19:20 2014
@@ -1511,10 +1511,8 @@ public class IndexWriter implements Clos
/**
* Updates a document's {@link NumericDocValues} for <code>field</code> to the
- * given <code>value</code>. This method can be used to 'unset' a document's
- * value by passing {@code null} as the new value. Also, you can only update
- * fields that already exist in the index, not add new fields through this
- * method.
+ * given <code>value</code>. You can only update fields that already exist in
+ * the index, not add new fields through this method.
*
* <p>
* <b>NOTE</b>: if this method hits an OutOfMemoryError you should immediately
@@ -1532,7 +1530,7 @@ public class IndexWriter implements Clos
* @throws IOException
* if there is a low-level IO error
*/
- public void updateNumericDocValue(Term term, String field, Long value) throws IOException {
+ public void updateNumericDocValue(Term term, String field, long value) throws IOException {
ensureOpen();
if (!globalFieldNumberMap.contains(field, DocValuesType.NUMERIC)) {
throw new IllegalArgumentException("can only update existing numeric-docvalues fields!");
@@ -1548,10 +1546,8 @@ public class IndexWriter implements Clos
/**
* Updates a document's {@link BinaryDocValues} for <code>field</code> to the
- * given <code>value</code>. This method can be used to 'unset' a document's
- * value by passing {@code null} as the new value. Also, you can only update
- * fields that already exist in the index, not add new fields through this
- * method.
+ * given <code>value</code>. You can only update fields that already exist in
+ * the index, not add new fields through this method.
*
* <p>
* <b>NOTE:</b> this method currently replaces the existing value of all
@@ -1575,6 +1571,9 @@ public class IndexWriter implements Clos
*/
public void updateBinaryDocValue(Term term, String field, BytesRef value) throws IOException {
ensureOpen();
+ if (value == null) {
+ throw new IllegalArgumentException("cannot update a field to a null value: " + field);
+ }
if (!globalFieldNumberMap.contains(field, DocValuesType.BINARY)) {
throw new IllegalArgumentException("can only update existing binary-docvalues fields!");
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java?rev=1597604&r1=1597603&r2=1597604&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java Mon May 26 15:19:20 2014
@@ -1,9 +1,7 @@
package org.apache.lucene.index;
import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.index.DocValuesUpdate.NumericDocValuesUpdate;
import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.InPlaceMergeSorter;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PagedGrowableWriter;
@@ -37,16 +35,14 @@ class NumericDocValuesFieldUpdates exten
final static class Iterator extends DocValuesFieldUpdates.Iterator {
private final int size;
private final PagedGrowableWriter values;
- private final FixedBitSet docsWithField;
private final PagedMutable docs;
private long idx = 0; // long so we don't overflow if size == Integer.MAX_VALUE
private int doc = -1;
private Long value = null;
- Iterator(int size, PagedGrowableWriter values, FixedBitSet docsWithField, PagedMutable docs) {
+ Iterator(int size, PagedGrowableWriter values, PagedMutable docs) {
this.size = size;
this.values = values;
- this.docsWithField = docsWithField;
this.docs = docs;
}
@@ -66,12 +62,8 @@ class NumericDocValuesFieldUpdates exten
while (idx < size && docs.get(idx) == doc) {
++idx;
}
- if (!docsWithField.get((int) (idx - 1))) {
- value = null;
- } else {
- // idx points to the "next" element
- value = Long.valueOf(values.get(idx - 1));
- }
+ // idx points to the "next" element
+ value = Long.valueOf(values.get(idx - 1));
return doc;
}
@@ -89,14 +81,12 @@ class NumericDocValuesFieldUpdates exten
}
private final int bitsPerValue;
- private FixedBitSet docsWithField;
private PagedMutable docs;
private PagedGrowableWriter values;
private int size;
public NumericDocValuesFieldUpdates(String field, int maxDoc) {
super(field, Type.NUMERIC);
- docsWithField = new FixedBitSet(64);
bitsPerValue = PackedInts.bitsRequired(maxDoc - 1);
docs = new PagedMutable(1, PAGE_SIZE, bitsPerValue, PackedInts.COMPACT);
values = new PagedGrowableWriter(1, PAGE_SIZE, 1, PackedInts.FAST);
@@ -111,20 +101,11 @@ class NumericDocValuesFieldUpdates exten
}
Long val = (Long) value;
- if (val == null) {
- val = NumericDocValuesUpdate.MISSING;
- }
// grow the structures to have room for more elements
if (docs.size() == size) {
docs = docs.grow(size + 1);
values = values.grow(size + 1);
- docsWithField = FixedBitSet.ensureCapacity(docsWithField, (int) docs.size());
- }
-
- if (val != NumericDocValuesUpdate.MISSING) {
- // only mark the document as having a value in that field if the value wasn't set to null (MISSING)
- docsWithField.set(size);
}
docs.set(size, doc);
@@ -136,7 +117,6 @@ class NumericDocValuesFieldUpdates exten
public Iterator iterator() {
final PagedMutable docs = this.docs;
final PagedGrowableWriter values = this.values;
- final FixedBitSet docsWithField = this.docsWithField;
new InPlaceMergeSorter() {
@Override
protected void swap(int i, int j) {
@@ -147,18 +127,6 @@ class NumericDocValuesFieldUpdates exten
long tmpVal = values.get(j);
values.set(j, values.get(i));
values.set(i, tmpVal);
-
- boolean tmpBool = docsWithField.get(j);
- if (docsWithField.get(i)) {
- docsWithField.set(j);
- } else {
- docsWithField.clear(j);
- }
- if (tmpBool) {
- docsWithField.set(i);
- } else {
- docsWithField.clear(i);
- }
}
@Override
@@ -169,7 +137,7 @@ class NumericDocValuesFieldUpdates exten
}
}.sort(0, size);
- return new Iterator(size, values, docsWithField, docs);
+ return new Iterator(size, values, docs);
}
@Override
@@ -183,12 +151,8 @@ class NumericDocValuesFieldUpdates exten
}
docs = docs.grow(size + otherUpdates.size);
values = values.grow(size + otherUpdates.size);
- docsWithField = FixedBitSet.ensureCapacity(docsWithField, (int) docs.size());
for (int i = 0; i < otherUpdates.size; i++) {
int doc = (int) otherUpdates.docs.get(i);
- if (otherUpdates.docsWithField.get(i)) {
- docsWithField.set(size);
- }
docs.set(size, doc);
values.set(size, otherUpdates.values.get(i));
++size;
@@ -202,7 +166,7 @@ class NumericDocValuesFieldUpdates exten
@Override
public long ramBytesPerDoc() {
- long bytesPerDoc = (long) Math.ceil((double) (bitsPerValue + 1 /* docsWithField */) / 8);
+ long bytesPerDoc = (long) Math.ceil((double) (bitsPerValue) / 8);
final int capacity = estimateCapacity(size);
bytesPerDoc += (long) Math.ceil((double) values.ramBytesUsed() / capacity); // values
return bytesPerDoc;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java?rev=1597604&r1=1597603&r2=1597604&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java Mon May 26 15:19:20 2014
@@ -1,7 +1,6 @@
package org.apache.lucene.index;
import java.io.IOException;
-import java.util.Arrays;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
@@ -465,80 +464,6 @@ public class TestBinaryDocValuesUpdates
dir.close();
}
- public void testUnsetValue() throws Exception {
- assumeTrue("codec does not support docsWithField", defaultCodecSupportsDocsWithField());
- Directory dir = newDirectory();
- IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- IndexWriter writer = new IndexWriter(dir, conf);
-
- for (int i = 0; i < 2; i++) {
- Document doc = new Document();
- doc.add(new StringField("id", "doc" + i, Store.NO));
- doc.add(new BinaryDocValuesField("bdv", toBytes(5L)));
- writer.addDocument(doc);
- }
- writer.commit();
-
- // unset the value of 'doc0'
- writer.updateBinaryDocValue(new Term("id", "doc0"), "bdv", null);
- writer.close();
-
- final DirectoryReader reader = DirectoryReader.open(dir);
- AtomicReader r = reader.leaves().get(0).reader();
- BinaryDocValues bdv = r.getBinaryDocValues("bdv");
- BytesRef scratch = new BytesRef();
- for (int i = 0; i < r.maxDoc(); i++) {
- if (i == 0) {
- bdv.get(i, scratch);
- assertEquals(0, scratch.length);
- } else {
- assertEquals(5, getValue(bdv, i, scratch));
- }
- }
-
- Bits docsWithField = r.getDocsWithField("bdv");
- assertFalse(docsWithField.get(0));
- assertTrue(docsWithField.get(1));
-
- reader.close();
- dir.close();
- }
-
- public void testUnsetAllValues() throws Exception {
- assumeTrue("codec does not support docsWithField", defaultCodecSupportsDocsWithField());
- Directory dir = newDirectory();
- IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- IndexWriter writer = new IndexWriter(dir, conf);
-
- for (int i = 0; i < 2; i++) {
- Document doc = new Document();
- doc.add(new StringField("id", "doc", Store.NO));
- doc.add(new BinaryDocValuesField("bdv", toBytes(5L)));
- writer.addDocument(doc);
- }
- writer.commit();
-
- // unset the value of 'doc'
- writer.updateBinaryDocValue(new Term("id", "doc"), "bdv", null);
- writer.close();
-
- final DirectoryReader reader = DirectoryReader.open(dir);
- AtomicReader r = reader.leaves().get(0).reader();
- BinaryDocValues bdv = r.getBinaryDocValues("bdv");
- BytesRef scratch = new BytesRef();
- for (int i = 0; i < r.maxDoc(); i++) {
- bdv.get(i, scratch);
- assertEquals(0, scratch.length);
- }
-
- Bits docsWithField = r.getDocsWithField("bdv");
- assertFalse(docsWithField.get(0));
- assertFalse(docsWithField.get(1));
-
- reader.close();
- dir.close();
- }
-
public void testUpdateNonBinaryDocValuesField() throws Exception {
// we don't support adding new fields or updating existing non-binary-dv
// fields through binary updates
@@ -756,8 +681,6 @@ public class TestBinaryDocValuesUpdates
final int numFields = random.nextInt(4) + 3; // 3-7
final long[] fieldValues = new long[numFields];
- final boolean[] fieldHasValue = new boolean[numFields];
- Arrays.fill(fieldHasValue, true);
for (int i = 0; i < fieldValues.length; i++) {
fieldValues[i] = 1;
}
@@ -779,24 +702,10 @@ public class TestBinaryDocValuesUpdates
++docID;
}
- // if field's value was unset before, unset it from all new added documents too
- for (int field = 0; field < fieldHasValue.length; field++) {
- if (!fieldHasValue[field]) {
- writer.updateBinaryDocValue(new Term("key", "all"), "f" + field, null);
- }
- }
-
int fieldIdx = random.nextInt(fieldValues.length);
String updateField = "f" + fieldIdx;
- if (random.nextBoolean()) {
-// System.out.println("[" + Thread.currentThread().getName() + "]: unset field '" + updateField + "'");
- fieldHasValue[fieldIdx] = false;
- writer.updateBinaryDocValue(new Term("key", "all"), updateField, null);
- } else {
- fieldHasValue[fieldIdx] = true;
- writer.updateBinaryDocValue(new Term("key", "all"), updateField, toBytes(++fieldValues[fieldIdx]));
-// System.out.println("[" + Thread.currentThread().getName() + "]: updated field '" + updateField + "' to value " + fieldValues[fieldIdx]);
- }
+// System.out.println("[" + Thread.currentThread().getName() + "]: updated field '" + updateField + "' to value " + fieldValues[fieldIdx]);
+ writer.updateBinaryDocValue(new Term("key", "all"), updateField, toBytes(++fieldValues[fieldIdx]));
if (random.nextDouble() < 0.2) {
int deleteDoc = random.nextInt(docID); // might also delete an already deleted document, ok!
@@ -830,12 +739,8 @@ public class TestBinaryDocValuesUpdates
for (int doc = 0; doc < maxDoc; doc++) {
if (liveDocs == null || liveDocs.get(doc)) {
// System.out.println("doc=" + (doc + context.docBase) + " f='" + f + "' vslue=" + getValue(bdv, doc, scratch));
- if (fieldHasValue[field]) {
- assertTrue(docsWithField.get(doc));
- assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], getValue(bdv, doc, scratch));
- } else {
- assertFalse(docsWithField.get(doc));
- }
+ assertTrue(docsWithField.get(doc));
+ assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], getValue(bdv, doc, scratch));
}
}
}
@@ -1047,14 +952,9 @@ public class TestBinaryDocValuesUpdates
else if (group < 0.8) t = new Term("updKey", "g2");
else t = new Term("updKey", "g3");
// System.out.println("[" + Thread.currentThread().getName() + "] numUpdates=" + numUpdates + " updateTerm=" + t);
- if (random.nextBoolean()) { // sometimes unset a value
- writer.updateBinaryDocValue(t, f, null);
- writer.updateBinaryDocValue(t, cf, null);
- } else {
- long updValue = random.nextInt();
- writer.updateBinaryDocValue(t, f, toBytes(updValue));
- writer.updateBinaryDocValue(t, cf, toBytes(updValue * 2));
- }
+ long updValue = random.nextInt();
+ writer.updateBinaryDocValue(t, f, toBytes(updValue));
+ writer.updateBinaryDocValue(t, cf, toBytes(updValue * 2));
if (random.nextDouble() < 0.2) {
// delete a random document
@@ -1118,10 +1018,9 @@ public class TestBinaryDocValuesUpdates
Bits liveDocs = r.getLiveDocs();
for (int j = 0; j < r.maxDoc(); j++) {
if (liveDocs == null || liveDocs.get(j)) {
- assertEquals(docsWithBdv.get(j), docsWithControl.get(j));
- if (docsWithBdv.get(j)) {
- assertEquals(getValue(control, j, scratch), getValue(bdv, j, scratch) * 2);
- }
+ assertTrue(docsWithBdv.get(j));
+ assertTrue(docsWithControl.get(j));
+ assertEquals(getValue(control, j, scratch), getValue(bdv, j, scratch) * 2);
}
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java?rev=1597604&r1=1597603&r2=1597604&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java Mon May 26 15:19:20 2014
@@ -1,7 +1,6 @@
package org.apache.lucene.index;
import java.io.IOException;
-import java.util.Arrays;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
@@ -65,8 +64,6 @@ public class TestMixedDocValuesUpdates e
final int numFields = random.nextInt(4) + 3; // 3-7
final int numNDVFields = random.nextInt(numFields/2) + 1; // 1-3
final long[] fieldValues = new long[numFields];
- final boolean[] fieldHasValue = new boolean[numFields];
- Arrays.fill(fieldHasValue, true);
for (int i = 0; i < fieldValues.length; i++) {
fieldValues[i] = 1;
}
@@ -92,36 +89,14 @@ public class TestMixedDocValuesUpdates e
++docID;
}
- // if field's value was unset before, unset it from all new added documents too
- for (int field = 0; field < fieldHasValue.length; field++) {
- if (!fieldHasValue[field]) {
- if (field < numNDVFields) {
- writer.updateNumericDocValue(new Term("key", "all"), "f" + field, null);
- } else {
- writer.updateBinaryDocValue(new Term("key", "all"), "f" + field, null);
- }
- }
- }
-
int fieldIdx = random.nextInt(fieldValues.length);
String updateField = "f" + fieldIdx;
- if (random.nextBoolean()) {
-// System.out.println("[" + Thread.currentThread().getName() + "]: unset field '" + updateField + "'");
- fieldHasValue[fieldIdx] = false;
- if (fieldIdx < numNDVFields) {
- writer.updateNumericDocValue(new Term("key", "all"), updateField, null);
- } else {
- writer.updateBinaryDocValue(new Term("key", "all"), updateField, null);
- }
+ if (fieldIdx < numNDVFields) {
+ writer.updateNumericDocValue(new Term("key", "all"), updateField, ++fieldValues[fieldIdx]);
} else {
- fieldHasValue[fieldIdx] = true;
- if (fieldIdx < numNDVFields) {
- writer.updateNumericDocValue(new Term("key", "all"), updateField, ++fieldValues[fieldIdx]);
- } else {
- writer.updateBinaryDocValue(new Term("key", "all"), updateField, TestBinaryDocValuesUpdates.toBytes(++fieldValues[fieldIdx]));
- }
-// System.out.println("[" + Thread.currentThread().getName() + "]: updated field '" + updateField + "' to value " + fieldValues[fieldIdx]);
+ writer.updateBinaryDocValue(new Term("key", "all"), updateField, TestBinaryDocValuesUpdates.toBytes(++fieldValues[fieldIdx]));
}
+// System.out.println("[" + Thread.currentThread().getName() + "]: updated field '" + updateField + "' to value " + fieldValues[fieldIdx]);
if (random.nextDouble() < 0.2) {
int deleteDoc = random.nextInt(docID); // might also delete an already deleted document, ok!
@@ -162,15 +137,11 @@ public class TestMixedDocValuesUpdates e
for (int doc = 0; doc < maxDoc; doc++) {
if (liveDocs == null || liveDocs.get(doc)) {
// System.out.println("doc=" + (doc + context.docBase) + " f='" + f + "' vslue=" + getValue(bdv, doc, scratch));
- if (fieldHasValue[field]) {
- assertTrue(docsWithField.get(doc));
- if (field < numNDVFields) {
- assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], ndv.get(doc));
- } else {
- assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], TestBinaryDocValuesUpdates.getValue(bdv, doc, scratch));
- }
+ assertTrue(docsWithField.get(doc));
+ if (field < numNDVFields) {
+ assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], ndv.get(doc));
} else {
- assertFalse(docsWithField.get(doc));
+ assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], TestBinaryDocValuesUpdates.getValue(bdv, doc, scratch));
}
}
}
@@ -231,16 +202,10 @@ public class TestMixedDocValuesUpdates e
else if (group < 0.8) t = new Term("updKey", "g2");
else t = new Term("updKey", "g3");
// System.out.println("[" + Thread.currentThread().getName() + "] numUpdates=" + numUpdates + " updateTerm=" + t);
- if (random.nextBoolean()) { // sometimes unset a value
-// System.err.println("[" + Thread.currentThread().getName() + "] t=" + t + ", f=" + f + ", updValue=UNSET");
- writer.updateBinaryDocValue(t, f, null);
- writer.updateNumericDocValue(t, cf, null);
- } else {
- long updValue = random.nextInt();
-// System.err.println("[" + Thread.currentThread().getName() + "] t=" + t + ", f=" + f + ", updValue=" + updValue);
- writer.updateBinaryDocValue(t, f, TestBinaryDocValuesUpdates.toBytes(updValue));
- writer.updateNumericDocValue(t, cf, updValue * 2);
- }
+ long updValue = random.nextInt();
+// System.err.println("[" + Thread.currentThread().getName() + "] t=" + t + ", f=" + f + ", updValue=" + updValue);
+ writer.updateBinaryDocValue(t, f, TestBinaryDocValuesUpdates.toBytes(updValue));
+ writer.updateNumericDocValue(t, cf, updValue * 2);
if (random.nextDouble() < 0.2) {
// delete a random document
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java?rev=1597604&r1=1597603&r2=1597604&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java Mon May 26 15:19:20 2014
@@ -1,7 +1,6 @@
package org.apache.lucene.index;
import java.io.IOException;
-import java.util.Arrays;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
@@ -444,77 +443,6 @@ public class TestNumericDocValuesUpdates
}
@Test
- public void testUnsetValue() throws Exception {
- assumeTrue("codec does not support docsWithField", defaultCodecSupportsDocsWithField());
- Directory dir = newDirectory();
- IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- IndexWriter writer = new IndexWriter(dir, conf);
-
- for (int i = 0; i < 2; i++) {
- Document doc = new Document();
- doc.add(new StringField("id", "doc" + i, Store.NO));
- doc.add(new NumericDocValuesField("ndv", 5));
- writer.addDocument(doc);
- }
- writer.commit();
-
- // unset the value of 'doc0'
- writer.updateNumericDocValue(new Term("id", "doc0"), "ndv", null);
- writer.close();
-
- final DirectoryReader reader = DirectoryReader.open(dir);
- AtomicReader r = reader.leaves().get(0).reader();
- NumericDocValues ndv = r.getNumericDocValues("ndv");
- for (int i = 0; i < r.maxDoc(); i++) {
- if (i == 0) {
- assertEquals(0, ndv.get(i));
- } else {
- assertEquals(5, ndv.get(i));
- }
- }
-
- Bits docsWithField = r.getDocsWithField("ndv");
- assertFalse(docsWithField.get(0));
- assertTrue(docsWithField.get(1));
-
- reader.close();
- dir.close();
- }
-
- public void testUnsetAllValues() throws Exception {
- assumeTrue("codec does not support docsWithField", defaultCodecSupportsDocsWithField());
- Directory dir = newDirectory();
- IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- IndexWriter writer = new IndexWriter(dir, conf);
-
- for (int i = 0; i < 2; i++) {
- Document doc = new Document();
- doc.add(new StringField("id", "doc", Store.NO));
- doc.add(new NumericDocValuesField("ndv", 5));
- writer.addDocument(doc);
- }
- writer.commit();
-
- // unset the value of 'doc'
- writer.updateNumericDocValue(new Term("id", "doc"), "ndv", null);
- writer.close();
-
- final DirectoryReader reader = DirectoryReader.open(dir);
- AtomicReader r = reader.leaves().get(0).reader();
- NumericDocValues ndv = r.getNumericDocValues("ndv");
- for (int i = 0; i < r.maxDoc(); i++) {
- assertEquals(0, ndv.get(i));
- }
-
- Bits docsWithField = r.getDocsWithField("ndv");
- assertFalse(docsWithField.get(0));
- assertFalse(docsWithField.get(1));
-
- reader.close();
- dir.close();
- }
-
- @Test
public void testUpdateNonNumericDocValuesField() throws Exception {
// we don't support adding new fields or updating existing non-numeric-dv
// fields through numeric updates
@@ -734,8 +662,6 @@ public class TestNumericDocValuesUpdates
final int numFields = random.nextInt(4) + 3; // 3-7
final long[] fieldValues = new long[numFields];
- final boolean[] fieldHasValue = new boolean[numFields];
- Arrays.fill(fieldHasValue, true);
for (int i = 0; i < fieldValues.length; i++) {
fieldValues[i] = 1;
}
@@ -757,24 +683,10 @@ public class TestNumericDocValuesUpdates
++docID;
}
- // if field's value was unset before, unset it from all new added documents too
- for (int field = 0; field < fieldHasValue.length; field++) {
- if (!fieldHasValue[field]) {
- writer.updateNumericDocValue(new Term("key", "all"), "f" + field, null);
- }
- }
-
int fieldIdx = random.nextInt(fieldValues.length);
String updateField = "f" + fieldIdx;
- if (random.nextBoolean()) {
-// System.out.println("[" + Thread.currentThread().getName() + "]: unset field '" + updateField + "'");
- fieldHasValue[fieldIdx] = false;
- writer.updateNumericDocValue(new Term("key", "all"), updateField, null);
- } else {
- fieldHasValue[fieldIdx] = true;
- writer.updateNumericDocValue(new Term("key", "all"), updateField, ++fieldValues[fieldIdx]);
-// System.out.println("[" + Thread.currentThread().getName() + "]: updated field '" + updateField + "' to value " + fieldValues[fieldIdx]);
- }
+ writer.updateNumericDocValue(new Term("key", "all"), updateField, ++fieldValues[fieldIdx]);
+// System.out.println("[" + Thread.currentThread().getName() + "]: updated field '" + updateField + "' to value " + fieldValues[fieldIdx]);
if (random.nextDouble() < 0.2) {
int deleteDoc = random.nextInt(docID); // might also delete an already deleted document, ok!
@@ -807,12 +719,8 @@ public class TestNumericDocValuesUpdates
for (int doc = 0; doc < maxDoc; doc++) {
if (liveDocs == null || liveDocs.get(doc)) {
// System.out.println("doc=" + (doc + context.docBase) + " f='" + f + "' vslue=" + ndv.get(doc));
- if (fieldHasValue[field]) {
- assertTrue(docsWithField.get(doc));
- assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], ndv.get(doc));
- } else {
- assertFalse(docsWithField.get(doc));
- }
+ assertTrue(docsWithField.get(doc));
+ assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], ndv.get(doc));
}
}
}
@@ -1026,14 +934,9 @@ public class TestNumericDocValuesUpdates
else if (group < 0.8) t = new Term("updKey", "g2");
else t = new Term("updKey", "g3");
// System.out.println("[" + Thread.currentThread().getName() + "] numUpdates=" + numUpdates + " updateTerm=" + t);
- if (random.nextBoolean()) { // sometimes unset a value
- writer.updateNumericDocValue(t, f, null);
- writer.updateNumericDocValue(t, cf, null);
- } else {
- long updValue = random.nextInt();
- writer.updateNumericDocValue(t, f, updValue);
- writer.updateNumericDocValue(t, cf, updValue * 2);
- }
+ long updValue = random.nextInt();
+ writer.updateNumericDocValue(t, f, updValue);
+ writer.updateNumericDocValue(t, cf, updValue * 2);
if (random.nextDouble() < 0.2) {
// delete a random document
@@ -1096,10 +999,9 @@ public class TestNumericDocValuesUpdates
Bits liveDocs = r.getLiveDocs();
for (int j = 0; j < r.maxDoc(); j++) {
if (liveDocs == null || liveDocs.get(j)) {
- assertEquals(docsWithNdv.get(j), docsWithControl.get(j));
- if (docsWithNdv.get(j)) {
- assertEquals(control.get(j), ndv.get(j) * 2);
- }
+ assertTrue(docsWithNdv.get(j));
+ assertTrue(docsWithControl.get(j));
+ assertEquals(control.get(j), ndv.get(j) * 2);
}
}
}