You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2018/12/07 07:28:06 UTC
lucene-solr:branch_7_6: LUCENE-8595: Fix interleaved DV update and
reset
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7_6 2d4435162 -> 10cadbe3b
LUCENE-8595: Fix interleaved DV update and reset
This change fixes a bug where interleaved update and reset value
to the same doc in the same updates package looses an update
if the reset comes before the update as well as loosing the reset
if the update comes frist.
Co-authored-by: Adrien Grand <jp...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/10cadbe3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/10cadbe3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/10cadbe3
Branch: refs/heads/branch_7_6
Commit: 10cadbe3bb545c122cb68f4434dfbaa8a8cb77e7
Parents: 2d44351
Author: Simon Willnauer <si...@apache.org>
Authored: Thu Dec 6 21:27:13 2018 +0100
Committer: Simon Willnauer <si...@apache.org>
Committed: Fri Dec 7 08:27:49 2018 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 ++
.../lucene/index/DocValuesFieldUpdates.java | 10 +++-
.../lucene/index/TestDocValuesFieldUpdates.java | 63 ++++++++++++++++++++
3 files changed, 74 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/10cadbe3/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 2ceb0c6..e548ed1 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -52,6 +52,10 @@ Bug fixes
* LUCENE-8589: Fix MultiPhraseQuery to not duplicate terms when building the phrase's weight.
(Jim Ferenczi)
+* LUCENE-8595: Fix interleaved DV update and reset. Interleaved update and reset value
+ to the same doc in the same updates package looses an update if the reset comes before
+ the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grant)
+
New Features
* LUCENE-8496: Selective indexing - modify BKDReader/BKDWriter to allow users
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/10cadbe3/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
index 093a428..058b80a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
@@ -298,7 +298,7 @@ abstract class DocValuesFieldUpdates implements Accountable {
// increasing docID order:
// NOTE: we can have ties here, when the same docID was updated in the same segment, in which case we rely on sort being
// stable and preserving original order so the last update to that docID wins
- return Long.compare(docs.get(i), docs.get(j));
+ return Long.compare(docs.get(i)>>>1, docs.get(j)>>>1);
}
}.sort(0, size);
}
@@ -392,9 +392,13 @@ abstract class DocValuesFieldUpdates implements Accountable {
}
long longDoc = docs.get(idx);
++idx;
- while (idx < size && docs.get(idx) == longDoc) {
+ for (; idx < size; idx++) {
// scan forward to last update to this doc
- ++idx;
+ final long nextLongDoc = docs.get(idx);
+ if ((longDoc >>> 1) != (nextLongDoc >>> 1)) {
+ break;
+ }
+ longDoc = nextLongDoc;
}
hasValue = (longDoc & HAS_VALUE_MASK) > 0;
if (hasValue) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/10cadbe3/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFieldUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFieldUpdates.java
index 2066dcd..6c60df3 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFieldUpdates.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFieldUpdates.java
@@ -69,4 +69,67 @@ public class TestDocValuesFieldUpdates extends LuceneTestCase {
assertEquals(24, iterator.longValue());
assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc());
}
+
+ public void testUpdateAndResetSameDoc() {
+ NumericDocValuesFieldUpdates updates = new NumericDocValuesFieldUpdates(0, "test", 2);
+ updates.add(0, 1);
+ updates.reset(0);
+ updates.finish();
+ NumericDocValuesFieldUpdates.Iterator iterator = updates.iterator();
+ assertEquals(0, iterator.nextDoc());
+ assertFalse(iterator.hasValue());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc());
+ }
+
+ public void testUpdateAndResetUpdateSameDoc() {
+ NumericDocValuesFieldUpdates updates = new NumericDocValuesFieldUpdates(0, "test", 3);
+ updates.add(0, 1);
+ updates.add(0);
+ updates.add(0, 2);
+ updates.finish();
+ NumericDocValuesFieldUpdates.Iterator iterator = updates.iterator();
+ assertEquals(0, iterator.nextDoc());
+ assertTrue(iterator.hasValue());
+ assertEquals(2, iterator.longValue());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc());
+ }
+
+ public void testUpdatesAndResetRandom() {
+ NumericDocValuesFieldUpdates updates = new NumericDocValuesFieldUpdates(0, "test", 10);
+ int numUpdates = 10 + random().nextInt(100);
+ Integer[] values = new Integer[5];
+ for (int i = 0; i < 5; i++) {
+ values[i] = random().nextBoolean() ? null : random().nextInt(100);
+ if (values[i] == null) {
+ updates.reset(i);
+ } else {
+ updates.add(i, values[i]);
+ }
+ }
+ for (int i = 0; i < numUpdates; i++) {
+ int docId = random().nextInt(5);
+ values[docId] = random().nextBoolean() ? null : random().nextInt(100);
+ if (values[docId] == null) {
+ updates.reset(docId);
+ } else {
+ updates.add(docId, values[docId]);
+ }
+ }
+
+ updates.finish();
+ NumericDocValuesFieldUpdates.Iterator iterator = updates.iterator();
+ int idx = 0;
+ while (iterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+ assertEquals(idx, iterator.docID());
+ if (values[idx] == null) {
+ assertFalse(iterator.hasValue());
+ } else {
+ assertTrue(iterator.hasValue());
+ assertEquals(values[idx].longValue(), iterator.longValue());
+ }
+ idx++;
+ }
+
+
+ }
}