You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2018/12/12 12:22:08 UTC
[1/4] lucene-solr:master: Revert "test"
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x a3c0def52 -> 0e525b40c
refs/heads/master ef2f0cd88 -> 55993ecb9
Revert "test"
This reverts commit 783f01bab3e096f4f843b9de62af25d6fbe07340.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/21dc2372
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/21dc2372
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/21dc2372
Branch: refs/heads/master
Commit: 21dc2372486177d87be4c84742306328eccd0ed4
Parents: adf73ff
Author: Adrien Grand <jp...@gmail.com>
Authored: Tue Dec 11 10:11:27 2018 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Wed Dec 12 13:20:37 2018 +0100
----------------------------------------------------------------------
.../index/BinaryDocValuesFieldUpdates.java | 8 ---
.../lucene/index/DocValuesFieldUpdates.java | 56 ++++----------------
.../index/NumericDocValuesFieldUpdates.java | 6 ---
3 files changed, 10 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/21dc2372/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
index 67227ed..fa60929 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
@@ -108,14 +108,6 @@ final class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
}
@Override
- protected void copy(int i, int j) {
- super.copy(i, j);
-
- offsets.set(j, offsets.get(i));
- lengths.set(j, lengths.get(i));
- }
-
- @Override
protected void grow(int size) {
super.grow(size);
offsets = offsets.grow(size);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/21dc2372/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
index 71d9c7d..464a7f2 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
@@ -21,10 +21,10 @@ import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.InPlaceMergeSorter;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.SparseFixedBitSet;
-import org.apache.lucene.util.TimSorter;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PagedMutable;
@@ -32,11 +32,11 @@ import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
/**
* Holds updates of a single DocValues field, for a set of documents within one segment.
- *
+ *
* @lucene.experimental
*/
abstract class DocValuesFieldUpdates implements Accountable {
-
+
protected static final int PAGE_SIZE = 1024;
private static final long HAS_VALUE_MASK = 1;
private static final long HAS_NO_VALUE_MASK = 0;
@@ -260,7 +260,7 @@ abstract class DocValuesFieldUpdates implements Accountable {
final boolean getFinished() {
return finished;
}
-
+
abstract void add(int doc, long value);
abstract void add(int doc, BytesRef value);
@@ -285,17 +285,11 @@ abstract class DocValuesFieldUpdates implements Accountable {
throw new IllegalStateException("already finished");
}
finished = true;
-
- // Swaps and comparisons are quite costly with all these packed ints,
- // so we use TimSorter instead of InPlaceMergeSorter, which has the nice
- // property of doing fewer swaps/copies at the expense of a higher memory
- // usage
- final int tempSlotsOffset = size;
- // we grow the current structures to get temporary storage
- resize(Math.addExact(size, size / 2)); // 50% extra memory for temp slots
- final int numTempSlots = Math.toIntExact(docs.size() - size);
- new TimSorter(numTempSlots) {
-
+ // shrink wrap
+ if (size < docs.size()) {
+ resize(size);
+ }
+ new InPlaceMergeSorter() {
@Override
protected void swap(int i, int j) {
DocValuesFieldUpdates.this.swap(i, j);
@@ -308,33 +302,7 @@ abstract class DocValuesFieldUpdates implements Accountable {
// stable and preserving original order so the last update to that docID wins
return Long.compare(docs.get(i)>>>1, docs.get(j)>>>1);
}
-
- @Override
- protected void save(int i, int len) {
- assert len <= numTempSlots;
- for (int k = 0; k < len; ++k) {
- copy(i + k, tempSlotsOffset + k);
- }
- }
-
- @Override
- protected void restore(int src, int dest) {
- copy(tempSlotsOffset + src, dest);
- }
-
- @Override
- protected void copy(int src, int dest) {
- DocValuesFieldUpdates.this.copy(src, dest);
- }
-
- @Override
- protected int compareSaved(int i, int j) {
- return compare(tempSlotsOffset + i, j);
- }
}.sort(0, size);
-
- // shrink wrap to save memory, this will also release temporary storage
- resize(size);
}
/** Returns true if this instance contains any updates. */
@@ -382,10 +350,6 @@ abstract class DocValuesFieldUpdates implements Accountable {
docs.set(i, tmpDoc);
}
- protected void copy(int i, int j) {
- docs.set(j, docs.get(i));
- }
-
protected void grow(int size) {
docs = docs.grow(size);
}
@@ -513,7 +477,7 @@ abstract class DocValuesFieldUpdates implements Accountable {
}
protected abstract BytesRef binaryValue();
-
+
protected abstract long longValue();
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/21dc2372/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
index 05d5c14..550a86a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
@@ -100,12 +100,6 @@ final class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
}
@Override
- protected void copy(int i, int j) {
- super.copy(i, j);
- values.set(j, values.get(i));
- }
-
- @Override
protected void grow(int size) {
super.grow(size);
values = values.grow(size);
[4/4] lucene-solr:branch_7x: Fix test failure when floorSegmentMB >
maxMergedSegmentMB.
Posted by jp...@apache.org.
Fix test failure when floorSegmentMB > maxMergedSegmentMB.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0e525b40
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0e525b40
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0e525b40
Branch: refs/heads/branch_7x
Commit: 0e525b40ce1e28cf85ee401043e2818760997ecf
Parents: a3c0def
Author: Adrien Grand <jp...@gmail.com>
Authored: Wed Dec 12 13:21:06 2018 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Wed Dec 12 13:21:47 2018 +0100
----------------------------------------------------------------------
.../src/test/org/apache/lucene/index/TestTieredMergePolicy.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0e525b40/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java
index b4a7ff5..bbc8051 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java
@@ -169,7 +169,8 @@ public class TestTieredMergePolicy extends BaseMergePolicyTestCase {
}
w.forceMerge(targetCount);
- final long max125Pct = (long) ((tmp.getMaxMergedSegmentMB() * 1024.0 * 1024.0) * 1.25);
+ final double maxSegmentSize = Math.max(tmp.getMaxMergedSegmentMB(), tmp.getFloorSegmentMB());
+ final long max125Pct = (long) ((maxSegmentSize * 1024.0 * 1024.0) * 1.25);
// Other than in the case where the target count is 1 we can't say much except no segment should be > 125% of max seg size.
if (targetCount == 1) {
assertEquals("Should have merged down to one segment", targetCount, w.getSegmentCount());
[3/4] lucene-solr:master: Fix test failure when floorSegmentMB >
maxMergedSegmentMB.
Posted by jp...@apache.org.
Fix test failure when floorSegmentMB > maxMergedSegmentMB.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/55993ecb
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/55993ecb
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/55993ecb
Branch: refs/heads/master
Commit: 55993ecb9b06d9c0d2bd54a752a08f19698a13bc
Parents: 21dc237
Author: Adrien Grand <jp...@gmail.com>
Authored: Wed Dec 12 13:21:06 2018 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Wed Dec 12 13:21:06 2018 +0100
----------------------------------------------------------------------
.../src/test/org/apache/lucene/index/TestTieredMergePolicy.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/55993ecb/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java
index b4a7ff5..bbc8051 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java
@@ -169,7 +169,8 @@ public class TestTieredMergePolicy extends BaseMergePolicyTestCase {
}
w.forceMerge(targetCount);
- final long max125Pct = (long) ((tmp.getMaxMergedSegmentMB() * 1024.0 * 1024.0) * 1.25);
+ final double maxSegmentSize = Math.max(tmp.getMaxMergedSegmentMB(), tmp.getFloorSegmentMB());
+ final long max125Pct = (long) ((maxSegmentSize * 1024.0 * 1024.0) * 1.25);
// Other than in the case where the target count is 1 we can't say much except no segment should be > 125% of max seg size.
if (targetCount == 1) {
assertEquals("Should have merged down to one segment", targetCount, w.getSegmentCount());
[2/4] lucene-solr:master: test
Posted by jp...@apache.org.
test
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/adf73ff6
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/adf73ff6
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/adf73ff6
Branch: refs/heads/master
Commit: adf73ff6bd49bd09701e8a960cdcfb492cd3c6e0
Parents: ef2f0cd
Author: Adrien Grand <jp...@gmail.com>
Authored: Tue Dec 11 10:11:04 2018 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Wed Dec 12 13:20:37 2018 +0100
----------------------------------------------------------------------
.../index/BinaryDocValuesFieldUpdates.java | 8 +++
.../lucene/index/DocValuesFieldUpdates.java | 56 ++++++++++++++++----
.../index/NumericDocValuesFieldUpdates.java | 6 +++
3 files changed, 60 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/adf73ff6/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
index fa60929..67227ed 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
@@ -108,6 +108,14 @@ final class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
}
@Override
+ protected void copy(int i, int j) {
+ super.copy(i, j);
+
+ offsets.set(j, offsets.get(i));
+ lengths.set(j, lengths.get(i));
+ }
+
+ @Override
protected void grow(int size) {
super.grow(size);
offsets = offsets.grow(size);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/adf73ff6/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
index 464a7f2..71d9c7d 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
@@ -21,10 +21,10 @@ import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.InPlaceMergeSorter;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.SparseFixedBitSet;
+import org.apache.lucene.util.TimSorter;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PagedMutable;
@@ -32,11 +32,11 @@ import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
/**
* Holds updates of a single DocValues field, for a set of documents within one segment.
- *
+ *
* @lucene.experimental
*/
abstract class DocValuesFieldUpdates implements Accountable {
-
+
protected static final int PAGE_SIZE = 1024;
private static final long HAS_VALUE_MASK = 1;
private static final long HAS_NO_VALUE_MASK = 0;
@@ -260,7 +260,7 @@ abstract class DocValuesFieldUpdates implements Accountable {
final boolean getFinished() {
return finished;
}
-
+
abstract void add(int doc, long value);
abstract void add(int doc, BytesRef value);
@@ -285,11 +285,17 @@ abstract class DocValuesFieldUpdates implements Accountable {
throw new IllegalStateException("already finished");
}
finished = true;
- // shrink wrap
- if (size < docs.size()) {
- resize(size);
- }
- new InPlaceMergeSorter() {
+
+ // Swaps and comparisons are quite costly with all these packed ints,
+ // so we use TimSorter instead of InPlaceMergeSorter, which has the nice
+ // property of doing fewer swaps/copies at the expense of a higher memory
+ // usage
+ final int tempSlotsOffset = size;
+ // we grow the current structures to get temporary storage
+ resize(Math.addExact(size, size / 2)); // 50% extra memory for temp slots
+ final int numTempSlots = Math.toIntExact(docs.size() - size);
+ new TimSorter(numTempSlots) {
+
@Override
protected void swap(int i, int j) {
DocValuesFieldUpdates.this.swap(i, j);
@@ -302,7 +308,33 @@ abstract class DocValuesFieldUpdates implements Accountable {
// stable and preserving original order so the last update to that docID wins
return Long.compare(docs.get(i)>>>1, docs.get(j)>>>1);
}
+
+ @Override
+ protected void save(int i, int len) {
+ assert len <= numTempSlots;
+ for (int k = 0; k < len; ++k) {
+ copy(i + k, tempSlotsOffset + k);
+ }
+ }
+
+ @Override
+ protected void restore(int src, int dest) {
+ copy(tempSlotsOffset + src, dest);
+ }
+
+ @Override
+ protected void copy(int src, int dest) {
+ DocValuesFieldUpdates.this.copy(src, dest);
+ }
+
+ @Override
+ protected int compareSaved(int i, int j) {
+ return compare(tempSlotsOffset + i, j);
+ }
}.sort(0, size);
+
+ // shrink wrap to save memory, this will also release temporary storage
+ resize(size);
}
/** Returns true if this instance contains any updates. */
@@ -350,6 +382,10 @@ abstract class DocValuesFieldUpdates implements Accountable {
docs.set(i, tmpDoc);
}
+ protected void copy(int i, int j) {
+ docs.set(j, docs.get(i));
+ }
+
protected void grow(int size) {
docs = docs.grow(size);
}
@@ -477,7 +513,7 @@ abstract class DocValuesFieldUpdates implements Accountable {
}
protected abstract BytesRef binaryValue();
-
+
protected abstract long longValue();
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/adf73ff6/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
index 550a86a..05d5c14 100644
--- a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
@@ -100,6 +100,12 @@ final class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
}
@Override
+ protected void copy(int i, int j) {
+ super.copy(i, j);
+ values.set(j, values.get(i));
+ }
+
+ @Override
protected void grow(int size) {
super.grow(size);
values = values.grow(size);