You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2026/05/28 13:00:22 UTC
(lucene) branch branch_10x updated: Optimize constant-score bulk scoring via intoBitSet batching (#16141)
This is an automated email from the ASF dual-hosted git repository.
romseygeek pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new 614ffdd6ff5 Optimize constant-score bulk scoring via intoBitSet batching (#16141)
614ffdd6ff5 is described below
commit 614ffdd6ff54fa7ae67b08c760222943f554bf01
Author: Costin Leau <co...@users.noreply.github.com>
AuthorDate: Thu May 28 05:50:12 2026 -0700
Optimize constant-score bulk scoring via intoBitSet batching (#16141)
DefaultBulkScorer iterates doc-by-doc calling collector.collect(int) for
each match. When the scorer is a ConstantScoreScorer with no two-phase
iterator and no competitive iterator, we can instead batch doc IDs into a
FixedBitSet window via intoBitSet, apply the live-docs mask in bulk, and
hand the result as a DocIdStream to the collector.
---
lucene/CHANGES.txt | 2 +
.../benchmark/jmh/DefaultBulkScorerBenchmark.java | 222 +++++++++++++++++++++
.../lucene/search/ConstantScoreBulkScorer.java | 125 ++++++++++++
.../lucene/search/ConstantScoreScorerSupplier.java | 7 +-
.../apache/lucene/search/TestBooleanScorer.java | 138 ++++++++++++-
5 files changed, 489 insertions(+), 5 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 3ffb25b2292..c968ddb2af3 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -184,6 +184,8 @@ Optimizations
* GITHUB#16126: Implement #intoBitSet and #docIDRunEnd in DocValuesWriter implementations. (Ignacio Vera)
+* GITHUB#16141: Optimize constant-score bulk scoring via intoBitSet batching. (Costin Leau)
+
Bug Fixes
---------------------
* GITHUB#15754: Fix HTMLStripCharFilter to prevent tags from incorrectly consuming subsequent
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/DefaultBulkScorerBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/DefaultBulkScorerBenchmark.java
new file mode 100644
index 00000000000..6787d92f4a5
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/DefaultBulkScorerBenchmark.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.search.BulkScorer;
+import org.apache.lucene.search.ConstantScoreScorer;
+import org.apache.lucene.search.ConstantScoreScorerSupplier;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.DocIdStream;
+import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.search.Scorable;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.util.BitSetIterator;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.FixedBitSet;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+/** Compares the old per-doc constant-score loop with batched DocIdStream collection. */
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@Threads(1)
+@Warmup(iterations = 5, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(
+ value = 1,
+ jvmArgsAppend = {"-Xmx1g", "-Xms1g", "-XX:+AlwaysPreTouch"})
+public class DefaultBulkScorerBenchmark {
+
+ @Param({"1000000"})
+ int maxDoc;
+
+ @Param({"0.01", "0.1", "0.5", "1.0"})
+ double density;
+
+ @Param({"false", "true"})
+ boolean withDeletions;
+
+ private FixedBitSet matchingDocs;
+ private FixedBitSet liveDocs;
+ private int matchCount;
+ private int expectedHitCount;
+ private Bits acceptDocs;
+ private BitSetIterator baselineIterator;
+ private BitSetIterator intoBitSetIterator;
+ private BulkScorer baselineScorer;
+ private BulkScorer intoBitSetScorer;
+ private CountingLeafCollector baselineCollector;
+ private DocIdStreamCountingLeafCollector intoBitSetCollector;
+
+ @Setup(Level.Trial)
+ public void setupTrial() throws IOException {
+ matchingDocs = new FixedBitSet(maxDoc);
+ int gap = Math.max(1, (int) Math.round(1d / density));
+ for (int doc = 0; doc < maxDoc; doc += gap) {
+ matchingDocs.set(doc);
+ }
+ matchCount = matchingDocs.cardinality();
+
+ liveDocs = new FixedBitSet(maxDoc);
+ liveDocs.set(0, maxDoc);
+
+ for (int doc = 0, matchOrd = 0; doc < maxDoc; doc += gap, matchOrd++) {
+ if (matchOrd % 10 == 0) {
+ liveDocs.clear(doc);
+ }
+ }
+
+ expectedHitCount = matchCount;
+ if (withDeletions) {
+ expectedHitCount = 0;
+ for (int doc = 0; doc < maxDoc; doc += gap) {
+ if (liveDocs.get(doc)) {
+ expectedHitCount++;
+ }
+ }
+ acceptDocs = liveDocs;
+ } else {
+ acceptDocs = null;
+ }
+
+ baselineIterator = new BitSetIterator(matchingDocs, matchCount);
+ intoBitSetIterator = new BitSetIterator(matchingDocs, matchCount);
+ baselineScorer = newBaselineBulkScorer(baselineIterator);
+ intoBitSetScorer = newConstantScoreBulkScorer(intoBitSetIterator, maxDoc);
+ baselineCollector = new CountingLeafCollector();
+ intoBitSetCollector = new DocIdStreamCountingLeafCollector();
+ }
+
+ @Setup(Level.Invocation)
+ public void setupInvocation() {
+ baselineIterator.setDocId(-1);
+ intoBitSetIterator.setDocId(-1);
+ baselineCollector.count = 0;
+ intoBitSetCollector.count = 0;
+ }
+
+ @Benchmark
+ public int scoreBaseline() throws IOException {
+ baselineScorer.score(baselineCollector, acceptDocs, 0, maxDoc);
+ checkCount("baseline", baselineCollector.count);
+ return baselineCollector.count;
+ }
+
+ @Benchmark
+ public int scoreIntoBitSet() throws IOException {
+ intoBitSetScorer.score(intoBitSetCollector, acceptDocs, 0, maxDoc);
+ checkCount("intoBitSet", intoBitSetCollector.count);
+ return intoBitSetCollector.count;
+ }
+
+ private void checkCount(String method, int count) {
+ if (count != expectedHitCount) {
+ throw new AssertionError(
+ "Expected " + expectedHitCount + " " + method + " hits but got " + count);
+ }
+ }
+
+ private static BulkScorer newBaselineBulkScorer(DocIdSetIterator iterator) {
+ return new PerDocBulkScorer(
+ new ConstantScoreScorer(1f, ScoreMode.COMPLETE_NO_SCORES, iterator));
+ }
+
+ private static BulkScorer newConstantScoreBulkScorer(DocIdSetIterator iterator, int maxDoc)
+ throws IOException {
+ return ConstantScoreScorerSupplier.fromIterator(
+ iterator, 1f, ScoreMode.COMPLETE_NO_SCORES, maxDoc)
+ .bulkScorer();
+ }
+
+ private static final class PerDocBulkScorer extends BulkScorer {
+ private final Scorer scorer;
+ private final DocIdSetIterator iterator;
+
+ private PerDocBulkScorer(Scorer scorer) {
+ this.scorer = scorer;
+ this.iterator = scorer.iterator();
+ }
+
+ @Override
+ public int score(LeafCollector collector, Bits acceptDocs, int min, int max)
+ throws IOException {
+ collector.setScorer(scorer);
+ if (iterator.docID() < min) {
+ if (iterator.docID() == min - 1) {
+ iterator.nextDoc();
+ } else {
+ iterator.advance(min);
+ }
+ }
+ for (int doc = iterator.docID(); doc < max; doc = iterator.nextDoc()) {
+ if (acceptDocs == null || acceptDocs.get(doc)) {
+ collector.collect(doc);
+ }
+ }
+ return iterator.docID();
+ }
+
+ @Override
+ public long cost() {
+ return iterator.cost();
+ }
+ }
+
+ private static class CountingLeafCollector implements LeafCollector {
+ int count;
+
+ @Override
+ public void setScorer(Scorable scorer) {}
+
+ @Override
+ public void collect(int doc) {
+ count++;
+ }
+
+ @Override
+ public void collect(DocIdStream stream) throws IOException {
+ throw new AssertionError("Baseline benchmark must use per-doc collection");
+ }
+ }
+
+ private static final class DocIdStreamCountingLeafCollector extends CountingLeafCollector {
+ @Override
+ public void collect(int doc) {
+ throw new AssertionError("Constant score bulk scoring must use DocIdStream collection");
+ }
+
+ @Override
+ public void collect(DocIdStream stream) throws IOException {
+ count += stream.count();
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreBulkScorer.java
new file mode 100644
index 00000000000..024e2f911a4
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreBulkScorer.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.MathUtil;
+
+/**
+ * Bulk scorer for no-score constant-score iterators that batches doc IDs via {@link
+ * DocIdSetIterator#intoBitSet}.
+ */
+final class ConstantScoreBulkScorer extends BulkScorer {
+ private final Scorer scorer;
+ private final DocIdSetIterator iterator;
+ private final FixedBitSet windowMatches = new FixedBitSet(DenseConjunctionBulkScorer.WINDOW_SIZE);
+
+ ConstantScoreBulkScorer(float score, ScoreMode scoreMode, DocIdSetIterator iterator) {
+ if (scoreMode.needsScores()) {
+ throw new IllegalArgumentException("ScoreMode must not need scores: " + scoreMode);
+ }
+ if (TwoPhaseIterator.unwrap(iterator) != null) {
+ throw new IllegalArgumentException("Iterator must not wrap a TwoPhaseIterator");
+ }
+ this.scorer = new ConstantScoreScorer(score, scoreMode, iterator);
+ this.iterator = iterator;
+ }
+
+ @Override
+ public int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException {
+ collector.setScorer(scorer);
+ DocIdSetIterator competitiveIterator = collector.competitiveIterator();
+ if (competitiveIterator != null) {
+ scoreIterator(collector, acceptDocs, iterator, competitiveIterator, min, max);
+ } else {
+ scoreIteratorIntoBitSet(collector, acceptDocs, iterator, min, max);
+ }
+ return iterator.docID();
+ }
+
+ @Override
+ public long cost() {
+ return iterator.cost();
+ }
+
+ private static void scoreIterator(
+ LeafCollector collector,
+ Bits acceptDocs,
+ DocIdSetIterator iterator,
+ DocIdSetIterator competitiveIterator,
+ int min,
+ int max)
+ throws IOException {
+ if (competitiveIterator.docID() > min) {
+ min = Math.min(competitiveIterator.docID(), max);
+ }
+ if (iterator.docID() < min) {
+ if (iterator.docID() == min - 1) {
+ iterator.nextDoc();
+ } else {
+ iterator.advance(min);
+ }
+ }
+ for (int doc = iterator.docID(); doc < max; ) {
+ assert competitiveIterator.docID() <= doc; // invariant
+ if (competitiveIterator.docID() < doc) {
+ int competitiveNext = competitiveIterator.advance(doc);
+ if (competitiveNext != doc) {
+ doc = iterator.advance(competitiveNext);
+ continue;
+ }
+ }
+
+ if (acceptDocs == null || acceptDocs.get(doc)) {
+ collector.collect(doc);
+ }
+
+ doc = iterator.nextDoc();
+ }
+ }
+
+ private void scoreIteratorIntoBitSet(
+ LeafCollector collector, Bits acceptDocs, DocIdSetIterator iterator, int min, int max)
+ throws IOException {
+ if (iterator.docID() < min) {
+ if (iterator.docID() == min - 1) {
+ iterator.nextDoc();
+ } else {
+ iterator.advance(min);
+ }
+ }
+ for (int doc = iterator.docID(); doc < max; ) {
+ int windowBase = doc;
+ int windowMax =
+ MathUtil.unsignedMin(max, windowBase + DenseConjunctionBulkScorer.WINDOW_SIZE);
+
+ assert windowMatches.scanIsEmpty();
+ iterator.intoBitSet(windowMax, windowMatches, windowBase);
+
+ if (acceptDocs != null) {
+ acceptDocs.applyMask(windowMatches, windowBase);
+ }
+
+ collector.collect(new BitSetDocIdStream(windowMatches, windowBase));
+ windowMatches.clear();
+
+ doc = iterator.docID();
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreScorerSupplier.java b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreScorerSupplier.java
index 3674a479de3..6df6d8871c9 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreScorerSupplier.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreScorerSupplier.java
@@ -19,7 +19,6 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
-import org.apache.lucene.search.Weight.DefaultBulkScorer;
/**
* Specialization of {@link ScorerSupplier} for queries that produce constant scores.
@@ -78,9 +77,9 @@ public abstract class ConstantScoreScorerSupplier extends ScorerSupplier {
@Override
public final BulkScorer bulkScorer() throws IOException {
DocIdSetIterator iterator = iterator(Long.MAX_VALUE);
+ TwoPhaseIterator twoPhase = TwoPhaseIterator.unwrap(iterator);
if (maxDoc >= DenseConjunctionBulkScorer.WINDOW_SIZE / 2
&& iterator.cost() >= maxDoc / DenseConjunctionBulkScorer.DENSITY_THRESHOLD_INVERSE) {
- TwoPhaseIterator twoPhase = TwoPhaseIterator.unwrap(iterator);
List<DocIdSetIterator> iterators;
List<TwoPhaseIterator> twoPhases;
if (twoPhase == null) {
@@ -91,8 +90,10 @@ public abstract class ConstantScoreScorerSupplier extends ScorerSupplier {
twoPhases = Collections.singletonList(twoPhase);
}
return new DenseConjunctionBulkScorer(iterators, twoPhases, maxDoc, score);
+ } else if (scoreMode.needsScores() == false && twoPhase == null) {
+ return new ConstantScoreBulkScorer(score, scoreMode, iterator);
} else {
- return new DefaultBulkScorer(new ConstantScoreScorer(score, scoreMode, iterator));
+ return new Weight.DefaultBulkScorer(new ConstantScoreScorer(score, scoreMode, iterator));
}
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java
index 69e0a8b3bda..b1bd4fe5c3c 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java
@@ -37,7 +37,9 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.search.QueryUtils;
import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.FixedBitSet;
public class TestBooleanScorer extends LuceneTestCase {
private static final String FIELD = "category";
@@ -189,11 +191,11 @@ public class TestBooleanScorer extends LuceneTestCase {
.add(new TermQuery(new Term("missing_field", "baz")), Occur.SHOULD) // missing term
.build();
- // no scores -> term scorer
+ // no scores -> constant-score term scorer
Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1);
ScorerSupplier ss = weight.scorerSupplier(ctx);
BulkScorer scorer = ((BooleanScorerSupplier) ss).booleanScorer();
- assertThat(scorer, instanceOf(DefaultBulkScorer.class)); // term scorer
+ assertThat(scorer, instanceOf(ConstantScoreBulkScorer.class)); // term scorer
// scores -> term scorer too
query =
@@ -211,6 +213,138 @@ public class TestBooleanScorer extends LuceneTestCase {
dir.close();
}
+ public void testConstantScoreScorerSupplierUsesIntoBitSet() throws IOException {
+ int[] docs = {1, 2, 4097, 5000, 8195};
+ CountingDocIdSetIterator iterator = new CountingDocIdSetIterator(docs);
+
+ FixedBitSet liveDocs = new FixedBitSet(9000);
+ liveDocs.set(0, 9000);
+ liveDocs.clear(2);
+ liveDocs.clear(4097);
+ liveDocs.clear(5000);
+
+ int[] collected = new int[docs.length];
+ int[] count = new int[1];
+ LeafCollector collector =
+ new LeafCollector() {
+ @Override
+ public void setScorer(Scorable scorer) {}
+
+ @Override
+ public void collect(int doc) {
+ fail("ConstantScoreBulkScorer should collect via DocIdStream");
+ }
+
+ @Override
+ public void collect(DocIdStream stream) throws IOException {
+ int[] buffer = new int[docs.length];
+ for (int size = stream.intoArray(buffer); size != 0; size = stream.intoArray(buffer)) {
+ System.arraycopy(buffer, 0, collected, count[0], size);
+ count[0] += size;
+ }
+ }
+ };
+
+ BulkScorer bulkScorer =
+ ConstantScoreScorerSupplier.fromIterator(iterator, 1f, ScoreMode.COMPLETE_NO_SCORES, 9000)
+ .bulkScorer();
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, bulkScorer.score(collector, liveDocs, 0, 9000));
+ assertArrayEquals(new int[] {1, 8195}, ArrayUtil.copyOfSubArray(collected, 0, count[0]));
+ assertEquals(3, iterator.intoBitSetCalls);
+ }
+
+ public void testConstantScoreBulkScorerRejectsScoreModeThatNeedsScores() {
+ expectThrows(
+ IllegalArgumentException.class,
+ () ->
+ new ConstantScoreBulkScorer(
+ 1f, ScoreMode.COMPLETE, new CountingDocIdSetIterator(new int[] {1})));
+ }
+
+ public void testDefaultBulkScorerDoesNotUseDocIdStreamForTopScores() throws IOException {
+ assertDefaultBulkScorerDoesNotUseDocIdStreamForScores(ScoreMode.TOP_SCORES);
+ }
+
+ public void testDefaultBulkScorerDoesNotUseDocIdStreamWhenScoresAreNeeded() throws IOException {
+ assertDefaultBulkScorerDoesNotUseDocIdStreamForScores(ScoreMode.COMPLETE);
+ }
+
+ private static void assertDefaultBulkScorerDoesNotUseDocIdStreamForScores(ScoreMode scoreMode)
+ throws IOException {
+ int[] docs = {1, 2, 4097};
+ CountingDocIdSetIterator iterator = new CountingDocIdSetIterator(docs);
+ Scorer scorer = new ConstantScoreScorer(1f, scoreMode, iterator);
+
+ int[] collected = new int[docs.length];
+ int[] count = new int[1];
+ LeafCollector collector =
+ new LeafCollector() {
+ @Override
+ public void setScorer(Scorable scorer) {}
+
+ @Override
+ public void collect(int doc) {
+ collected[count[0]++] = doc;
+ }
+
+ @Override
+ public void collect(DocIdStream stream) {
+ fail("ScoreMode " + scoreMode + " must preserve per-doc collection");
+ }
+ };
+
+ BulkScorer bulkScorer = new DefaultBulkScorer(scorer);
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, bulkScorer.score(collector, null, 0, 9000));
+ assertArrayEquals(docs, ArrayUtil.copyOfSubArray(collected, 0, count[0]));
+ assertEquals(0, iterator.intoBitSetCalls);
+ }
+
+ private static class CountingDocIdSetIterator extends DocIdSetIterator {
+ private final int[] docs;
+ private int index = -1;
+ private int doc = -1;
+ private int intoBitSetCalls;
+
+ CountingDocIdSetIterator(int[] docs) {
+ this.docs = docs;
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ @Override
+ public int nextDoc() {
+ if (++index == docs.length) {
+ return doc = NO_MORE_DOCS;
+ }
+ return doc = docs[index];
+ }
+
+ @Override
+ public int advance(int target) {
+ do {
+ nextDoc();
+ } while (doc < target);
+ return doc;
+ }
+
+ @Override
+ public long cost() {
+ return docs.length;
+ }
+
+ @Override
+ public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) {
+ intoBitSetCalls++;
+ while (doc < upTo) {
+ bitSet.set(doc - offset);
+ nextDoc();
+ }
+ }
+ }
+
public void testOptimizeProhibitedClauses() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir);