You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by mk...@apache.org on 2026/09/24 20:44:14 UTC
(solr) branch main updated: SOLR-18477: faster auxIndexJoin (#4937)
This is an automated email from the ASF dual-hosted git repository.
mkhludnev pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 8de8f7a603b SOLR-18477: faster auxIndexJoin (#4937)
8de8f7a603b is described below
commit 8de8f7a603b7a713b847f74c3be5b57d3819cc8e
Author: Mikhail Khludnev <mk...@apache.org>
AuthorDate: Thu Sep 24 23:44:08 2026 +0300
SOLR-18477: faster auxIndexJoin (#4937)
- adaptive intersection
- NoStoredFieldsCodec for faster merges
---
.../SOLR-18477-perf-opt-for-auxindexjoin.yml | 10 ++
.../search/join/AuxIndexJoinQParserPlugin.java | 7 +
.../join/auxindexjoin/AuxIndexJoinConfig.java | 19 ++
.../search/join/auxindexjoin/AuxIndexManager.java | 23 ++-
.../join/auxindexjoin/JoinColumnDocWriter.java | 42 +++++
.../join/auxindexjoin/JoinIndexScorerSupplier.java | 198 +++++++++++++++------
.../search/join/auxindexjoin/JoinIndexUtils.java | 20 ++-
.../join/auxindexjoin/NoStoredFieldsCodec.java | 174 ++++++++++++++++++
.../services/org.apache.lucene.codecs.Codec | 16 ++
.../join/auxindexjoin/TestJoinColumnModel.java | 14 ++
.../join/auxindexjoin/TestNoStoredFieldsCodec.java | 176 ++++++++++++++++++
11 files changed, 640 insertions(+), 59 deletions(-)
diff --git a/changelog/unreleased/SOLR-18477-perf-opt-for-auxindexjoin.yml b/changelog/unreleased/SOLR-18477-perf-opt-for-auxindexjoin.yml
new file mode 100644
index 00000000000..2b0f8f8f73e
--- /dev/null
+++ b/changelog/unreleased/SOLR-18477-perf-opt-for-auxindexjoin.yml
@@ -0,0 +1,10 @@
+title: >
+ Faster intersection in {!auxIndexJoin} query; introduce NoStoredFieldsCodec.
+type: changed
+authors:
+ - name: Mikhail Khludnev
+ nick: mkhl
+links:
+ - name: SOLR-18477
+ url: https://issues.apache.org/jira/browse/SOLR-18477
+
diff --git a/solr/core/src/java/org/apache/solr/search/join/AuxIndexJoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/join/AuxIndexJoinQParserPlugin.java
index 2d3f7b5cf68..9535dcf5226 100644
--- a/solr/core/src/java/org/apache/solr/search/join/AuxIndexJoinQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/join/AuxIndexJoinQParserPlugin.java
@@ -170,6 +170,12 @@ public class AuxIndexJoinQParserPlugin extends QParserPlugin
*/
public static final String COMMIT_INTERVAL_MS = "commitIntervalMs";
+ /**
+ * Init parameter: name of the Lucene codec the sidecar is written with. See {@link
+ * AuxIndexJoinConfig#setCodecName}.
+ */
+ public static final String CODEC_NAME = "codecName";
+
private String configuredDir = DEFAULT_DIR;
private final AuxIndexJoinConfig joinIndexConfig = new AuxIndexJoinConfig();
@@ -205,6 +211,7 @@ public class AuxIndexJoinQParserPlugin extends QParserPlugin
MIN_RECLAIMABLE_BYTES_TO_PURGE, joinIndexConfig.getMinReclaimableBytesToPurge()));
joinIndexConfig.setCommitIntervalMs(
params.getLong(COMMIT_INTERVAL_MS, joinIndexConfig.getCommitIntervalMs()));
+ joinIndexConfig.setCodecName(params.get(CODEC_NAME, joinIndexConfig.getCodecName()));
}
}
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexJoinConfig.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexJoinConfig.java
index a998337be49..4679df5eea8 100644
--- a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexJoinConfig.java
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexJoinConfig.java
@@ -46,6 +46,7 @@ public final class AuxIndexJoinConfig {
private long minReclaimableBytesToPurge =
AuxIndexJoinMergePolicy.DEFAULT_MIN_RECLAIMABLE_BYTES_TO_PURGE;
private long commitIntervalMs = TimeUnit.SECONDS.toMillis(30);
+ private String codecName = NoStoredFieldsCodec.NAME;
/** Sole constructor, using the default settings documented on each setter. */
public AuxIndexJoinConfig() {}
@@ -204,4 +205,22 @@ public final class AuxIndexJoinConfig {
public long getCommitIntervalMs() {
return commitIntervalMs;
}
+
+ /**
+ * Name of the Lucene {@link org.apache.lucene.codecs.Codec} the sidecar is written with, looked
+ * up through SPI. Default is {@link NoStoredFieldsCodec#NAME}, which writes no stored fields; the
+ * sidecar holds doc values only.
+ */
+ public AuxIndexJoinConfig setCodecName(String codecName) {
+ if (codecName == null || codecName.isBlank()) {
+ throw new IllegalArgumentException("codecName must not be blank");
+ }
+ this.codecName = codecName;
+ return this;
+ }
+
+ /** Returns the current value set via {@link #setCodecName}. */
+ public String getCodecName() {
+ return codecName;
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexManager.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexManager.java
index e4e822a56d6..b1e6b51975a 100644
--- a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexManager.java
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/AuxIndexManager.java
@@ -28,6 +28,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.ConcurrentMergeScheduler;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexCommit;
@@ -149,7 +150,10 @@ public final class AuxIndexManager implements Closeable {
this.writer =
new IndexWriter(
directory,
- new IndexWriterConfig().setMergePolicy(mergePolicy).setMergeScheduler(mergeScheduler));
+ new IndexWriterConfig()
+ .setCodec(Codec.forName(config.getCodecName()))
+ .setMergePolicy(mergePolicy)
+ .setMergeScheduler(mergeScheduler));
// set once, not per batch: live commit data sticks on the writer and is applied by whichever
// commit comes next, including the wipe's just below and the one close() makes
this.writer.setLiveCommitData(
@@ -320,15 +324,32 @@ public final class AuxIndexManager implements Closeable {
* names.
*/
synchronized void writeBatch(Map<String, JoinColumnModel> mappings) throws IOException {
+ long startNanos = System.nanoTime();
this.writerDelegate.writeJoinColumns(writer, mappings);
+ long writtenNanos = System.nanoTime();
if (this.commitPerBatch) {
writer.commit();
}
+ long committedNanos = System.nanoTime();
if (this.blockingRefresh) {
manager.maybeRefreshBlocking();
} else {
manager.maybeRefresh(); // perhaps it should be carried out the enclosing synchronize
}
+ if (JoinIndexUtils.diagnosticsEnabled(log)) {
+ long refreshedNanos = System.nanoTime();
+ // lock wait is not measured here: it is evt=build's persistMs minus this line's totalMs
+ JoinIndexUtils.logDiagnostic(
+ log,
+ "AUXIJOIN evt=writeBatch columns={} writer={} writeMs={} commitMs={} refreshMs={}"
+ + " totalMs={}",
+ mappings.size(),
+ writerDelegate.getClass().getSimpleName(),
+ (writtenNanos - startNanos) / 1_000_000L,
+ (committedNanos - writtenNanos) / 1_000_000L,
+ (refreshedNanos - committedNanos) / 1_000_000L,
+ (refreshedNanos - startNanos) / 1_000_000L);
+ }
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinColumnDocWriter.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinColumnDocWriter.java
index 4e8290695ef..3d325c71bd7 100644
--- a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinColumnDocWriter.java
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinColumnDocWriter.java
@@ -18,6 +18,7 @@ package org.apache.solr.search.join.auxindexjoin;
import java.io.IOException;
import java.io.UncheckedIOException;
+import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -29,6 +30,8 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.solr.search.join.auxindexjoin.JoinIndexUtils.DocEdges;
import org.apache.solr.search.join.auxindexjoin.JoinIndexUtils.JoinColumnModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Sibling of {@code AIJoinColumnWriter} writing the same pair columns through the plain {@link
@@ -53,6 +56,8 @@ import org.apache.solr.search.join.auxindexjoin.JoinIndexUtils.JoinColumnModel;
*/
final class JoinColumnDocWriter extends JoinColumWriter {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
JoinColumnDocWriter() {}
@Override
@@ -64,8 +69,10 @@ final class JoinColumnDocWriter extends JoinColumWriter {
}
assert mappings.isEmpty() || batchNumDocs > 0
: "a batch with columns to write needs a doc 0 to carry their edges: " + mappings.keySet();
+ long startNanos = System.nanoTime();
// a single block: IndexWriter guarantees no intermediate flush splits it across segments
writer.addDocuments(new BatchDocuments(mappings, batchNumDocs));
+ long addedNanos = System.nanoTime();
// seal the batch into a segment of its own, so the next batch starts again at doc 0 -- the
// invariant every column depends on, since a sidecar doc number IS a from-doc id. This used
// to be a commit(), which sealed the segment only incidentally and paid an fsync of every new
@@ -73,6 +80,41 @@ final class JoinColumnDocWriter extends JoinColumWriter {
// flush() writes the segment and nothing else; durability and file reclamation are the
// periodic commit's job.
writer.flush();
+ logPersistDiagnostics(mappings, batchNumDocs, startNanos, addedNanos);
+ }
+
+ /**
+ * Emits the {@code evt=persist} diagnostic line: one per sidecar segment written, so that {@code
+ * addDocsMs} and {@code flushMs} can be regressed against {@code batchNumDocs} (the segment
+ * width) and against {@code columns} / {@code toCount} (the data actually stored).
+ */
+ private static void logPersistDiagnostics(
+ Map<String, JoinColumnModel> mappings, int batchNumDocs, long startNanos, long addedNanos) {
+ if (!JoinIndexUtils.diagnosticsEnabled(log)) {
+ return;
+ }
+ long flushedNanos = System.nanoTime();
+ long toCount = 0;
+ long modelBytes = 0;
+ int sparseModels = 0;
+ for (JoinColumnModel model : mappings.values()) {
+ toCount += model.edges().toCount();
+ modelBytes += model.ramBytesUsed();
+ if (model.isSparse()) {
+ sparseModels++;
+ }
+ }
+ JoinIndexUtils.logDiagnostic(
+ log,
+ "AUXIJOIN evt=persist batchNumDocs={} columns={} sparseModels={} toCount={} modelBytes={}"
+ + " addDocsMs={} flushMs={}",
+ batchNumDocs,
+ mappings.size(),
+ sparseModels,
+ toCount,
+ modelBytes,
+ (addedNanos - startNanos) / 1_000_000L,
+ (flushedNanos - addedNanos) / 1_000_000L);
}
/**
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinIndexScorerSupplier.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinIndexScorerSupplier.java
index cb727bb36a8..3f725cb09c4 100644
--- a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinIndexScorerSupplier.java
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinIndexScorerSupplier.java
@@ -48,6 +48,7 @@ import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.ScorerSupplier;
import org.apache.lucene.search.TwoPhaseIterator;
+import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.FixedBitSet;
import org.apache.solr.search.join.auxindexjoin.AuxIndexManager.JoinSegmentReference;
@@ -156,6 +157,13 @@ class JoinIndexScorerSupplier extends ScorerSupplier {
*/
private int rebindsAfterReap;
+ /**
+ * from-segments whose foreign-key column {@link #fromColumnsFor} had to read on the query path
+ * because the weight skipped it. Steady-state zero, like {@link #rebindsAfterReap}, but also
+ * counts pairs reaped before their cell was ever resolved, which that counter misses.
+ */
+ private int fkLateLoads;
+
// how many cells handed their built model back for the column on disk, see adoptOnDiskColumn
private int modelsReleasedToDisk;
@@ -204,19 +212,16 @@ class JoinIndexScorerSupplier extends ScorerSupplier {
this.shift = approximation.docID();
falseNegToDocsBits = new FixedBitSet(lastToDoc + 1 - shift);
}
- int walked = joinTask.dumpMatchesInto(falseNegToDocsBits, shift, approximation.docID());
- if (!joinTask.fromSegIterIsNotExausted()) {
- JoinIndexScorerSupplier.this.dropJoinLeaf(joinTask);
- leafsDrained++;
+ DrainOutcome outcome =
+ joinTask.dumpMatchesInto(falseNegToDocsBits, shift, approximation.docID());
+ fromDocsWalked += joinTask.lastDrainWalked();
+ if (outcome == DrainOutcome.CONFIRMED) {
+ logDrain(joinTask, true);
+ return true;
}
- fromDocsWalked += walked;
- if (!leafJoins.isEmpty()) {
- if (falseNegToDocsBits.get(approximation.docID() - shift)) {
- logDrain(joinTask, walked, true);
- return true;
- } // otherwise we don't know if 0 is real false
- }
- logDrain(joinTask, walked, false);
+ JoinIndexScorerSupplier.this.dropJoinLeaf(joinTask);
+ leafsDrained++;
+ logDrain(joinTask, false);
}
} catch (ExecutionException e) {
throw new RuntimeException(e);
@@ -243,7 +248,7 @@ class JoinIndexScorerSupplier extends ScorerSupplier {
* for: the doc under test was found before the remaining {@code cellsLeft} columns were
* touched.
*/
- private void logDrain(LeafJoin joinTask, int walked, boolean confirmed) {
+ private void logDrain(LeafJoin joinTask, boolean confirmed) {
if (!JoinIndexUtils.diagnosticsEnabled(log)) {
return;
}
@@ -255,7 +260,7 @@ class JoinIndexScorerSupplier extends ScorerSupplier {
JoinIndexUtils.segmentName(toContext),
joinTask.pairFieldName,
confirmed,
- walked,
+ joinTask.lastDrainWalked(),
joinTask.toCount(),
leafJoins.size(),
falseNegToDocsBits == null ? 0 : falseNegToDocsBits.cardinality(),
@@ -276,6 +281,14 @@ class JoinIndexScorerSupplier extends ScorerSupplier {
* #resolveFromCalculatedModel}: the pair's edges plus an in-memory {@link JoinColumnModel} that
* needs no further indirection to read.
*/
+ /** How one {@link LeafJoin#dumpMatchesInto} call ended. */
+ enum DrainOutcome {
+ /** Found the doc under confirmation; the cell stays live and the next drain resumes there. */
+ CONFIRMED,
+ /** Nothing left in the cell's from-range or column, so it can be dropped. */
+ EXHAUSTED
+ }
+
class LeafJoin implements DocEdges {
final String pairFieldName;
final DocIdSetIterator fromSegmentDocIdIter;
@@ -286,6 +299,7 @@ class JoinIndexScorerSupplier extends ScorerSupplier {
// set only when resolved from the indexer; null means real docvalues are opened through
// joinSegmentRef instead
private JoinColumnModel docMapping;
+ private int lastDrainWalked;
LeafJoin(
String pairFieldName,
@@ -320,8 +334,8 @@ class JoinIndexScorerSupplier extends ScorerSupplier {
*
* <p>Rebinding mid-query is safe because a cell's iteration state lives in {@link
* #fromSegmentDocIdIter}, never in the column: {@link #toDocsByFromDocsDV()} is re-opened on
- * every {@link #dumpMatchesInto} call and read by {@code advanceExact(fromDoc)}, so swapping
- * what backs it between calls is invisible to the walk. A rebuilt column maps the same
+ * every {@link #dumpMatchesInto} call and read forward from the from-iterator's position, so
+ * swapping what backs it between calls is invisible to the walk. A rebuilt column maps the same
* from-docs to the same to-docs as the one it replaces -- pair field names pin both segments --
* which the assertion below states.
*/
@@ -387,48 +401,117 @@ class JoinIndexScorerSupplier extends ScorerSupplier {
}
/**
- * Walks this cell's from-iterator from its current (prepositioned) doc through its edges' last
- * from-doc, setting every to-doc it maps to -- shifted by {@code shift} -- in {@code
- * matchedToDocs}.
+ * Walks this cell's from-docs from the current (prepositioned) one through its edges' last
+ * from-doc, setting every to-doc they map to -- shifted by {@code shift} -- in {@code
+ * matchedToDocs}, until {@code earlyExitDoc} is set or the cell has nothing left.
+ *
+ * <p>The from-docs and the column are intersected, and which side drives mirrors Lucene's
+ * {@code ConjunctionDISI}: when the from-side matches are a {@link FixedBitSet} and the column
+ * holds fewer values than there are from-matches, the column leads and each of its docs is
+ * tested against the bitset; otherwise the two leapfrog. Probing the column at every from-match
+ * instead costs one {@code advanceExact} per from-doc, wasted on those a sparse column has no
+ * value for. {@code ConjunctionUtils} itself can't be used: it wants every iterator on the same
+ * doc, while this cell's from-iterator resumes mid-stream across drains.
*
- * @return how many from-docs were walked, i.e. the column-read work this drain cost
+ * @return {@link DrainOutcome#CONFIRMED} with the from-iterator left on the confirmed doc, so
+ * the next drain resumes there; or {@link DrainOutcome#EXHAUSTED}, after which the cell is
+ * never drained again. The work either cost is {@link #lastDrainWalked()}.
*/
- int dumpMatchesInto(FixedBitSet matchedToDocs, int shift, int earlyExitDoc) throws IOException {
+ DrainOutcome dumpMatchesInto(FixedBitSet matchedToDocs, int shift, int earlyExitDoc)
+ throws IOException {
SortedNumericDocValues toDocsByFromDoc = toDocsByFromDocsDV();
- int walked = 0;
- boolean confirmedCurrent = false;
- for (int fromDoc = fromSegmentDocIdIter.docID(); // prepositioned to the first match
- fromSegIterIsNotExausted();
- fromDoc = fromSegmentDocIdIter.nextDoc()) {
- walked++;
- if (toDocsByFromDoc.advanceExact(
- fromDoc)) { // TODO will it be faster to use advance() and leapfrog?
- for (int i = 0; i < toDocsByFromDoc.docValueCount(); i++) {
- int toDocMatch = (int) toDocsByFromDoc.nextValue();
- assert toDocMatch <= toDocEdges()[1] && toDocMatch >= toDocEdges()[0]
- : "to doc "
- + toDocMatch
- + " above edges union max "
- + Arrays.toString(toDocEdges());
- // shift is wherever the approximation iterator first landed, which need not be
- // the global firstToDoc (e.g. under a boolean conjunction); a match below shift
- // is unreachable -- the iterator only moves forward -- so it's dropped rather
- // than written at a negative offset
- // Also. we don't need toDocs less than one we're confirming currently
- if (toDocMatch >= shift && toDocMatch >= earlyExitDoc) {
- matchedToDocs.set(toDocMatch - shift);
- if (toDocMatch == earlyExitDoc) {
- confirmedCurrent = true;
- }
- }
+ lastDrainWalked = 0;
+ if (fromSegmentDocIdIter instanceof BitSetIterator fromBits && toCount() < fromMatchCount) {
+ return columnLedDrain(fromBits, toDocsByFromDoc, matchedToDocs, shift, earlyExitDoc);
+ }
+ return leapfrogDrain(toDocsByFromDoc, matchedToDocs, shift, earlyExitDoc);
+ }
+
+ /**
+ * How many entries of the driving side -- column values or from-docs -- the last {@link
+ * #dumpMatchesInto} visited, i.e. the column-read work it cost.
+ */
+ int lastDrainWalked() {
+ return lastDrainWalked;
+ }
+
+ /** The column leads; each of its from-docs is checked against the from-side bitset. */
+ private DrainOutcome columnLedDrain(
+ BitSetIterator fromIter,
+ SortedNumericDocValues column,
+ FixedBitSet matchedToDocs,
+ int shift,
+ int earlyExitDoc)
+ throws IOException {
+ BitSet fromBits = fromIter.getBitSet();
+ int lastFromDoc = fromDocEdges()[1];
+ for (int fromDoc = column.advance(fromIter.docID());
+ fromDoc != DocIdSetIterator.NO_MORE_DOCS && fromDoc <= lastFromDoc;
+ fromDoc = column.nextDoc()) {
+ lastDrainWalked++;
+ if (fromDoc < fromBits.length()
+ && fromBits.get(fromDoc)
+ && collectToDocs(column, matchedToDocs, shift, earlyExitDoc)) {
+ if (fromDoc > fromIter.docID()) {
+ fromIter.advance(fromDoc); // a set bit, so it lands exactly here
}
- if (confirmedCurrent) {
- // we've just confirmed the doc, which was matches() is called for
- return walked; // giveup
+ return DrainOutcome.CONFIRMED;
+ }
+ }
+ return DrainOutcome.EXHAUSTED;
+ }
+
+ /** From-docs and column leapfrog, each advancing to the other's doc. */
+ private DrainOutcome leapfrogDrain(
+ SortedNumericDocValues column, FixedBitSet matchedToDocs, int shift, int earlyExitDoc)
+ throws IOException {
+ int lastFromDoc = fromDocEdges()[1];
+ int columnDoc = -1;
+ for (int fromDoc = fromSegmentDocIdIter.docID(); fromSegIterIsNotExausted(); ) {
+ lastDrainWalked++;
+ if (columnDoc < fromDoc) {
+ columnDoc = column.advance(fromDoc);
+ }
+ if (columnDoc == DocIdSetIterator.NO_MORE_DOCS || columnDoc > lastFromDoc) {
+ return DrainOutcome.EXHAUSTED;
+ }
+ if (columnDoc > fromDoc) {
+ fromDoc = fromSegmentDocIdIter.advance(columnDoc);
+ continue;
+ }
+ if (collectToDocs(column, matchedToDocs, shift, earlyExitDoc)) {
+ return DrainOutcome.CONFIRMED;
+ }
+ fromDoc = fromSegmentDocIdIter.nextDoc();
+ }
+ return DrainOutcome.EXHAUSTED;
+ }
+
+ /**
+ * Sets the to-docs the column maps its current from-doc to, and reports whether {@code
+ * earlyExitDoc} -- the doc {@code matches()} was asked about -- was among them.
+ */
+ private boolean collectToDocs(
+ SortedNumericDocValues column, FixedBitSet matchedToDocs, int shift, int earlyExitDoc)
+ throws IOException {
+ boolean confirmedCurrent = false;
+ for (int i = 0; i < column.docValueCount(); i++) {
+ int toDocMatch = (int) column.nextValue();
+ assert toDocMatch <= toDocEdges()[1] && toDocMatch >= toDocEdges()[0]
+ : "to doc " + toDocMatch + " above edges union max " + Arrays.toString(toDocEdges());
+ // shift is wherever the approximation iterator first landed, which need not be
+ // the global firstToDoc (e.g. under a boolean conjunction); a match below shift
+ // is unreachable -- the iterator only moves forward -- so it's dropped rather
+ // than written at a negative offset
+ // Also. we don't need toDocs less than one we're confirming currently
+ if (toDocMatch >= shift && toDocMatch >= earlyExitDoc) {
+ matchedToDocs.set(toDocMatch - shift);
+ if (toDocMatch == earlyExitDoc) {
+ confirmedCurrent = true;
}
}
}
- return walked;
+ return confirmedCurrent;
}
private boolean fromSegIterIsNotExausted() {
@@ -629,7 +712,8 @@ class JoinIndexScorerSupplier extends ScorerSupplier {
JoinIndexUtils.logDiagnostic(
log,
"AUXIJOIN evt=done ctx={} toSeg={} reason={} confirmCalls={} freeHits={} cellsDrained={}"
- + " cellsLive={} fromDocsWalked={} rebindsAfterReap={} modelsReleased={} buildMs={}",
+ + " cellsLive={} fromDocsWalked={} rebindsAfterReap={} fkLateLoads={} modelsReleased={}"
+ + " buildMs={}",
ctxId,
JoinIndexUtils.segmentName(toContext),
reason,
@@ -639,6 +723,7 @@ class JoinIndexScorerSupplier extends ScorerSupplier {
leafJoins.size(),
fromDocsWalked,
rebindsAfterReap,
+ fkLateLoads,
modelsReleasedToDisk,
joinIndexBuildNanos / 1_000_000L);
}
@@ -683,6 +768,15 @@ class JoinIndexScorerSupplier extends ScorerSupplier {
if (loaded == null) {
return fromColumnFutures;
}
+ fkLateLoads += loaded.size();
+ if (JoinIndexUtils.diagnosticsEnabled(log)) {
+ JoinIndexUtils.logDiagnostic(
+ log,
+ "AUXIJOIN evt=fkLateLoad ctx={} toSeg={} fromOrds={}",
+ ctxId,
+ JoinIndexUtils.segmentName(toContext),
+ loaded.keySet());
+ }
Map<Integer, Future<FromLeafJoinContext>> augmented = new HashMap<>(fromColumnFutures);
loaded.forEach(
(ord, context) -> augmented.put(ord, CompletableFuture.completedFuture(context)));
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinIndexUtils.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinIndexUtils.java
index bc1114c39e1..173adb5fe57 100644
--- a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinIndexUtils.java
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/JoinIndexUtils.java
@@ -373,7 +373,7 @@ final class JoinIndexUtils {
*/
SortedNumericDocValues toDocByFromDoc() {
return toDocPlusOneByFromDoc != null
- ? new DenseColumnValues(toDocPlusOneByFromDoc)
+ ? new DenseColumnValues(toDocPlusOneByFromDoc, edges.toCount())
: new SparseColumnValues(matchedFromDocs, matchedToDocs);
}
@@ -408,14 +408,23 @@ final class JoinIndexUtils {
private static final class DenseColumnValues extends SortedNumericDocValues {
private final PackedLongValues toDocPlusOneByFromDoc;
private final int size;
+
+ /**
+ * Slots holding a match, for {@link #cost()}: {@link #size} also counts the empty slots between
+ * matches, and a conjunction choosing its lead by cost would take this column for denser than
+ * it is.
+ */
+ private final int matches;
+
private int doc = -1;
/** The to-doc at {@link #doc}, read by the same probe that found it. */
private long toDoc;
- DenseColumnValues(PackedLongValues toDocPlusOneByFromDoc) {
+ DenseColumnValues(PackedLongValues toDocPlusOneByFromDoc, int matches) {
this.toDocPlusOneByFromDoc = toDocPlusOneByFromDoc;
this.size = Math.toIntExact(toDocPlusOneByFromDoc.size());
+ this.matches = matches;
}
@Override
@@ -465,7 +474,7 @@ final class JoinIndexUtils {
@Override
public long cost() {
- return size;
+ return matches;
}
}
@@ -479,9 +488,8 @@ final class JoinIndexUtils {
* {@link org.apache.lucene.index.DocValuesIterator} contract, and the one the on-disk columns
* already impose on the same call sites; both of them here -- {@code
* JoinColumnDocWriter.PairColumn} walking {@code nextDoc()} up the batch, and {@code
- * JoinIndexScorerSupplier.LeafJoin dumpMatchesInto} calling {@code advanceExact(fromDoc)} along a
- * forward-only from-doc iterator -- take a freshly positioned cursor and only ever move it
- * forward.
+ * JoinIndexScorerSupplier.LeafJoin#dumpMatchesInto} advancing it along a forward-only from-doc
+ * iterator -- take a freshly positioned cursor and only ever move it forward.
*/
private static final class SparseColumnValues extends SortedNumericDocValues {
private final PackedLongValues matchedFromDocs;
diff --git a/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/NoStoredFieldsCodec.java b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/NoStoredFieldsCodec.java
new file mode 100644
index 00000000000..aad6f32c06f
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/join/auxindexjoin/NoStoredFieldsCodec.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import org.apache.lucene.codecs.FilterCodec;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.StoredFieldsReader;
+import org.apache.lucene.codecs.StoredFieldsWriter;
+import org.apache.lucene.codecs.lucene104.Lucene104Codec;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.MergeState;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * The sidecar's codec: {@link Lucene104Codec} without stored fields. The sidecar holds doc values
+ * only, yet every sidecar document would still pass through the stored-fields writer, when a batch
+ * is indexed and again on every merge -- and the sidecar's own merges hand Lucene wrapped readers
+ * ({@link DocAlignedMerge}, column purges), which keeps it off its bulk-copy path and makes it
+ * visit each of the from-segment's documents one by one. This format writes nothing and merges by
+ * counting.
+ *
+ * <p>The delegate is pinned rather than taken from {@code Codec.getDefault()}: segments record this
+ * codec's name, not the delegate's, so a change of delegate needs a new {@link #NAME}.
+ *
+ * @lucene.experimental
+ */
+public final class NoStoredFieldsCodec extends FilterCodec {
+
+ /** Recorded in every sidecar segment; must change whenever the delegate does. */
+ public static final String NAME = "NoStoredFieldsLucene104";
+
+ private final StoredFieldsFormat storedFieldsFormat = new NoStoredFieldsFormat();
+
+ /** No-arg constructor, as SPI requires. */
+ public NoStoredFieldsCodec() {
+ super(NAME, new Lucene104Codec());
+ }
+
+ @Override
+ public StoredFieldsFormat storedFieldsFormat() {
+ return storedFieldsFormat;
+ }
+
+ /** How many of {@code maxDoc} documents {@code liveDocs} keeps; {@code null} keeps them all. */
+ static int liveDocCount(Bits liveDocs, int maxDoc) {
+ if (liveDocs == null || liveDocs instanceof Bits.MatchAllBits) {
+ return maxDoc;
+ }
+ if (liveDocs instanceof Bits.MatchNoBits) {
+ return 0;
+ }
+ int live = 0;
+ for (int doc = 0; doc < maxDoc; doc++) {
+ if (liveDocs.get(doc)) {
+ live++;
+ }
+ }
+ return live;
+ }
+
+ private static final class NoStoredFieldsFormat extends StoredFieldsFormat {
+ @Override
+ public StoredFieldsReader fieldsReader(
+ Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) {
+ return new EmptyStoredFieldsReader();
+ }
+
+ @Override
+ public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) {
+ return new NoStoredFieldsWriter();
+ }
+ }
+
+ private static final class EmptyStoredFieldsReader extends StoredFieldsReader {
+ @Override
+ public void document(int docID, StoredFieldVisitor visitor) {}
+
+ @Override
+ public StoredFieldsReader clone() {
+ return this;
+ }
+
+ @Override
+ public void checkIntegrity() {}
+
+ @Override
+ public void close() {}
+ }
+
+ /** Accepts documents, never a field: a stored field in the sidecar is a bug. */
+ private static final class NoStoredFieldsWriter extends StoredFieldsWriter {
+ @Override
+ public void startDocument() {}
+
+ @Override
+ public void writeField(FieldInfo info, int value) {
+ throw storedField(info);
+ }
+
+ @Override
+ public void writeField(FieldInfo info, long value) {
+ throw storedField(info);
+ }
+
+ @Override
+ public void writeField(FieldInfo info, float value) {
+ throw storedField(info);
+ }
+
+ @Override
+ public void writeField(FieldInfo info, double value) {
+ throw storedField(info);
+ }
+
+ @Override
+ public void writeField(FieldInfo info, BytesRef value) {
+ throw storedField(info);
+ }
+
+ @Override
+ public void writeField(FieldInfo info, String value) {
+ throw storedField(info);
+ }
+
+ @Override
+ public void finish(int numDocs) {}
+
+ /**
+ * Returns what the default merge would have counted -- every live document of every input --
+ * since {@code SegmentMerger} checks it against the merged segment's {@code maxDoc}.
+ */
+ @Override
+ public int merge(MergeState mergeState) {
+ int docCount = 0;
+ for (int i = 0; i < mergeState.maxDocs.length; i++) {
+ docCount += liveDocCount(mergeState.liveDocs[i], mergeState.maxDocs[i]);
+ }
+ return docCount;
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public long ramBytesUsed() {
+ return 0;
+ }
+
+ private static UnsupportedOperationException storedField(FieldInfo info) {
+ return new UnsupportedOperationException(
+ NAME + " stores no fields, but was given stored field '" + info.name + "'");
+ }
+ }
+}
diff --git a/solr/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/solr/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
new file mode 100644
index 00000000000..42b0f80bc7e
--- /dev/null
+++ b/solr/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.solr.search.join.auxindexjoin.NoStoredFieldsCodec
diff --git a/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/TestJoinColumnModel.java b/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/TestJoinColumnModel.java
index e3191e63ebc..61b5eb9a079 100644
--- a/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/TestJoinColumnModel.java
+++ b/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/TestJoinColumnModel.java
@@ -48,6 +48,20 @@ public class TestJoinColumnModel extends SolrTestCase {
}
/** The layouts have to agree at the extremes too: nothing matched, and everything matched. */
+ /**
+ * Both layouts report as their cost the matches they hold, not the width they span: a conjunction
+ * picks its lead by cost, so a dense column counting its empty slots would pass for denser than
+ * it is.
+ */
+ public void testCostIsTheMatchCountInEitherLayout() throws IOException {
+ for (int iter = 0; iter < 50; iter++) {
+ int[] toDocByFromDoc = randomMap(TestUtil.nextInt(random(), 1, 500), random().nextFloat());
+ long matches = expectedWalk(toDocByFromDoc).size();
+ assertEquals(matches, dense(toDocByFromDoc).toDocByFromDoc().cost());
+ assertEquals(matches, sparse(toDocByFromDoc).toDocByFromDoc().cost());
+ }
+ }
+
public void testDegenerateColumns() throws IOException {
int maxDoc = TestUtil.nextInt(random(), 1, 100);
int[] empty = new int[maxDoc];
diff --git a/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/TestNoStoredFieldsCodec.java b/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/TestNoStoredFieldsCodec.java
new file mode 100644
index 00000000000..e788c515fed
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/search/join/auxindexjoin/TestNoStoredFieldsCodec.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.join.auxindexjoin;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.index.TieredMergePolicy;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
+import org.apache.solr.SolrTestCase;
+import org.apache.solr.search.join.auxindexjoin.JoinIndexUtils.Edges;
+import org.apache.solr.search.join.auxindexjoin.JoinIndexUtils.JoinColumnModel;
+
+/** The sidecar codec keeps doc values intact while writing and merging no stored fields. */
+public class TestNoStoredFieldsCodec extends SolrTestCase {
+
+ private static final String FIELD = "value";
+
+ public void testResolvesByNameThroughSpi() {
+ assertTrue(Codec.forName(NoStoredFieldsCodec.NAME) instanceof NoStoredFieldsCodec);
+ }
+
+ public void testMergeKeepsDocValuesAndWritesNoStoredFields() throws IOException {
+ int segments = 3;
+ int perSegment = 50;
+ long deleted = 3;
+ try (Directory dir = newDirectory()) {
+ TieredMergePolicy mergePolicy = new TieredMergePolicy();
+ mergePolicy.setNoCFSRatio(0.0); // keep each format's files visible, not packed in a .cfs
+ IndexWriterConfig config =
+ new IndexWriterConfig()
+ .setCodec(new NoStoredFieldsCodec())
+ .setUseCompoundFile(false)
+ .setMergePolicy(mergePolicy);
+ try (IndexWriter writer = new IndexWriter(dir, config)) {
+ for (int segment = 0; segment < segments; segment++) {
+ for (int i = 0; i < perSegment; i++) {
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField(FIELD, segment * perSegment + i));
+ writer.addDocument(doc);
+ }
+ writer.commit();
+ }
+ // a deletion forces the merge to count live docs rather than take each input's maxDoc
+ writer.deleteDocuments(NumericDocValuesField.newSlowExactQuery(FIELD, deleted));
+ writer.forceMerge(1);
+ writer.commit();
+ }
+
+ Set<Long> expected = new TreeSet<>();
+ for (long value = 0; value < segments * perSegment; value++) {
+ if (value != deleted) {
+ expected.add(value);
+ }
+ }
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ assertEquals(1, reader.leaves().size());
+ assertEquals(expected.size(), reader.maxDoc());
+ NumericDocValues values = reader.leaves().get(0).reader().getNumericDocValues(FIELD);
+ Set<Long> actual = new TreeSet<>();
+ for (int doc = values.nextDoc();
+ doc != DocIdSetIterator.NO_MORE_DOCS;
+ doc = values.nextDoc()) {
+ actual.add(values.longValue());
+ }
+ assertEquals(expected, actual);
+ assertTrue(reader.storedFields().document(0).getFields().isEmpty());
+ }
+ for (String file : dir.listAll()) {
+ assertFalse(
+ "stored-fields file written: " + file,
+ file.endsWith(".fdt") || file.endsWith(".fdx") || file.endsWith(".fdm"));
+ }
+ assertEverySegmentUsesTheCodec(dir);
+ }
+ }
+
+ public void testLiveDocCount() {
+ assertEquals(7, NoStoredFieldsCodec.liveDocCount(null, 7));
+ assertEquals(7, NoStoredFieldsCodec.liveDocCount(new Bits.MatchAllBits(7), 7));
+ assertEquals(0, NoStoredFieldsCodec.liveDocCount(new Bits.MatchNoBits(7), 7));
+ Bits evens =
+ new Bits() {
+ @Override
+ public boolean get(int index) {
+ return index % 2 == 0;
+ }
+
+ @Override
+ public int length() {
+ return 7;
+ }
+ };
+ assertEquals(4, NoStoredFieldsCodec.liveDocCount(evens, 7));
+ }
+
+ public void testSidecarSegmentsRecordTheCodec() throws IOException {
+ try (Directory joinDir = newDirectory()) {
+ try (AuxIndexManager joinIndex =
+ new AuxIndexManager(joinDir, new AuxIndexJoinConfig().setCommitIntervalMs(0))) {
+ joinIndex.writeBatch(Map.of("fromSeg:dv0_toSeg:dv0", sampleColumn()));
+ }
+ assertEverySegmentUsesTheCodec(joinDir);
+ }
+ }
+
+ public void testDefaultCodecNameIsTheNoStoredFieldsCodec() {
+ assertEquals(NoStoredFieldsCodec.NAME, new AuxIndexJoinConfig().getCodecName());
+ expectThrows(IllegalArgumentException.class, () -> new AuxIndexJoinConfig().setCodecName(" "));
+ }
+
+ public void testSidecarUsesTheConfiguredCodec() throws IOException {
+ String codecName = Codec.getDefault().getName();
+ try (Directory joinDir = newDirectory()) {
+ AuxIndexJoinConfig config =
+ new AuxIndexJoinConfig().setCommitIntervalMs(0).setCodecName(codecName);
+ try (AuxIndexManager joinIndex = new AuxIndexManager(joinDir, config)) {
+ joinIndex.writeBatch(Map.of("fromSeg:dv0_toSeg:dv0", sampleColumn()));
+ }
+ assertEverySegmentUsesTheCodec(joinDir, codecName);
+ }
+ }
+
+ public void testUnknownCodecNameFailsAtOpen() throws IOException {
+ try (Directory joinDir = newDirectory()) {
+ AuxIndexJoinConfig config = new AuxIndexJoinConfig().setCodecName("NoSuchCodec");
+ IllegalArgumentException e =
+ expectThrows(IllegalArgumentException.class, () -> new AuxIndexManager(joinDir, config));
+ assertTrue(e.getMessage(), e.getMessage().contains("NoSuchCodec"));
+ }
+ }
+
+ private static JoinColumnModel sampleColumn() {
+ return JoinColumnModel.dense(
+ new int[] {3, -1, 7}, new Edges(new int[] {0, 2}, new int[] {3, 7}, 2));
+ }
+
+ private static void assertEverySegmentUsesTheCodec(Directory dir) throws IOException {
+ assertEverySegmentUsesTheCodec(dir, NoStoredFieldsCodec.NAME);
+ }
+
+ private static void assertEverySegmentUsesTheCodec(Directory dir, String codecName)
+ throws IOException {
+ SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
+ assertFalse("expected at least one segment", infos.asList().isEmpty());
+ for (SegmentCommitInfo info : infos) {
+ assertEquals(codecName, info.info.getCodec().getName());
+ }
+ }
+}