You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ja...@apache.org on 2026/07/09 10:23:17 UTC
(lucene) branch branch_10x updated: Accumulate join Total/Avg scores in double precision (#16378)
This is an automated email from the ASF dual-hosted git repository.
javanna pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new 11d03214bc6 Accumulate join Total/Avg scores in double precision (#16378)
11d03214bc6 is described below
commit 11d03214bc693c48952267d33e48fe37abc7663d
Author: Luca Cavanna <ja...@apache.org>
AuthorDate: Thu Jul 9 12:21:33 2026 +0200
Accumulate join Total/Avg scores in double precision (#16378)
TermsWithScoreCollector, JoinUtil's numeric point join, and TestJoinUtil
summed per-document scores in float. Float addition isn't associative,
so summing the same scores in a different order can round to a different float,
causing TestJoinUtil.testMultiValueRandomJoin to intermittently fail on exact
score/doc-order assertions.
Scores are now accumulated in double and narrowed to float only where exposed.
---
lucene/CHANGES.txt | 6 +++
.../org/apache/lucene/search/join/JoinUtil.java | 57 +++++++++++++++++-----
.../search/join/TermsWithScoreCollector.java | 50 ++++++++++---------
.../apache/lucene/search/join/TestJoinUtil.java | 9 ++--
4 files changed, 82 insertions(+), 40 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index f5c6dccc8aa..42a30e38058 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -67,8 +67,14 @@ Bug Fixes
to the wrapped NumericDocValues, enabling optimized range evaluation for
single-valued sorted numeric fields. (Costin Leau)
+<<<<<<< join_util_scores
+* GITHUB#16378: Accumulate join Total/Avg scores in double precision in
+ TermsWithScoreCollector and JoinUtil's numeric point join, fixing
+ intermittent failures caused by non-associative float addition. (Luca Cavanna)
+=======
* GITHUB#16296: Fix missing null check in RamUsageEstimator.sizeOf(Accountable)
to be consistent with sizeOf(String) and sizeOf(Accountable[]). (Tim Grein)
+>>>>>>> main
Other
---------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java b/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java
index 2cf4c394460..e85e83db428 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java
@@ -35,9 +35,9 @@ import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.internal.hppc.LongArrayList;
import org.apache.lucene.internal.hppc.LongCursor;
-import org.apache.lucene.internal.hppc.LongFloatHashMap;
import org.apache.lucene.internal.hppc.LongHashSet;
import org.apache.lucene.internal.hppc.LongIntHashMap;
+import org.apache.lucene.internal.hppc.LongObjectHashMap;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchNoDocsQuery;
@@ -149,7 +149,8 @@ public final class JoinUtil {
ScoreMode scoreMode)
throws IOException {
LongHashSet joinValues = new LongHashSet();
- LongFloatHashMap aggregatedScores = new LongFloatHashMap();
+ // Maps a join value to its aggregated score cell; see ScoreCell for why double precision.
+ LongObjectHashMap<ScoreCell> aggregatedScores = new LongObjectHashMap<>();
LongIntHashMap occurrences = new LongIntHashMap();
boolean needsScore = scoreMode != ScoreMode.None;
LongFloatProcedure scoreAggregator;
@@ -158,10 +159,12 @@ public final class JoinUtil {
(key, score) -> {
int index = aggregatedScores.indexOf(key);
if (index < 0) {
- aggregatedScores.indexInsert(index, key, score);
+ ScoreCell cell = new ScoreCell();
+ cell.value = score;
+ aggregatedScores.indexInsert(index, key, cell);
} else {
- float currentScore = aggregatedScores.indexGet(index);
- aggregatedScores.indexReplace(index, Math.max(currentScore, score));
+ ScoreCell cell = aggregatedScores.indexGet(index);
+ cell.value = Math.max(cell.value, score);
}
};
} else if (scoreMode == ScoreMode.Min) {
@@ -169,18 +172,37 @@ public final class JoinUtil {
(key, score) -> {
int index = aggregatedScores.indexOf(key);
if (index < 0) {
- aggregatedScores.indexInsert(index, key, score);
+ ScoreCell cell = new ScoreCell();
+ cell.value = score;
+ aggregatedScores.indexInsert(index, key, cell);
} else {
- float currentScore = aggregatedScores.indexGet(index);
- aggregatedScores.indexReplace(index, Math.min(currentScore, score));
+ ScoreCell cell = aggregatedScores.indexGet(index);
+ cell.value = Math.min(cell.value, score);
}
};
} else if (scoreMode == ScoreMode.Total) {
- scoreAggregator = aggregatedScores::addTo;
+ scoreAggregator =
+ (key, score) -> {
+ int index = aggregatedScores.indexOf(key);
+ if (index < 0) {
+ ScoreCell cell = new ScoreCell();
+ cell.value = score;
+ aggregatedScores.indexInsert(index, key, cell);
+ } else {
+ aggregatedScores.indexGet(index).value += score;
+ }
+ };
} else if (scoreMode == ScoreMode.Avg) {
scoreAggregator =
(key, score) -> {
- aggregatedScores.addTo(key, score);
+ int index = aggregatedScores.indexOf(key);
+ if (index < 0) {
+ ScoreCell cell = new ScoreCell();
+ cell.value = score;
+ aggregatedScores.indexInsert(index, key, cell);
+ } else {
+ aggregatedScores.indexGet(index).value += score;
+ }
occurrences.addTo(key, 1);
};
} else {
@@ -194,12 +216,12 @@ public final class JoinUtil {
if (scoreMode == ScoreMode.Avg) {
joinScorer =
(joinValue) -> {
- float aggregatedScore = aggregatedScores.get(joinValue);
+ double aggregatedScore = aggregatedScores.get(joinValue).value;
int occurrence = occurrences.get(joinValue);
- return aggregatedScore / occurrence;
+ return (float) (aggregatedScore / occurrence);
};
} else {
- joinScorer = aggregatedScores::get;
+ joinScorer = (joinValue) -> (float) aggregatedScores.get(joinValue).value;
}
Collector collector;
@@ -583,6 +605,15 @@ public final class JoinUtil {
searcher.getTopReaderContext().id());
}
+ /**
+ * A mutable double precision score accumulator, kept in double rather than float since float
+ * addition isn't associative: summing the same per-document scores in a different order may round
+ * to a different float.
+ */
+ private static final class ScoreCell {
+ double value;
+ }
+
/** Similar to {@link java.util.function.LongFunction} for primitive argument and result. */
private interface LongFloatFunction {
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java
index 84252a7b15a..315f8fe85b1 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java
@@ -34,15 +34,17 @@ abstract class TermsWithScoreCollector<DV> extends DocValuesTermsCollector<DV>
final ScoreMode scoreMode;
Scorable scorer;
- float[] scoreSums = new float[INITIAL_ARRAY_SIZE];
+ // Accumulated in double precision because float addition isn't associative, so summing the
+ // same per-document scores in a different order may round to a different float.
+ double[] scoreSums = new double[INITIAL_ARRAY_SIZE];
TermsWithScoreCollector(Function<DV> docValuesCall, ScoreMode scoreMode) {
super(docValuesCall);
this.scoreMode = scoreMode;
if (scoreMode == ScoreMode.Min) {
- Arrays.fill(scoreSums, Float.POSITIVE_INFINITY);
+ Arrays.fill(scoreSums, Double.POSITIVE_INFINITY);
} else if (scoreMode == ScoreMode.Max) {
- Arrays.fill(scoreSums, Float.NEGATIVE_INFINITY);
+ Arrays.fill(scoreSums, Double.NEGATIVE_INFINITY);
}
}
@@ -53,7 +55,11 @@ abstract class TermsWithScoreCollector<DV> extends DocValuesTermsCollector<DV>
@Override
public float[] getScoresPerTerm() {
- return scoreSums;
+ float[] scores = new float[scoreSums.length];
+ for (int i = 0; i < scoreSums.length; i++) {
+ scores[i] = (float) scoreSums[i];
+ }
+ return scores;
}
@Override
@@ -119,16 +125,16 @@ abstract class TermsWithScoreCollector<DV> extends DocValuesTermsCollector<DV>
int begin = scoreSums.length;
scoreSums = ArrayUtil.grow(scoreSums);
if (scoreMode == ScoreMode.Min) {
- Arrays.fill(scoreSums, begin, scoreSums.length, Float.POSITIVE_INFINITY);
+ Arrays.fill(scoreSums, begin, scoreSums.length, Double.POSITIVE_INFINITY);
} else if (scoreMode == ScoreMode.Max) {
- Arrays.fill(scoreSums, begin, scoreSums.length, Float.NEGATIVE_INFINITY);
+ Arrays.fill(scoreSums, begin, scoreSums.length, Double.NEGATIVE_INFINITY);
}
}
}
float current = scorer.score();
- float existing = scoreSums[ord];
- if (Float.compare(existing, 0.0f) == 0) {
+ double existing = scoreSums[ord];
+ if (Double.compare(existing, 0.0d) == 0) {
scoreSums[ord] = current;
} else {
switch (scoreMode) {
@@ -180,8 +186,8 @@ abstract class TermsWithScoreCollector<DV> extends DocValuesTermsCollector<DV>
}
float current = scorer.score();
- float existing = scoreSums[ord];
- if (Float.compare(existing, 0.0f) == 0) {
+ double existing = scoreSums[ord];
+ if (Double.compare(existing, 0.0d) == 0) {
scoreSums[ord] = current;
scoreCounts[ord] = 1;
} else {
@@ -192,13 +198,11 @@ abstract class TermsWithScoreCollector<DV> extends DocValuesTermsCollector<DV>
@Override
public float[] getScoresPerTerm() {
- if (scoreCounts != null) {
- for (int i = 0; i < scoreCounts.length; i++) {
- scoreSums[i] = scoreSums[i] / scoreCounts[i];
- }
- scoreCounts = null;
+ float[] scores = new float[scoreSums.length];
+ for (int i = 0; i < scores.length; i++) {
+ scores[i] = (float) (scoreSums[i] / scoreCounts[i]);
}
- return scoreSums;
+ return scores;
}
}
}
@@ -222,9 +226,9 @@ abstract class TermsWithScoreCollector<DV> extends DocValuesTermsCollector<DV>
int begin = scoreSums.length;
scoreSums = ArrayUtil.grow(scoreSums);
if (scoreMode == ScoreMode.Min) {
- Arrays.fill(scoreSums, begin, scoreSums.length, Float.POSITIVE_INFINITY);
+ Arrays.fill(scoreSums, begin, scoreSums.length, Double.POSITIVE_INFINITY);
} else if (scoreMode == ScoreMode.Max) {
- Arrays.fill(scoreSums, begin, scoreSums.length, Float.NEGATIVE_INFINITY);
+ Arrays.fill(scoreSums, begin, scoreSums.length, Double.NEGATIVE_INFINITY);
}
}
}
@@ -278,13 +282,11 @@ abstract class TermsWithScoreCollector<DV> extends DocValuesTermsCollector<DV>
@Override
public float[] getScoresPerTerm() {
- if (scoreCounts != null) {
- for (int i = 0; i < scoreCounts.length; i++) {
- scoreSums[i] = scoreSums[i] / scoreCounts[i];
- }
- scoreCounts = null;
+ float[] scores = new float[scoreSums.length];
+ for (int i = 0; i < scores.length; i++) {
+ scores[i] = (float) (scoreSums[i] / scoreCounts[i]);
}
- return scoreSums;
+ return scores;
}
}
}
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
index 37431b73a4a..d7cc7abaf3e 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
@@ -2049,7 +2049,10 @@ public class TestJoinUtil extends LuceneTestCase {
float minScore = Float.POSITIVE_INFINITY;
float maxScore = Float.NEGATIVE_INFINITY;
- float total;
+ // Accumulated in double precision, because float addition isn't associative, so summing the
+ // same scores
+ // in a different order may round to a different float.
+ double total;
int count;
void addScore(float score) {
@@ -2068,9 +2071,9 @@ public class TestJoinUtil extends LuceneTestCase {
case None:
return 1f;
case Total:
- return total;
+ return (float) total;
case Avg:
- return total / count;
+ return (float) (total / count);
case Min:
return minScore;
case Max: