You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ja...@apache.org on 2026/10/07 10:18:35 UTC
(lucene) branch main updated: Remove TermGroupFacetCollector and GroupFacetCollector (#16779)
This is an automated email from the ASF dual-hosted git repository.
javanna pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 3e4a7a02ce0 Remove TermGroupFacetCollector and GroupFacetCollector (#16779)
3e4a7a02ce0 is described below
commit 3e4a7a02ce06a4f6ae796de79827d53202c65336
Author: Luca Cavanna <ja...@apache.org>
AuthorDate: Wed Oct 7 12:18:29 2026 +0200
Remove TermGroupFacetCollector and GroupFacetCollector (#16779)
TermGroupFacetCollector and GroupFacetCollector are deprecated and have been replaced by the the newly introduced TermGroupFacetCollectorManager. This commits removes the deprecated classes and corresponding test class.
Relates to #16292
---
lucene/CHANGES.txt | 4 +
lucene/MIGRATE.md | 39 +
.../search/grouping/GroupFacetCollector.java | 273 -------
.../search/grouping/TermGroupFacetCollector.java | 439 -----------
.../grouping/TermGroupFacetCollectorManager.java | 3 +-
.../search/grouping/TestGroupFacetCollector.java | 807 ---------------------
6 files changed, 44 insertions(+), 1521 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 740576fc23f..c43a10541f9 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -126,6 +126,10 @@ API Changes
so that per-file bookkeeping in subclasses applies to copied files; subclasses that want an
optimized copy (e.g. HardlinkCopyDirectoryWrapper) may override to delegate. (Tim Allison)
+* GITHUB#16779: Remove TermGroupFacetCollector and GroupFacetCollector. Use
+ TermGroupFacetCollectorManager, which supports concurrent search. GroupedFacetResult
+ and FacetEntry are now top-level types. (Luca Cavanna)
+
New Features
---------------------
* GITHUB#16241: Add HangulCompositionCharFilter to analysis-nori for opt-in composition of
diff --git a/lucene/MIGRATE.md b/lucene/MIGRATE.md
index 341588ef9cd..45fb497db89 100644
--- a/lucene/MIGRATE.md
+++ b/lucene/MIGRATE.md
@@ -48,6 +48,45 @@ called `LuceneTestCaseParent` but you should reference them either
without an explicit type or via the type of the parent class
for your test framework. The parent class may be removed in the future.
+### TermGroupFacetCollector and GroupFacetCollector removed (GITHUB#16292, GITHUB#16779)
+
+`TermGroupFacetCollector` and its base class `GroupFacetCollector` have been removed.
+Use `TermGroupFacetCollectorManager` with `IndexSearcher#search(Query, CollectorManager)`
+instead. `size`, `minCount`, and `orderByCount` are now constructor arguments; merging
+happens in `reduce()`.
+
+`GroupedFacetResult` and `FacetEntry` are now top-level types in
+`org.apache.lucene.search.grouping`. `FacetEntry` is nested in `GroupedFacetResult`
+(`GroupedFacetResult.FacetEntry`).
+
+Before:
+
+```java
+TermGroupFacetCollector collector =
+ TermGroupFacetCollector.createTermGroupFacetCollector(
+ groupField, facetField, facetFieldMultivalued, facetPrefix, initialSize);
+searcher.search(query, collector);
+TermGroupFacetCollector.GroupedFacetResult result =
+ collector.mergeSegmentResults(offset + limit, minCount, orderByCount);
+List<TermGroupFacetCollector.FacetEntry> entries = result.getFacetEntries(offset, limit);
+```
+
+After:
+
+```java
+TermGroupFacetCollectorManager manager =
+ new TermGroupFacetCollectorManager(
+ groupField,
+ facetField,
+ facetFieldMultivalued,
+ facetPrefix,
+ offset + limit,
+ minCount,
+ orderByCount);
+GroupedFacetResult result = searcher.search(query, manager);
+List<GroupedFacetResult.FacetEntry> entries = result.getFacetEntries(offset, limit);
+```
+
### Directory#copyFrom is now abstract (GITHUB#16530)
`Directory#copyFrom` no longer has a default implementation, so classes that extend
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupFacetCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupFacetCollector.java
deleted file mode 100644
index fcad0a70193..00000000000
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/GroupFacetCollector.java
+++ /dev/null
@@ -1,273 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.grouping;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.NavigableSet;
-import java.util.TreeSet;
-import org.apache.lucene.search.Scorable;
-import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.SimpleCollector;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.PriorityQueue;
-
-/**
- * Base class for computing grouped facets.
- *
- * @lucene.experimental
- * @deprecated Use {@code TermGroupFacetCollectorManager} instead, which supports concurrent search.
- * {@code GroupedFacetResult} and {@code GroupedFacetResult.FacetEntry} are now top-level
- * classes.
- */
-@Deprecated
-public abstract class GroupFacetCollector extends SimpleCollector {
-
- protected final String groupField;
- protected final String facetField;
- protected final BytesRef facetPrefix;
- protected final List<SegmentResult> segmentResults;
-
- protected int[] segmentFacetCounts;
- protected int segmentTotalCount;
- protected int startFacetOrd;
- protected int endFacetOrd;
-
- protected GroupFacetCollector(String groupField, String facetField, BytesRef facetPrefix) {
- this.groupField = groupField;
- this.facetField = facetField;
- this.facetPrefix = facetPrefix;
- segmentResults = new ArrayList<>();
- }
-
- /**
- * Returns grouped facet results that were computed over zero or more segments. Grouped facet
- * counts are merged from zero or more segment results.
- *
- * @param size The total number of facets to include. This is typically offset + limit
- * @param minCount The minimum count a facet entry should have to be included in the grouped facet
- * result
- * @param orderByCount Whether to sort the facet entries by facet entry count. If <code>false
- * </code> then the facets are sorted lexicographically in ascending order.
- * @return grouped facet results
- * @throws IOException If I/O related errors occur during merging segment grouped facet counts.
- * @deprecated Use {@link TermGroupFacetCollectorManager} instead. The {@code size}, {@code
- * minCount}, and {@code orderByCount} parameters are now passed to its constructor, and
- * merging happens automatically inside {@link
- * org.apache.lucene.search.IndexSearcher#search(org.apache.lucene.search.Query,
- * org.apache.lucene.search.CollectorManager)}.
- */
- @Deprecated
- public GroupedFacetResult mergeSegmentResults(int size, int minCount, boolean orderByCount)
- throws IOException {
- int totalCount = 0;
- int missingCount = 0;
- PriorityQueue<SegmentResult> segments =
- PriorityQueue.usingComparator(
- segmentResults.size(), Comparator.comparing(sr -> sr.mergeTerm));
- for (SegmentResult segmentResult : segmentResults) {
- missingCount += segmentResult.missing;
- if (segmentResult.mergePos >= segmentResult.maxTermPos) {
- continue;
- }
- totalCount += segmentResult.total;
- segments.add(segmentResult);
- }
-
- GroupedFacetResult facetResult =
- new GroupedFacetResult(size, minCount, orderByCount, totalCount, missingCount);
- while (segments.size() > 0) {
- SegmentResult segmentResult = segments.top();
- BytesRef currentFacetValue = BytesRef.deepCopyOf(segmentResult.mergeTerm);
- int count = 0;
-
- do {
- count += segmentResult.counts[segmentResult.mergePos++];
- if (segmentResult.mergePos < segmentResult.maxTermPos) {
- segmentResult.nextTerm();
- segmentResult = segments.updateTop();
- } else {
- segments.pop();
- segmentResult = segments.top();
- if (segmentResult == null) {
- break;
- }
- }
- } while (currentFacetValue.equals(segmentResult.mergeTerm));
- facetResult.addFacetCount(currentFacetValue, count);
- }
- return facetResult;
- }
-
- @Override
- public void finish() throws IOException {
- segmentResults.add(createSegmentResult());
- segmentFacetCounts = null;
- }
-
- protected abstract SegmentResult createSegmentResult() throws IOException;
-
- @Override
- public void setScorer(Scorable scorer) throws IOException {}
-
- @Override
- public ScoreMode scoreMode() {
- return ScoreMode.COMPLETE_NO_SCORES;
- }
-
- /**
- * The grouped facet result. Containing grouped facet entries, total count and total missing
- * count.
- */
- public static class GroupedFacetResult {
-
- private final int maxSize;
- private final NavigableSet<FacetEntry> facetEntries;
- private final int totalMissingCount;
- private final int totalCount;
-
- private int currentMin;
-
- public GroupedFacetResult(
- int size, int minCount, boolean orderByCount, int totalCount, int totalMissingCount) {
- this.facetEntries =
- new TreeSet<>(
- orderByCount
- ? (a, b) -> {
- int cmp = b.count - a.count; // Highest count first!
- if (cmp != 0) {
- return cmp;
- }
- return a.value.compareTo(b.value);
- }
- : (a, b) -> a.value.compareTo(b.value));
- this.totalMissingCount = totalMissingCount;
- this.totalCount = totalCount;
- maxSize = size;
- currentMin = minCount;
- }
-
- public void addFacetCount(BytesRef facetValue, int count) {
- if (count < currentMin) {
- return;
- }
-
- FacetEntry facetEntry = new FacetEntry(facetValue, count);
- if (facetEntries.size() == maxSize) {
- if (facetEntries.higher(facetEntry) == null) {
- return;
- }
- facetEntries.pollLast();
- }
- facetEntries.add(facetEntry);
-
- if (facetEntries.size() == maxSize) {
- currentMin = facetEntries.last().count;
- }
- }
-
- /**
- * Returns a list of facet entries to be rendered based on the specified offset and limit. The
- * facet entries are retrieved from the facet entries collected during merging.
- *
- * @param offset The offset in the collected facet entries during merging
- * @param limit The number of facets to return starting from the offset.
- * @return a list of facet entries to be rendered based on the specified offset and limit
- */
- public List<FacetEntry> getFacetEntries(int offset, int limit) {
- if (offset >= facetEntries.size()) {
- return Collections.emptyList();
- }
-
- List<FacetEntry> entries = new ArrayList<>(Math.min(limit, facetEntries.size() - offset));
-
- int skipped = 0;
- int included = 0;
- for (FacetEntry facetEntry : facetEntries) {
- if (skipped < offset) {
- skipped++;
- continue;
- }
- if (included++ >= limit) {
- break;
- }
- entries.add(facetEntry);
- }
- return entries;
- }
-
- /**
- * Returns the sum of all facet entries counts.
- *
- * @return the sum of all facet entries counts
- */
- public int getTotalCount() {
- return totalCount;
- }
-
- /**
- * Returns the number of groups that didn't have a facet value.
- *
- * @return the number of groups that didn't have a facet value
- */
- public int getTotalMissingCount() {
- return totalMissingCount;
- }
- }
-
- /** Represents a facet entry with a value and a count. */
- public record FacetEntry(BytesRef value, int count) {
-
- @Override
- public String toString() {
- return "FacetEntry{" + "value=" + value.utf8ToString() + ", count=" + count + '}';
- }
- }
-
- /**
- * Contains the local grouped segment counts for a particular segment. Each <code>SegmentResult
- * </code> must be added together.
- */
- protected abstract static class SegmentResult {
-
- protected final int[] counts;
- protected final int total;
- protected final int missing;
- protected final int maxTermPos;
-
- protected BytesRef mergeTerm;
- protected int mergePos;
-
- protected SegmentResult(int[] counts, int total, int missing, int maxTermPos) {
- this.counts = counts;
- this.total = total;
- this.missing = missing;
- this.maxTermPos = maxTermPos;
- }
-
- /**
- * Go to next term in this <code>SegmentResult</code> in order to retrieve the grouped facet
- * counts.
- *
- * @throws IOException If I/O related errors occur
- */
- protected abstract void nextTerm() throws IOException;
- }
-}
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/TermGroupFacetCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/TermGroupFacetCollector.java
deleted file mode 100644
index 6a4831b59c0..00000000000
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/TermGroupFacetCollector.java
+++ /dev/null
@@ -1,439 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.grouping;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.SentinelIntSet;
-import org.apache.lucene.util.UnicodeUtil;
-
-/**
- * An implementation of {@link GroupFacetCollector} that computes grouped facets based on the
- * indexed terms from DocValues.
- *
- * @lucene.experimental
- * @deprecated Use {@code TermGroupFacetCollectorManager} instead, which supports concurrent search.
- */
-@Deprecated
-public abstract class TermGroupFacetCollector extends GroupFacetCollector {
-
- final List<GroupedFacetHit> groupedFacetHits;
- final SentinelIntSet segmentGroupedFacetHits;
-
- SortedDocValues groupFieldTermsIndex;
-
- /**
- * Factory method for creating the right implementation based on the fact whether the facet field
- * contains multiple tokens per documents.
- *
- * @param groupField The group field
- * @param facetField The facet field
- * @param facetFieldMultivalued Whether the facet field has multiple tokens per document
- * @param facetPrefix The facet prefix a facet entry should start with to be included.
- * @param initialSize The initial allocation size of the internal int set and group facet list
- * which should roughly match the total number of expected unique groups. Be aware that the
- * heap usage is 4 bytes * initialSize.
- * @return <code>TermGroupFacetCollector</code> implementation
- * @deprecated Use {@code TermGroupFacetCollectorManager} instead.
- */
- @Deprecated
- public static TermGroupFacetCollector createTermGroupFacetCollector(
- String groupField,
- String facetField,
- boolean facetFieldMultivalued,
- BytesRef facetPrefix,
- int initialSize) {
- if (facetFieldMultivalued) {
- return new MV(groupField, facetField, facetPrefix, initialSize);
- } else {
- return new SV(groupField, facetField, facetPrefix, initialSize);
- }
- }
-
- TermGroupFacetCollector(
- String groupField, String facetField, BytesRef facetPrefix, int initialSize) {
- super(groupField, facetField, facetPrefix);
- groupedFacetHits = new ArrayList<>(initialSize);
- segmentGroupedFacetHits = new SentinelIntSet(initialSize, Integer.MIN_VALUE);
- }
-
- // Implementation for single valued facet fields.
- static class SV extends TermGroupFacetCollector {
-
- private SortedDocValues facetFieldTermsIndex;
-
- SV(String groupField, String facetField, BytesRef facetPrefix, int initialSize) {
- super(groupField, facetField, facetPrefix, initialSize);
- }
-
- @Override
- public void collect(int doc) throws IOException {
- if (doc > facetFieldTermsIndex.docID()) {
- facetFieldTermsIndex.advance(doc);
- }
-
- int facetOrd;
- if (doc == facetFieldTermsIndex.docID()) {
- facetOrd = facetFieldTermsIndex.ordValue();
- } else {
- facetOrd = -1;
- }
-
- if (facetOrd < startFacetOrd || facetOrd >= endFacetOrd) {
- return;
- }
-
- if (doc > groupFieldTermsIndex.docID()) {
- groupFieldTermsIndex.advance(doc);
- }
-
- int groupOrd;
- if (doc == groupFieldTermsIndex.docID()) {
- groupOrd = groupFieldTermsIndex.ordValue();
- } else {
- groupOrd = -1;
- }
- int segmentGroupedFacetsIndex =
- groupOrd * (facetFieldTermsIndex.getValueCount() + 1) + facetOrd;
- if (segmentGroupedFacetHits.exists(segmentGroupedFacetsIndex)) {
- return;
- }
-
- segmentTotalCount++;
- segmentFacetCounts[facetOrd + 1]++;
-
- segmentGroupedFacetHits.put(segmentGroupedFacetsIndex);
-
- BytesRef groupKey;
- if (groupOrd == -1) {
- groupKey = null;
- } else {
- groupKey = BytesRef.deepCopyOf(groupFieldTermsIndex.lookupOrd(groupOrd));
- }
-
- BytesRef facetKey;
- if (facetOrd == -1) {
- facetKey = null;
- } else {
- facetKey = BytesRef.deepCopyOf(facetFieldTermsIndex.lookupOrd(facetOrd));
- }
-
- groupedFacetHits.add(new GroupedFacetHit(groupKey, facetKey));
- }
-
- @Override
- protected void doSetNextReader(LeafReaderContext context) throws IOException {
- groupFieldTermsIndex = DocValues.getSorted(context.reader(), groupField);
- facetFieldTermsIndex = DocValues.getSorted(context.reader(), facetField);
-
- // 1+ to allow for the -1 "not set":
- segmentFacetCounts = new int[facetFieldTermsIndex.getValueCount() + 1];
- segmentTotalCount = 0;
-
- segmentGroupedFacetHits.clear();
- for (GroupedFacetHit groupedFacetHit : groupedFacetHits) {
- int facetOrd =
- groupedFacetHit.facetValue == null
- ? -1
- : facetFieldTermsIndex.lookupTerm(groupedFacetHit.facetValue);
- if (groupedFacetHit.facetValue != null && facetOrd < 0) {
- continue;
- }
-
- int groupOrd =
- groupedFacetHit.groupValue == null
- ? -1
- : groupFieldTermsIndex.lookupTerm(groupedFacetHit.groupValue);
- if (groupedFacetHit.groupValue != null && groupOrd < 0) {
- continue;
- }
-
- int segmentGroupedFacetsIndex =
- groupOrd * (facetFieldTermsIndex.getValueCount() + 1) + facetOrd;
- segmentGroupedFacetHits.put(segmentGroupedFacetsIndex);
- }
-
- if (facetPrefix != null) {
- startFacetOrd = facetFieldTermsIndex.lookupTerm(facetPrefix);
- if (startFacetOrd < 0) {
- // Points to the ord one higher than facetPrefix
- startFacetOrd = -startFacetOrd - 1;
- }
- BytesRefBuilder facetEndPrefix = new BytesRefBuilder();
- facetEndPrefix.append(facetPrefix);
- facetEndPrefix.append(UnicodeUtil.BIG_TERM);
- endFacetOrd = facetFieldTermsIndex.lookupTerm(facetEndPrefix.get());
- assert endFacetOrd < 0;
- endFacetOrd = -endFacetOrd - 1; // Points to the ord one higher than facetEndPrefix
- } else {
- startFacetOrd = -1;
- endFacetOrd = facetFieldTermsIndex.getValueCount();
- }
- }
-
- @Override
- protected SegmentResult createSegmentResult() throws IOException {
- return new SegmentResult(
- segmentFacetCounts,
- segmentTotalCount,
- facetFieldTermsIndex.termsEnum(),
- startFacetOrd,
- endFacetOrd);
- }
-
- private static class SegmentResult extends GroupFacetCollector.SegmentResult {
-
- final TermsEnum tenum;
-
- SegmentResult(int[] counts, int total, TermsEnum tenum, int startFacetOrd, int endFacetOrd)
- throws IOException {
- super(counts, total - counts[0], counts[0], endFacetOrd + 1);
- this.tenum = tenum;
- this.mergePos = startFacetOrd == -1 ? 1 : startFacetOrd + 1;
- if (mergePos < maxTermPos) {
- assert tenum != null;
- tenum.seekExact(startFacetOrd == -1 ? 0 : startFacetOrd);
- mergeTerm = tenum.term();
- }
- }
-
- @Override
- protected void nextTerm() throws IOException {
- mergeTerm = tenum.next();
- }
- }
- }
-
- // Implementation for multi valued facet fields.
- static class MV extends TermGroupFacetCollector {
-
- private SortedSetDocValues facetFieldDocTermOrds;
- private TermsEnum facetOrdTermsEnum;
- private int facetFieldNumTerms;
-
- MV(String groupField, String facetField, BytesRef facetPrefix, int initialSize) {
- super(groupField, facetField, facetPrefix, initialSize);
- }
-
- @Override
- public void collect(int doc) throws IOException {
- if (doc > groupFieldTermsIndex.docID()) {
- groupFieldTermsIndex.advance(doc);
- }
-
- int groupOrd;
- if (doc == groupFieldTermsIndex.docID()) {
- groupOrd = groupFieldTermsIndex.ordValue();
- } else {
- groupOrd = -1;
- }
-
- if (facetFieldNumTerms == 0) {
- int segmentGroupedFacetsIndex = groupOrd * (facetFieldNumTerms + 1);
- if (facetPrefix != null || segmentGroupedFacetHits.exists(segmentGroupedFacetsIndex)) {
- return;
- }
-
- segmentTotalCount++;
- segmentFacetCounts[facetFieldNumTerms]++;
-
- segmentGroupedFacetHits.put(segmentGroupedFacetsIndex);
- BytesRef groupKey;
- if (groupOrd == -1) {
- groupKey = null;
- } else {
- groupKey = BytesRef.deepCopyOf(groupFieldTermsIndex.lookupOrd(groupOrd));
- }
- groupedFacetHits.add(new GroupedFacetHit(groupKey, null));
- return;
- }
-
- if (doc > facetFieldDocTermOrds.docID()) {
- facetFieldDocTermOrds.advance(doc);
- }
- boolean empty = true;
- if (doc == facetFieldDocTermOrds.docID()) {
- for (int i = 0; i < facetFieldDocTermOrds.docValueCount(); i++) {
- process(groupOrd, (int) facetFieldDocTermOrds.nextOrd());
- empty = false;
- }
- }
-
- if (empty) {
- process(
- groupOrd,
- facetFieldNumTerms); // this facet ord is reserved for docs not containing facet field.
- }
- }
-
- private void process(int groupOrd, int facetOrd) throws IOException {
- if (facetOrd < startFacetOrd || facetOrd >= endFacetOrd) {
- return;
- }
-
- int segmentGroupedFacetsIndex = groupOrd * (facetFieldNumTerms + 1) + facetOrd;
- if (segmentGroupedFacetHits.exists(segmentGroupedFacetsIndex)) {
- return;
- }
-
- segmentTotalCount++;
- segmentFacetCounts[facetOrd]++;
-
- segmentGroupedFacetHits.put(segmentGroupedFacetsIndex);
-
- BytesRef groupKey;
- if (groupOrd == -1) {
- groupKey = null;
- } else {
- groupKey = BytesRef.deepCopyOf(groupFieldTermsIndex.lookupOrd(groupOrd));
- }
-
- final BytesRef facetValue;
- if (facetOrd == facetFieldNumTerms) {
- facetValue = null;
- } else {
- facetValue = BytesRef.deepCopyOf(facetFieldDocTermOrds.lookupOrd(facetOrd));
- }
- groupedFacetHits.add(new GroupedFacetHit(groupKey, facetValue));
- }
-
- @Override
- protected void doSetNextReader(LeafReaderContext context) throws IOException {
- groupFieldTermsIndex = DocValues.getSorted(context.reader(), groupField);
- facetFieldDocTermOrds = DocValues.getSortedSet(context.reader(), facetField);
- facetFieldNumTerms = (int) facetFieldDocTermOrds.getValueCount();
- if (facetFieldNumTerms == 0) {
- facetOrdTermsEnum = null;
- } else {
- facetOrdTermsEnum = facetFieldDocTermOrds.termsEnum();
- }
- // [facetFieldNumTerms() + 1] for all possible facet values and docs not containing facet
- // field
- segmentFacetCounts = new int[facetFieldNumTerms + 1];
- segmentTotalCount = 0;
-
- segmentGroupedFacetHits.clear();
- for (GroupedFacetHit groupedFacetHit : groupedFacetHits) {
- int groupOrd =
- groupedFacetHit.groupValue == null
- ? -1
- : groupFieldTermsIndex.lookupTerm(groupedFacetHit.groupValue);
- if (groupedFacetHit.groupValue != null && groupOrd < 0) {
- continue;
- }
-
- int facetOrd;
- if (groupedFacetHit.facetValue != null) {
- if (facetOrdTermsEnum == null
- || !facetOrdTermsEnum.seekExact(groupedFacetHit.facetValue)) {
- continue;
- }
- facetOrd = (int) facetOrdTermsEnum.ord();
- } else {
- facetOrd = facetFieldNumTerms;
- }
-
- // (facetFieldDocTermOrds.numTerms() + 1) for all possible facet values and docs not
- // containing facet field
- int segmentGroupedFacetsIndex = groupOrd * (facetFieldNumTerms + 1) + facetOrd;
- segmentGroupedFacetHits.put(segmentGroupedFacetsIndex);
- }
-
- if (facetPrefix != null) {
- TermsEnum.SeekStatus seekStatus;
- if (facetOrdTermsEnum != null) {
- seekStatus = facetOrdTermsEnum.seekCeil(facetPrefix);
- } else {
- seekStatus = TermsEnum.SeekStatus.END;
- }
-
- if (seekStatus != TermsEnum.SeekStatus.END) {
- startFacetOrd = (int) facetOrdTermsEnum.ord();
- } else {
- startFacetOrd = 0;
- endFacetOrd = 0;
- return;
- }
-
- BytesRefBuilder facetEndPrefix = new BytesRefBuilder();
- facetEndPrefix.append(facetPrefix);
- facetEndPrefix.append(UnicodeUtil.BIG_TERM);
- seekStatus = facetOrdTermsEnum.seekCeil(facetEndPrefix.get());
- if (seekStatus != TermsEnum.SeekStatus.END) {
- endFacetOrd = (int) facetOrdTermsEnum.ord();
- } else {
- endFacetOrd = facetFieldNumTerms; // Don't include null...
- }
- } else {
- startFacetOrd = 0;
- endFacetOrd = facetFieldNumTerms + 1;
- }
- }
-
- @Override
- protected SegmentResult createSegmentResult() throws IOException {
- return new SegmentResult(
- segmentFacetCounts,
- segmentTotalCount,
- facetFieldNumTerms,
- facetOrdTermsEnum,
- startFacetOrd,
- endFacetOrd);
- }
-
- private static class SegmentResult extends GroupFacetCollector.SegmentResult {
-
- final TermsEnum tenum;
-
- SegmentResult(
- int[] counts,
- int total,
- int missingCountIndex,
- TermsEnum tenum,
- int startFacetOrd,
- int endFacetOrd)
- throws IOException {
- super(
- counts,
- total - counts[missingCountIndex],
- counts[missingCountIndex],
- endFacetOrd == missingCountIndex + 1 ? missingCountIndex : endFacetOrd);
- this.tenum = tenum;
- this.mergePos = startFacetOrd;
- if (tenum != null) {
- tenum.seekExact(mergePos);
- mergeTerm = tenum.term();
- }
- }
-
- @Override
- protected void nextTerm() throws IOException {
- mergeTerm = tenum.next();
- }
- }
- }
-
- private record GroupedFacetHit(BytesRef groupValue, BytesRef facetValue) {}
-}
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/TermGroupFacetCollectorManager.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/TermGroupFacetCollectorManager.java
index b74c6aea631..0a4db4de2f5 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/TermGroupFacetCollectorManager.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/TermGroupFacetCollectorManager.java
@@ -120,8 +120,7 @@ public class TermGroupFacetCollectorManager
}
// Terms must be presented to addFacetCount in ascending byte order: GroupedFacetResult's
- // internal currentMin optimization assumes sorted input (as mergeSegmentResults guarantees
- // via its priority queue). TreeSet gives us that order for free.
+ // internal currentMin optimization assumes sorted input. TreeSet gives us that order.
Set<BytesRef> sortedTerms = new TreeSet<>(facetCounts.keySet());
if (minCount == 0) {
// Union in-range terms from all collectors so that zero-count terms are included.
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGroupFacetCollector.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGroupFacetCollector.java
deleted file mode 100644
index 5e8d1ca9cd8..00000000000
--- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGroupFacetCollector.java
+++ /dev/null
@@ -1,807 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.grouping;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.Random;
-import java.util.Set;
-import java.util.TreeSet;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.SortedDocValuesField;
-import org.apache.lucene.document.SortedSetDocValuesField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.NoMergePolicy;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.tests.analysis.MockAnalyzer;
-import org.apache.lucene.tests.index.RandomIndexWriter;
-import org.apache.lucene.tests.util.TestUtil;
-import org.apache.lucene.util.BytesRef;
-
-public class TestGroupFacetCollector extends AbstractGroupingTestCase {
-
- public void testSimple() throws Exception {
- final String groupField = "hotel";
- FieldType customType = new FieldType();
- customType.setStored(true);
-
- Directory dir = newDirectory();
- RandomIndexWriter w =
- new RandomIndexWriter(
- random(),
- dir,
- newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
- boolean useDv = true;
-
- // 0
- Document doc = new Document();
- addField(doc, groupField, "a", useDv);
- addField(doc, "airport", "ams", useDv);
- addField(doc, "duration", "5", useDv);
- w.addDocument(doc);
-
- // 1
- doc = new Document();
- addField(doc, groupField, "a", useDv);
- addField(doc, "airport", "dus", useDv);
- addField(doc, "duration", "10", useDv);
- w.addDocument(doc);
-
- // 2
- doc = new Document();
- addField(doc, groupField, "b", useDv);
- addField(doc, "airport", "ams", useDv);
- addField(doc, "duration", "10", useDv);
- w.addDocument(doc);
- w.commit(); // To ensure a second segment
-
- // 3
- doc = new Document();
- addField(doc, groupField, "b", useDv);
- addField(doc, "airport", "ams", useDv);
- addField(doc, "duration", "5", useDv);
- w.addDocument(doc);
-
- // 4
- doc = new Document();
- addField(doc, groupField, "b", useDv);
- addField(doc, "airport", "ams", useDv);
- addField(doc, "duration", "5", useDv);
- w.addDocument(doc);
-
- IndexSearcher indexSearcher = newSearcher(w.getReader());
-
- List<TermGroupFacetCollector.FacetEntry> entries;
- GroupFacetCollector groupedAirportFacetCollector;
- TermGroupFacetCollector.GroupedFacetResult airportResult;
-
- for (int limit : new int[] {2, 10, 100, Integer.MAX_VALUE}) {
- // any of these limits is plenty for the data we have
-
- groupedAirportFacetCollector =
- createRandomCollector(
- useDv ? "hotel_dv" : "hotel", useDv ? "airport_dv" : "airport", null, false);
- indexSearcher.search(MatchAllDocsQuery.INSTANCE, groupedAirportFacetCollector);
- int maxOffset = 5;
- airportResult =
- groupedAirportFacetCollector.mergeSegmentResults(
- Integer.MAX_VALUE == limit ? limit : maxOffset + limit, 0, false);
-
- assertEquals(3, airportResult.getTotalCount());
- assertEquals(0, airportResult.getTotalMissingCount());
-
- entries = airportResult.getFacetEntries(maxOffset, limit);
- assertEquals(0, entries.size());
-
- entries = airportResult.getFacetEntries(0, limit);
- assertEquals(2, entries.size());
- assertEquals("ams", entries.get(0).value().utf8ToString());
- assertEquals(2, entries.get(0).count());
- assertEquals("dus", entries.get(1).value().utf8ToString());
- assertEquals(1, entries.get(1).count());
-
- entries = airportResult.getFacetEntries(1, limit);
- assertEquals(1, entries.size());
- assertEquals("dus", entries.get(0).value().utf8ToString());
- assertEquals(1, entries.get(0).count());
- }
-
- GroupFacetCollector groupedDurationFacetCollector =
- createRandomCollector(
- useDv ? "hotel_dv" : "hotel", useDv ? "duration_dv" : "duration", null, false);
- indexSearcher.search(MatchAllDocsQuery.INSTANCE, groupedDurationFacetCollector);
- TermGroupFacetCollector.GroupedFacetResult durationResult =
- groupedDurationFacetCollector.mergeSegmentResults(10, 0, false);
- assertEquals(4, durationResult.getTotalCount());
- assertEquals(0, durationResult.getTotalMissingCount());
-
- entries = durationResult.getFacetEntries(0, 10);
- assertEquals(2, entries.size());
- assertEquals("10", entries.get(0).value().utf8ToString());
- assertEquals(2, entries.get(0).count());
- assertEquals("5", entries.get(1).value().utf8ToString());
- assertEquals(2, entries.get(1).count());
-
- // 5
- doc = new Document();
- addField(doc, groupField, "b", useDv);
- // missing airport
- if (useDv) {
- addField(doc, "airport", "", useDv);
- }
- addField(doc, "duration", "5", useDv);
- w.addDocument(doc);
-
- // 6
- doc = new Document();
- addField(doc, groupField, "b", useDv);
- addField(doc, "airport", "bru", useDv);
- addField(doc, "duration", "10", useDv);
- w.addDocument(doc);
-
- // 7
- doc = new Document();
- addField(doc, groupField, "b", useDv);
- addField(doc, "airport", "bru", useDv);
- addField(doc, "duration", "15", useDv);
- w.addDocument(doc);
-
- // 8
- doc = new Document();
- addField(doc, groupField, "a", useDv);
- addField(doc, "airport", "bru", useDv);
- addField(doc, "duration", "10", useDv);
- w.addDocument(doc);
-
- indexSearcher.getIndexReader().close();
- indexSearcher = newSearcher(w.getReader());
- groupedAirportFacetCollector =
- createRandomCollector(
- useDv ? "hotel_dv" : "hotel", useDv ? "airport_dv" : "airport", null, !useDv);
- indexSearcher.search(MatchAllDocsQuery.INSTANCE, groupedAirportFacetCollector);
- airportResult = groupedAirportFacetCollector.mergeSegmentResults(3, 0, true);
- entries = airportResult.getFacetEntries(1, 2);
- assertEquals(2, entries.size());
- if (useDv) {
- assertEquals(6, airportResult.getTotalCount());
- assertEquals(0, airportResult.getTotalMissingCount());
- assertEquals("bru", entries.get(0).value().utf8ToString());
- assertEquals(2, entries.get(0).count());
- assertEquals("", entries.get(1).value().utf8ToString());
- assertEquals(1, entries.get(1).count());
- } else {
- assertEquals(5, airportResult.getTotalCount());
- assertEquals(1, airportResult.getTotalMissingCount());
- assertEquals("bru", entries.get(0).value().utf8ToString());
- assertEquals(2, entries.get(0).count());
- assertEquals("dus", entries.get(1).value().utf8ToString());
- assertEquals(1, entries.get(1).count());
- }
-
- groupedDurationFacetCollector =
- createRandomCollector(
- useDv ? "hotel_dv" : "hotel", useDv ? "duration_dv" : "duration", null, false);
- indexSearcher.search(MatchAllDocsQuery.INSTANCE, groupedDurationFacetCollector);
- durationResult = groupedDurationFacetCollector.mergeSegmentResults(10, 2, true);
- assertEquals(5, durationResult.getTotalCount());
- assertEquals(0, durationResult.getTotalMissingCount());
-
- entries = durationResult.getFacetEntries(1, 1);
- assertEquals(1, entries.size());
- assertEquals("5", entries.get(0).value().utf8ToString());
- assertEquals(2, entries.get(0).count());
-
- // 9
- doc = new Document();
- addField(doc, groupField, "c", useDv);
- addField(doc, "airport", "bru", useDv);
- addField(doc, "duration", "15", useDv);
- w.addDocument(doc);
-
- // 10
- doc = new Document();
- addField(doc, groupField, "c", useDv);
- addField(doc, "airport", "dus", useDv);
- addField(doc, "duration", "10", useDv);
- w.addDocument(doc);
-
- indexSearcher.getIndexReader().close();
- indexSearcher = newSearcher(w.getReader());
- groupedAirportFacetCollector =
- createRandomCollector(
- useDv ? "hotel_dv" : "hotel", useDv ? "airport_dv" : "airport", null, false);
- indexSearcher.search(MatchAllDocsQuery.INSTANCE, groupedAirportFacetCollector);
- airportResult = groupedAirportFacetCollector.mergeSegmentResults(10, 0, false);
- entries = airportResult.getFacetEntries(0, 10);
- if (useDv) {
- assertEquals(8, airportResult.getTotalCount());
- assertEquals(0, airportResult.getTotalMissingCount());
- assertEquals(4, entries.size());
- assertEquals("", entries.get(0).value().utf8ToString());
- assertEquals(1, entries.get(0).count());
- assertEquals("ams", entries.get(1).value().utf8ToString());
- assertEquals(2, entries.get(1).count());
- assertEquals("bru", entries.get(2).value().utf8ToString());
- assertEquals(3, entries.get(2).count());
- assertEquals("dus", entries.get(3).value().utf8ToString());
- assertEquals(2, entries.get(3).count());
- } else {
- assertEquals(7, airportResult.getTotalCount());
- assertEquals(1, airportResult.getTotalMissingCount());
- assertEquals(3, entries.size());
- assertEquals("ams", entries.get(0).value().utf8ToString());
- assertEquals(2, entries.get(0).count());
- assertEquals("bru", entries.get(1).value().utf8ToString());
- assertEquals(3, entries.get(1).count());
- assertEquals("dus", entries.get(2).value().utf8ToString());
- assertEquals(2, entries.get(2).count());
- }
-
- groupedDurationFacetCollector =
- createRandomCollector(
- useDv ? "hotel_dv" : "hotel", useDv ? "duration_dv" : "duration", "1", false);
- indexSearcher.search(MatchAllDocsQuery.INSTANCE, groupedDurationFacetCollector);
- durationResult = groupedDurationFacetCollector.mergeSegmentResults(10, 0, true);
- assertEquals(5, durationResult.getTotalCount());
- assertEquals(0, durationResult.getTotalMissingCount());
-
- entries = durationResult.getFacetEntries(0, 10);
- assertEquals(2, entries.size());
- assertEquals("10", entries.get(0).value().utf8ToString());
- assertEquals(3, entries.get(0).count());
- assertEquals("15", entries.get(1).value().utf8ToString());
- assertEquals(2, entries.get(1).count());
-
- w.close();
- indexSearcher.getIndexReader().close();
- dir.close();
- }
-
- public void testMVGroupedFacetingWithDeletes() throws Exception {
- final String groupField = "hotel";
- FieldType customType = new FieldType();
- customType.setStored(true);
-
- Directory dir = newDirectory();
- RandomIndexWriter w =
- new RandomIndexWriter(
- random(),
- dir,
- newIndexWriterConfig(new MockAnalyzer(random()))
- .setMergePolicy(NoMergePolicy.INSTANCE));
- boolean useDv = true;
-
- // Cannot assert this since we use NoMergePolicy:
- w.setDoRandomForceMergeAssert(false);
-
- // 0
- Document doc = new Document();
- doc.add(new StringField("x", "x", Field.Store.NO));
- w.addDocument(doc);
-
- // 1
- doc = new Document();
- addField(doc, groupField, "a", useDv);
- doc.add(new SortedSetDocValuesField("airport", new BytesRef("ams")));
- w.addDocument(doc);
-
- w.commit();
- w.deleteDocuments(new TermQuery(new Term("airport", "ams")));
-
- // 2
- doc = new Document();
- addField(doc, groupField, "a", useDv);
- doc.add(new SortedSetDocValuesField("airport", new BytesRef("ams")));
- w.addDocument(doc);
-
- // 3
- doc = new Document();
- addField(doc, groupField, "a", useDv);
- doc.add(new SortedSetDocValuesField("airport", new BytesRef("dus")));
- w.addDocument(doc);
-
- // 4
- doc = new Document();
- addField(doc, groupField, "b", useDv);
- doc.add(new SortedSetDocValuesField("airport", new BytesRef("ams")));
- w.addDocument(doc);
-
- // 5
- doc = new Document();
- addField(doc, groupField, "b", useDv);
- doc.add(new SortedSetDocValuesField("airport", new BytesRef("ams")));
- w.addDocument(doc);
-
- // 6
- doc = new Document();
- addField(doc, groupField, "b", useDv);
- doc.add(new SortedSetDocValuesField("airport", new BytesRef("ams")));
- w.addDocument(doc);
- w.commit();
-
- // 7
- doc = new Document();
- doc.add(new StringField("x", "x", Field.Store.NO));
- w.addDocument(doc);
- w.commit();
-
- w.close();
- IndexSearcher indexSearcher = newSearcher(DirectoryReader.open(dir));
- GroupFacetCollector groupedAirportFacetCollector =
- createRandomCollector(groupField + "_dv", "airport", null, true);
- indexSearcher.search(MatchAllDocsQuery.INSTANCE, groupedAirportFacetCollector);
- TermGroupFacetCollector.GroupedFacetResult airportResult =
- groupedAirportFacetCollector.mergeSegmentResults(10, 0, false);
- assertEquals(3, airportResult.getTotalCount());
- assertEquals(1, airportResult.getTotalMissingCount());
-
- List<TermGroupFacetCollector.FacetEntry> entries = airportResult.getFacetEntries(0, 10);
- assertEquals(2, entries.size());
- assertEquals("ams", entries.get(0).value().utf8ToString());
- assertEquals(2, entries.get(0).count());
- assertEquals("dus", entries.get(1).value().utf8ToString());
- assertEquals(1, entries.get(1).count());
-
- indexSearcher.getIndexReader().close();
- dir.close();
- }
-
- private void addField(Document doc, String field, String value, boolean canUseIDV) {
- assert canUseIDV;
- doc.add(new SortedDocValuesField(field + "_dv", new BytesRef(value)));
- }
-
- public void testRandom() throws Exception {
- Random random = random();
- int numberOfRuns = atLeast(1);
- for (int indexIter = 0; indexIter < numberOfRuns; indexIter++) {
- boolean multipleFacetsPerDocument = random.nextBoolean();
- IndexContext context = createIndexContext(multipleFacetsPerDocument);
- final IndexSearcher searcher = newSearcher(context.indexReader);
-
- if (VERBOSE) {
- System.out.println("TEST: searcher=" + searcher);
- }
-
- for (int searchIter = 0; searchIter < 100; searchIter++) {
- if (VERBOSE) {
- System.out.println("TEST: searchIter=" + searchIter);
- }
- String searchTerm = context.contentStrings[random.nextInt(context.contentStrings.length)];
- int limit = random.nextInt(context.facetValues.size());
- int offset = random.nextInt(context.facetValues.size() - limit);
- int size = offset + limit;
- int minCount =
- random.nextBoolean() ? 0 : random.nextInt(1 + context.facetWithMostGroups / 10);
- boolean orderByCount = random.nextBoolean();
- String randomStr =
- getFromSet(context.facetValues, random.nextInt(context.facetValues.size()));
- final String facetPrefix;
- if (randomStr == null) {
- facetPrefix = null;
- } else {
- int codePointLen = randomStr.codePointCount(0, randomStr.length());
- int randomLen = random.nextInt(codePointLen);
- if (codePointLen == randomLen - 1) {
- facetPrefix = null;
- } else {
- int end = randomStr.offsetByCodePoints(0, randomLen);
- facetPrefix = random.nextBoolean() ? null : randomStr.substring(end);
- }
- }
-
- GroupedFacetResult expectedFacetResult =
- createExpectedFacetResult(
- searchTerm, context, offset, limit, minCount, orderByCount, facetPrefix);
- GroupFacetCollector groupFacetCollector =
- createRandomCollector("group", "facet", facetPrefix, multipleFacetsPerDocument);
- searcher.search(new TermQuery(new Term("content", searchTerm)), groupFacetCollector);
- TermGroupFacetCollector.GroupedFacetResult actualFacetResult =
- groupFacetCollector.mergeSegmentResults(size, minCount, orderByCount);
-
- List<TermGroupFacetCollector.FacetEntry> expectedFacetEntries =
- expectedFacetResult.facetEntries();
- List<TermGroupFacetCollector.FacetEntry> actualFacetEntries =
- actualFacetResult.getFacetEntries(offset, limit);
-
- if (VERBOSE) {
- System.out.println("Collector: " + groupFacetCollector.getClass().getSimpleName());
- System.out.println("Num group: " + context.numGroups);
- System.out.println("Num doc: " + context.numDocs);
- System.out.println("Index iter: " + indexIter);
- System.out.println("multipleFacetsPerDocument: " + multipleFacetsPerDocument);
- System.out.println("Search iter: " + searchIter);
-
- System.out.println("Search term: " + searchTerm);
- System.out.println("Min count: " + minCount);
- System.out.println("Facet offset: " + offset);
- System.out.println("Facet limit: " + limit);
- System.out.println("Facet prefix: " + facetPrefix);
- System.out.println("Order by count: " + orderByCount);
-
- System.out.println("\n=== Expected: \n");
- System.out.println("Total count " + expectedFacetResult.totalCount());
- System.out.println("Total missing count " + expectedFacetResult.totalMissingCount());
- int counter = 0;
- for (TermGroupFacetCollector.FacetEntry expectedFacetEntry : expectedFacetEntries) {
- System.out.printf(
- Locale.ROOT,
- "%d. Expected facet value %s with count %d%n",
- counter++,
- expectedFacetEntry.value().utf8ToString(),
- expectedFacetEntry.count());
- }
-
- System.out.println("\n=== Actual: \n");
- System.out.println("Total count " + actualFacetResult.getTotalCount());
- System.out.println("Total missing count " + actualFacetResult.getTotalMissingCount());
- counter = 0;
- for (TermGroupFacetCollector.FacetEntry actualFacetEntry : actualFacetEntries) {
- System.out.printf(
- Locale.ROOT,
- "%d. Actual facet value %s with count %d%n",
- counter++,
- actualFacetEntry.value().utf8ToString(),
- actualFacetEntry.count());
- }
- System.out.println(
- "\n===================================================================================");
- }
-
- assertEquals(expectedFacetResult.totalCount(), actualFacetResult.getTotalCount());
- assertEquals(
- expectedFacetResult.totalMissingCount(), actualFacetResult.getTotalMissingCount());
- assertEquals(expectedFacetEntries.size(), actualFacetEntries.size());
- for (int i = 0; i < expectedFacetEntries.size(); i++) {
- TermGroupFacetCollector.FacetEntry expectedFacetEntry = expectedFacetEntries.get(i);
- TermGroupFacetCollector.FacetEntry actualFacetEntry = actualFacetEntries.get(i);
- assertEquals(
- "i="
- + i
- + ": "
- + expectedFacetEntry.value().utf8ToString()
- + " != "
- + actualFacetEntry.value().utf8ToString(),
- expectedFacetEntry.value(),
- actualFacetEntry.value());
- assertEquals(
- "i=" + i + ": " + expectedFacetEntry.count() + " != " + actualFacetEntry.count(),
- expectedFacetEntry.count(),
- actualFacetEntry.count());
- }
- }
-
- context.indexReader.close();
- context.dir.close();
- }
- }
-
- private IndexContext createIndexContext(boolean multipleFacetValuesPerDocument)
- throws IOException {
- final Random random = random();
- final int numDocs = TestUtil.nextInt(random, 138, 1145) * RANDOM_MULTIPLIER;
- final int numGroups = TestUtil.nextInt(random, 1, numDocs / 4);
- final int numFacets = TestUtil.nextInt(random, 1, numDocs / 6);
-
- if (VERBOSE) {
- System.out.println("TEST: numDocs=" + numDocs + " numGroups=" + numGroups);
- }
-
- final List<String> groups = new ArrayList<>();
- for (int i = 0; i < numGroups; i++) {
- groups.add(generateRandomNonEmptyString());
- }
- final List<String> facetValues = new ArrayList<>();
- for (int i = 0; i < numFacets; i++) {
- facetValues.add(generateRandomNonEmptyString());
- }
- final String[] contentBrs = new String[TestUtil.nextInt(random, 2, 20)];
- if (VERBOSE) {
- System.out.println("TEST: create fake content");
- }
- for (int contentIDX = 0; contentIDX < contentBrs.length; contentIDX++) {
- contentBrs[contentIDX] = generateRandomNonEmptyString();
- if (VERBOSE) {
- System.out.println(" content=" + contentBrs[contentIDX]);
- }
- }
-
- Directory dir = newDirectory();
- RandomIndexWriter writer =
- new RandomIndexWriter(random, dir, newIndexWriterConfig(new MockAnalyzer(random)));
- Document doc = new Document();
- Document docNoGroup = new Document();
- Document docNoFacet = new Document();
- Document docNoGroupNoFacet = new Document();
- Field group = newStringField("group", "", Field.Store.NO);
- Field groupDc = new SortedDocValuesField("group", new BytesRef());
- doc.add(groupDc);
- docNoFacet.add(groupDc);
- doc.add(group);
- docNoFacet.add(group);
- Field[] facetFields;
- if (multipleFacetValuesPerDocument == false) {
- facetFields = new Field[2];
- facetFields[0] = newStringField("facet", "", Field.Store.NO);
- doc.add(facetFields[0]);
- docNoGroup.add(facetFields[0]);
- facetFields[1] = new SortedDocValuesField("facet", new BytesRef());
- doc.add(facetFields[1]);
- docNoGroup.add(facetFields[1]);
- } else {
- facetFields =
- multipleFacetValuesPerDocument ? new Field[2 + random.nextInt(6)] : new Field[1];
- for (int i = 0; i < facetFields.length; i++) {
- facetFields[i] = new SortedSetDocValuesField("facet", new BytesRef());
- doc.add(facetFields[i]);
- docNoGroup.add(facetFields[i]);
- }
- }
- Field content = newStringField("content", "", Field.Store.NO);
- doc.add(content);
- docNoGroup.add(content);
- docNoFacet.add(content);
- docNoGroupNoFacet.add(content);
-
- NavigableSet<String> uniqueFacetValues =
- new TreeSet<>(
- (a, b) -> {
- if (a == b) {
- return 0;
- } else if (a == null) {
- return -1;
- } else if (b == null) {
- return 1;
- } else {
- return a.compareTo(b);
- }
- });
- Map<String, Map<String, Set<String>>> searchTermToFacetToGroups = new HashMap<>();
- int facetWithMostGroups = 0;
- for (int i = 0; i < numDocs; i++) {
- final String groupValue;
- if (random.nextInt(24) == 17) {
- // So we test the "doc doesn't have the group'd
- // field" case:
- groupValue = "";
- } else {
- groupValue = groups.get(random.nextInt(groups.size()));
- }
-
- String contentStr = contentBrs[random.nextInt(contentBrs.length)];
- if (!searchTermToFacetToGroups.containsKey(contentStr)) {
- searchTermToFacetToGroups.put(contentStr, new HashMap<>());
- }
- Map<String, Set<String>> facetToGroups = searchTermToFacetToGroups.get(contentStr);
-
- List<String> facetVals = new ArrayList<>();
- if (multipleFacetValuesPerDocument == false) {
- String facetValue = facetValues.get(random.nextInt(facetValues.size()));
- uniqueFacetValues.add(facetValue);
- if (!facetToGroups.containsKey(facetValue)) {
- facetToGroups.put(facetValue, new HashSet<>());
- }
- Set<String> groupsInFacet = facetToGroups.get(facetValue);
- groupsInFacet.add(groupValue);
- if (groupsInFacet.size() > facetWithMostGroups) {
- facetWithMostGroups = groupsInFacet.size();
- }
- facetFields[0].setStringValue(facetValue);
- facetFields[1].setBytesValue(new BytesRef(facetValue));
- facetVals.add(facetValue);
- } else {
- for (Field facetField : facetFields) {
- String facetValue = facetValues.get(random.nextInt(facetValues.size()));
- uniqueFacetValues.add(facetValue);
- if (!facetToGroups.containsKey(facetValue)) {
- facetToGroups.put(facetValue, new HashSet<>());
- }
- Set<String> groupsInFacet = facetToGroups.get(facetValue);
- groupsInFacet.add(groupValue);
- if (groupsInFacet.size() > facetWithMostGroups) {
- facetWithMostGroups = groupsInFacet.size();
- }
- facetField.setBytesValue(new BytesRef(facetValue));
- facetVals.add(facetValue);
- }
- }
-
- if (VERBOSE) {
- System.out.println(
- " doc content="
- + contentStr
- + " group="
- + (groupValue == null ? "null" : groupValue)
- + " facetVals="
- + facetVals);
- }
-
- if (groupValue != null) {
- groupDc.setBytesValue(new BytesRef(groupValue));
- group.setStringValue(groupValue);
- } else {
- // TODO: not true
- // DV cannot have missing values:
- groupDc.setBytesValue(new BytesRef());
- }
- content.setStringValue(contentStr);
- if (groupValue == null && facetVals.isEmpty()) {
- writer.addDocument(docNoGroupNoFacet);
- } else if (facetVals.isEmpty()) {
- writer.addDocument(docNoFacet);
- } else if (groupValue == null) {
- writer.addDocument(docNoGroup);
- } else {
- writer.addDocument(doc);
- }
- }
-
- DirectoryReader reader = writer.getReader();
- writer.close();
-
- return new IndexContext(
- searchTermToFacetToGroups,
- reader,
- numDocs,
- dir,
- facetWithMostGroups,
- numGroups,
- contentBrs,
- uniqueFacetValues);
- }
-
- private GroupedFacetResult createExpectedFacetResult(
- String searchTerm,
- IndexContext context,
- int offset,
- int limit,
- int minCount,
- final boolean orderByCount,
- String facetPrefix) {
- Map<String, Set<String>> facetGroups = context.searchTermToFacetGroups.get(searchTerm);
- if (facetGroups == null) {
- facetGroups = new HashMap<>();
- }
-
- int totalCount = 0;
- int totalMissCount = 0;
- Set<String> facetValues;
- if (facetPrefix != null) {
- facetValues = new HashSet<>();
- for (String facetValue : context.facetValues) {
- if (facetValue != null && facetValue.startsWith(facetPrefix)) {
- facetValues.add(facetValue);
- }
- }
- } else {
- facetValues = context.facetValues;
- }
-
- List<TermGroupFacetCollector.FacetEntry> entries = new ArrayList<>(facetGroups.size());
- // also includes facets with count 0
- for (String facetValue : facetValues) {
- if (facetValue == null) {
- continue;
- }
-
- Set<String> groups = facetGroups.get(facetValue);
- int count = groups != null ? groups.size() : 0;
- if (count >= minCount) {
- entries.add(new TermGroupFacetCollector.FacetEntry(new BytesRef(facetValue), count));
- }
- totalCount += count;
- }
-
- // Only include null count when no facet prefix is specified
- if (facetPrefix == null) {
- Set<String> groups = facetGroups.get(null);
- if (groups != null) {
- totalMissCount = groups.size();
- }
- }
-
- entries.sort(
- (a, b) -> {
- if (orderByCount) {
- int cmp = b.count() - a.count();
- if (cmp != 0) {
- return cmp;
- }
- }
- return a.value().compareTo(b.value());
- });
-
- int endOffset = offset + limit;
- List<TermGroupFacetCollector.FacetEntry> entriesResult;
- if (offset >= entries.size()) {
- entriesResult = Collections.emptyList();
- } else if (endOffset >= entries.size()) {
- entriesResult = entries.subList(offset, entries.size());
- } else {
- entriesResult = entries.subList(offset, endOffset);
- }
- return new GroupedFacetResult(totalCount, totalMissCount, entriesResult);
- }
-
- private GroupFacetCollector createRandomCollector(
- String groupField, String facetField, String facetPrefix, boolean multipleFacetsPerDocument) {
- BytesRef facetPrefixBR = facetPrefix == null ? null : new BytesRef(facetPrefix);
- return TermGroupFacetCollector.createTermGroupFacetCollector(
- groupField, facetField, multipleFacetsPerDocument, facetPrefixBR, random().nextInt(1024));
- }
-
- private String getFromSet(Set<String> set, int index) {
- int currentIndex = 0;
- for (String bytesRef : set) {
- if (currentIndex++ == index) {
- return bytesRef;
- }
- }
-
- return null;
- }
-
- private static class IndexContext {
-
- final int numDocs;
- final DirectoryReader indexReader;
- final Map<String, Map<String, Set<String>>> searchTermToFacetGroups;
- final NavigableSet<String> facetValues;
- final Directory dir;
- final int facetWithMostGroups;
- final int numGroups;
- final String[] contentStrings;
-
- public IndexContext(
- Map<String, Map<String, Set<String>>> searchTermToFacetGroups,
- DirectoryReader r,
- int numDocs,
- Directory dir,
- int facetWithMostGroups,
- int numGroups,
- String[] contentStrings,
- NavigableSet<String> facetValues) {
- this.searchTermToFacetGroups = searchTermToFacetGroups;
- this.indexReader = r;
- this.numDocs = numDocs;
- this.dir = dir;
- this.facetWithMostGroups = facetWithMostGroups;
- this.numGroups = numGroups;
- this.contentStrings = contentStrings;
- this.facetValues = facetValues;
- }
- }
-
- private record GroupedFacetResult(
- int totalCount,
- int totalMissingCount,
- List<TermGroupFacetCollector.FacetEntry> facetEntries) {}
-}