You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/05/19 12:34:40 UTC
svn commit: r1595824 [5/9] - in /lucene/dev/branches/lucene5675: ./ lucene/
lucene/analysis/ lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/collation/
lucene/analysis/common/src/test/org/apache/lucene/collation/ lucene/analys...
Modified: lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupsCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupsCollector.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupsCollector.java (original)
+++ lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupsCollector.java Mon May 19 10:34:35 2014
@@ -18,9 +18,9 @@ package org.apache.lucene.search.groupin
*/
import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.LeafCollector;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.grouping.AbstractAllGroupsCollector;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SentinelIntSet;
@@ -105,7 +105,7 @@ public class TermAllGroupsCollector exte
@Override
protected void doSetNextReader(AtomicReaderContext context) throws IOException {
- index = FieldCache.DEFAULT.getTermsIndex(context.reader(), groupField);
+ index = DocValues.getSorted(context.reader(), groupField);
// Clear ordSet and fill it with previous encountered groups that can occur in the current segment.
ordSet.clear();
Modified: lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermDistinctValuesCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermDistinctValuesCollector.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermDistinctValuesCollector.java (original)
+++ lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermDistinctValuesCollector.java Mon May 19 10:34:35 2014
@@ -18,9 +18,9 @@ package org.apache.lucene.search.groupin
*/
import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.LeafCollector;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.grouping.AbstractDistinctValuesCollector;
import org.apache.lucene.search.grouping.SearchGroup;
import org.apache.lucene.util.BytesRef;
@@ -109,8 +109,8 @@ public class TermDistinctValuesCollector
@Override
protected void doSetNextReader(AtomicReaderContext context) throws IOException {
- groupFieldTermIndex = FieldCache.DEFAULT.getTermsIndex(context.reader(), groupField);
- countFieldTermIndex = FieldCache.DEFAULT.getTermsIndex(context.reader(), countField);
+ groupFieldTermIndex = DocValues.getSorted(context.reader(), groupField);
+ countFieldTermIndex = DocValues.getSorted(context.reader(), countField);
ordSet.clear();
for (GroupCount group : groups) {
int groupOrd = group.groupValue == null ? -1 : groupFieldTermIndex.lookupTerm(group.groupValue);
Modified: lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java (original)
+++ lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java Mon May 19 10:34:35 2014
@@ -20,9 +20,9 @@ package org.apache.lucene.search.groupin
import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.LeafCollector;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.grouping.AbstractFirstPassGroupingCollector;
import org.apache.lucene.util.BytesRef;
@@ -46,7 +46,7 @@ public class TermFirstPassGroupingCollec
*
* @param groupField The field used to group
* documents. This field must be single-valued and
- * indexed (FieldCache is used to access its value
+ * indexed (DocValues is used to access its value
* per-document).
* @param groupSort The {@link Sort} used to sort the
* groups. The top sorted document within each group
@@ -88,6 +88,6 @@ public class TermFirstPassGroupingCollec
@Override
protected void doSetNextReader(AtomicReaderContext readerContext) throws IOException {
super.doSetNextReader(readerContext);
- index = FieldCache.DEFAULT.getTermsIndex(readerContext.reader(), groupField);
+ index = DocValues.getSorted(readerContext.reader(), groupField);
}
}
Modified: lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java (original)
+++ lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java Mon May 19 10:34:35 2014
@@ -18,11 +18,11 @@ package org.apache.lucene.search.groupin
*/
import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.LeafCollector;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.grouping.AbstractGroupFacetCollector;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SentinelIntSet;
@@ -34,7 +34,7 @@ import java.util.List;
/**
* An implementation of {@link AbstractGroupFacetCollector} that computes grouped facets based on the indexed terms
- * from the {@link FieldCache}.
+ * from DocValues.
*
* @lucene.experimental
*/
@@ -128,8 +128,8 @@ public abstract class TermGroupFacetColl
segmentResults.add(createSegmentResult());
}
- groupFieldTermsIndex = FieldCache.DEFAULT.getTermsIndex(context.reader(), groupField);
- facetFieldTermsIndex = FieldCache.DEFAULT.getTermsIndex(context.reader(), facetField);
+ groupFieldTermsIndex = DocValues.getSorted(context.reader(), groupField);
+ facetFieldTermsIndex = DocValues.getSorted(context.reader(), facetField);
// 1+ to allow for the -1 "not set":
segmentFacetCounts = new int[facetFieldTermsIndex.getValueCount()+1];
@@ -283,8 +283,8 @@ public abstract class TermGroupFacetColl
segmentResults.add(createSegmentResult());
}
- groupFieldTermsIndex = FieldCache.DEFAULT.getTermsIndex(context.reader(), groupField);
- facetFieldDocTermOrds = FieldCache.DEFAULT.getDocTermOrds(context.reader(), facetField);
+ groupFieldTermsIndex = DocValues.getSorted(context.reader(), groupField);
+ facetFieldDocTermOrds = DocValues.getSortedSet(context.reader(), facetField);
facetFieldNumTerms = (int) facetFieldDocTermOrds.getValueCount();
if (facetFieldNumTerms == 0) {
facetOrdTermsEnum = null;
Modified: lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermSecondPassGroupingCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermSecondPassGroupingCollector.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermSecondPassGroupingCollector.java (original)
+++ lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermSecondPassGroupingCollector.java Mon May 19 10:34:35 2014
@@ -21,9 +21,9 @@ import java.io.IOException;
import java.util.Collection;
import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.LeafCollector;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.grouping.AbstractSecondPassGroupingCollector;
import org.apache.lucene.search.grouping.SearchGroup;
@@ -56,7 +56,7 @@ public class TermSecondPassGroupingColle
@Override
protected void doSetNextReader(AtomicReaderContext readerContext) throws IOException {
super.doSetNextReader(readerContext);
- index = FieldCache.DEFAULT.getTermsIndex(readerContext.reader(), groupField);
+ index = DocValues.getSorted(readerContext.reader(), groupField);
// Rebuild ordSet
ordSet.clear();
Modified: lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/package.html?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/package.html (original)
+++ lucene/dev/branches/lucene5675/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/package.html Mon May 19 10:34:35 2014
@@ -16,6 +16,6 @@
-->
<html>
<body>
-Support for grouping by indexed terms via {@link org.apache.lucene.search.FieldCache}.
+Support for grouping by indexed terms via {@link org.apache.lucene.index.DocValues}.
</body>
</html>
Modified: lucene/dev/branches/lucene5675/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java (original)
+++ lucene/dev/branches/lucene5675/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java Mon May 19 10:34:35 2014
@@ -17,23 +17,35 @@ package org.apache.lucene.search.groupin
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource;
import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.QueryUtils;
import org.apache.lucene.search.ScoreDoc;
@@ -48,22 +60,8 @@ import org.apache.lucene.util.FixedBitSe
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
public class AllGroupHeadsCollectorTest extends LuceneTestCase {
- private static final DocValuesType[] vts = new DocValuesType[]{
- DocValuesType.BINARY, DocValuesType.SORTED
- };
-
public void testBasic() throws Exception {
final String groupField = "author";
Directory dir = newDirectory();
@@ -72,30 +70,30 @@ public class AllGroupHeadsCollectorTest
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
- DocValuesType valueType = vts[random().nextInt(vts.length)];
+ DocValuesType valueType = DocValuesType.SORTED;
// 0
Document doc = new Document();
addGroupField(doc, groupField, "author1", valueType);
doc.add(newTextField("content", "random text", Field.Store.NO));
- doc.add(new IntField("id_1", 1, Field.Store.NO));
- doc.add(newStringField("id_2", "1", Field.Store.NO));
+ doc.add(new NumericDocValuesField("id_1", 1));
+ doc.add(new SortedDocValuesField("id_2", new BytesRef("1")));
w.addDocument(doc);
// 1
doc = new Document();
addGroupField(doc, groupField, "author1", valueType);
doc.add(newTextField("content", "some more random text blob", Field.Store.NO));
- doc.add(new IntField("id_1", 2, Field.Store.NO));
- doc.add(newStringField("id_2", "2", Field.Store.NO));
+ doc.add(new NumericDocValuesField("id_1", 2));
+ doc.add(new SortedDocValuesField("id_2", new BytesRef("2")));
w.addDocument(doc);
// 2
doc = new Document();
addGroupField(doc, groupField, "author1", valueType);
doc.add(newTextField("content", "some more random textual data", Field.Store.NO));
- doc.add(new IntField("id_1", 3, Field.Store.NO));
- doc.add(newStringField("id_2", "3", Field.Store.NO));
+ doc.add(new NumericDocValuesField("id_1", 3));
+ doc.add(new SortedDocValuesField("id_2", new BytesRef("3")));
w.addDocument(doc);
w.commit(); // To ensure a second segment
@@ -103,38 +101,38 @@ public class AllGroupHeadsCollectorTest
doc = new Document();
addGroupField(doc, groupField, "author2", valueType);
doc.add(newTextField("content", "some random text", Field.Store.NO));
- doc.add(new IntField("id_1", 4, Field.Store.NO));
- doc.add(newStringField("id_2", "4", Field.Store.NO));
+ doc.add(new NumericDocValuesField("id_1", 4));
+ doc.add(new SortedDocValuesField("id_2", new BytesRef("4")));
w.addDocument(doc);
// 4
doc = new Document();
addGroupField(doc, groupField, "author3", valueType);
doc.add(newTextField("content", "some more random text", Field.Store.NO));
- doc.add(new IntField("id_1", 5, Field.Store.NO));
- doc.add(newStringField("id_2", "5", Field.Store.NO));
+ doc.add(new NumericDocValuesField("id_1", 5));
+ doc.add(new SortedDocValuesField("id_2", new BytesRef("5")));
w.addDocument(doc);
// 5
doc = new Document();
addGroupField(doc, groupField, "author3", valueType);
doc.add(newTextField("content", "random blob", Field.Store.NO));
- doc.add(new IntField("id_1", 6, Field.Store.NO));
- doc.add(newStringField("id_2", "6", Field.Store.NO));
+ doc.add(new NumericDocValuesField("id_1", 6));
+ doc.add(new SortedDocValuesField("id_2", new BytesRef("6")));
w.addDocument(doc);
// 6 -- no author field
doc = new Document();
doc.add(newTextField("content", "random word stuck in alot of other text", Field.Store.NO));
- doc.add(new IntField("id_1", 6, Field.Store.NO));
- doc.add(newStringField("id_2", "6", Field.Store.NO));
+ doc.add(new NumericDocValuesField("id_1", 6));
+ doc.add(new SortedDocValuesField("id_2", new BytesRef("6")));
w.addDocument(doc);
// 7 -- no author field
doc = new Document();
doc.add(newTextField("content", "random word stuck in alot of other text", Field.Store.NO));
- doc.add(new IntField("id_1", 7, Field.Store.NO));
- doc.add(newStringField("id_2", "7", Field.Store.NO));
+ doc.add(new NumericDocValuesField("id_1", 7));
+ doc.add(new SortedDocValuesField("id_2", new BytesRef("7")));
w.addDocument(doc);
IndexReader reader = w.getReader();
@@ -198,6 +196,7 @@ public class AllGroupHeadsCollectorTest
// B/c of DV based impl we can't see the difference between an empty string and a null value.
// For that reason we don't generate empty string groups.
randomValue = TestUtil.randomRealisticUnicodeString(random());
+ //randomValue = TestUtil.randomSimpleString(random());
} while ("".equals(randomValue));
groups.add(new BytesRef(randomValue));
}
@@ -224,31 +223,20 @@ public class AllGroupHeadsCollectorTest
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random())));
- DocValuesType valueType = vts[random().nextInt(vts.length)];
+ DocValuesType valueType = DocValuesType.SORTED;
Document doc = new Document();
Document docNoGroup = new Document();
- Field group = newStringField("group", "", Field.Store.NO);
- doc.add(group);
Field valuesField = null;
- switch(valueType) {
- case BINARY:
- valuesField = new BinaryDocValuesField("group_dv", new BytesRef());
- break;
- case SORTED:
- valuesField = new SortedDocValuesField("group_dv", new BytesRef());
- break;
- default:
- fail("unhandled type");
- }
+ valuesField = new SortedDocValuesField("group", new BytesRef());
doc.add(valuesField);
- Field sort1 = newStringField("sort1", "", Field.Store.NO);
+ Field sort1 = new SortedDocValuesField("sort1", new BytesRef());
doc.add(sort1);
docNoGroup.add(sort1);
- Field sort2 = newStringField("sort2", "", Field.Store.NO);
+ Field sort2 = new SortedDocValuesField("sort2", new BytesRef());
doc.add(sort2);
docNoGroup.add(sort2);
- Field sort3 = newStringField("sort3", "", Field.Store.NO);
+ Field sort3 = new SortedDocValuesField("sort3", new BytesRef());
doc.add(sort3);
docNoGroup.add(sort3);
Field content = newTextField("content", "", Field.Store.NO);
@@ -257,6 +245,9 @@ public class AllGroupHeadsCollectorTest
IntField id = new IntField("id", 0, Field.Store.NO);
doc.add(id);
docNoGroup.add(id);
+ NumericDocValuesField idDV = new NumericDocValuesField("id", 0);
+ doc.add(idDV);
+ docNoGroup.add(idDV);
final GroupDoc[] groupDocs = new GroupDoc[numDocs];
for (int i = 0; i < numDocs; i++) {
final BytesRef groupValue;
@@ -283,14 +274,14 @@ public class AllGroupHeadsCollectorTest
groupDocs[i] = groupDoc;
if (groupDoc.group != null) {
- group.setStringValue(groupDoc.group.utf8ToString());
valuesField.setBytesValue(new BytesRef(groupDoc.group.utf8ToString()));
}
- sort1.setStringValue(groupDoc.sort1.utf8ToString());
- sort2.setStringValue(groupDoc.sort2.utf8ToString());
- sort3.setStringValue(groupDoc.sort3.utf8ToString());
+ sort1.setBytesValue(groupDoc.sort1);
+ sort2.setBytesValue(groupDoc.sort2);
+ sort3.setBytesValue(groupDoc.sort3);
content.setStringValue(groupDoc.content);
id.setIntValue(groupDoc.id);
+ idDV.setLongValue(groupDoc.id);
if (groupDoc.group == null) {
w.addDocument(docNoGroup);
} else {
@@ -301,91 +292,86 @@ public class AllGroupHeadsCollectorTest
final DirectoryReader r = w.getReader();
w.shutdown();
- // NOTE: intentional but temporary field cache insanity!
- final FieldCache.Ints docIdToFieldId = FieldCache.DEFAULT.getInts(SlowCompositeReaderWrapper.wrap(r), "id", false);
+ final NumericDocValues docIdToFieldId = MultiDocValues.getNumericValues(r, "id");
final int[] fieldIdToDocID = new int[numDocs];
for (int i = 0; i < numDocs; i++) {
- int fieldId = docIdToFieldId.get(i);
+ int fieldId = (int) docIdToFieldId.get(i);
fieldIdToDocID[fieldId] = i;
}
- try {
- final IndexSearcher s = newSearcher(r);
-
- for (int contentID = 0; contentID < 3; contentID++) {
- final ScoreDoc[] hits = s.search(new TermQuery(new Term("content", "real" + contentID)), numDocs).scoreDocs;
- for (ScoreDoc hit : hits) {
- final GroupDoc gd = groupDocs[docIdToFieldId.get(hit.doc)];
- assertTrue(gd.score == 0.0);
- gd.score = hit.score;
- int docId = gd.id;
- assertEquals(docId, docIdToFieldId.get(hit.doc));
- }
- }
-
- for (GroupDoc gd : groupDocs) {
- assertTrue(gd.score != 0.0);
+ final IndexSearcher s = newSearcher(r);
+
+ for (int contentID = 0; contentID < 3; contentID++) {
+ final ScoreDoc[] hits = s.search(new TermQuery(new Term("content", "real" + contentID)), numDocs).scoreDocs;
+ for (ScoreDoc hit : hits) {
+ final GroupDoc gd = groupDocs[(int) docIdToFieldId.get(hit.doc)];
+ assertTrue(gd.score == 0.0);
+ gd.score = hit.score;
+ int docId = gd.id;
+ assertEquals(docId, docIdToFieldId.get(hit.doc));
+ }
+ }
+
+ for (GroupDoc gd : groupDocs) {
+ assertTrue(gd.score != 0.0);
+ }
+
+ for (int searchIter = 0; searchIter < 100; searchIter++) {
+
+ if (VERBOSE) {
+ System.out.println("TEST: searchIter=" + searchIter);
}
-
- for (int searchIter = 0; searchIter < 100; searchIter++) {
-
- if (VERBOSE) {
- System.out.println("TEST: searchIter=" + searchIter);
+
+ final String searchTerm = "real" + random().nextInt(3);
+ boolean sortByScoreOnly = random().nextBoolean();
+ Sort sortWithinGroup = getRandomSort(sortByScoreOnly);
+ AbstractAllGroupHeadsCollector<?> allGroupHeadsCollector = createRandomCollector("group", sortWithinGroup);
+ s.search(new TermQuery(new Term("content", searchTerm)), allGroupHeadsCollector);
+ int[] expectedGroupHeads = createExpectedGroupHeads(searchTerm, groupDocs, sortWithinGroup, sortByScoreOnly, fieldIdToDocID);
+ int[] actualGroupHeads = allGroupHeadsCollector.retrieveGroupHeads();
+ // The actual group heads contains Lucene ids. Need to change them into our id value.
+ for (int i = 0; i < actualGroupHeads.length; i++) {
+ actualGroupHeads[i] = (int) docIdToFieldId.get(actualGroupHeads[i]);
+ }
+ // Allows us the easily iterate and assert the actual and expected results.
+ Arrays.sort(expectedGroupHeads);
+ Arrays.sort(actualGroupHeads);
+
+ if (VERBOSE) {
+ System.out.println("Collector: " + allGroupHeadsCollector.getClass().getSimpleName());
+ System.out.println("Sort within group: " + sortWithinGroup);
+ System.out.println("Num group: " + numGroups);
+ System.out.println("Num doc: " + numDocs);
+ System.out.println("\n=== Expected: \n");
+ for (int expectedDocId : expectedGroupHeads) {
+ GroupDoc expectedGroupDoc = groupDocs[expectedDocId];
+ String expectedGroup = expectedGroupDoc.group == null ? null : expectedGroupDoc.group.utf8ToString();
+ System.out.println(
+ String.format(Locale.ROOT,
+ "Group:%10s score%5f Sort1:%10s Sort2:%10s Sort3:%10s doc:%5d",
+ expectedGroup, expectedGroupDoc.score, expectedGroupDoc.sort1.utf8ToString(),
+ expectedGroupDoc.sort2.utf8ToString(), expectedGroupDoc.sort3.utf8ToString(), expectedDocId
+ )
+ );
}
-
- final String searchTerm = "real" + random().nextInt(3);
- boolean sortByScoreOnly = random().nextBoolean();
- Sort sortWithinGroup = getRandomSort(sortByScoreOnly);
- AbstractAllGroupHeadsCollector<?> allGroupHeadsCollector = createRandomCollector("group", sortWithinGroup);
- s.search(new TermQuery(new Term("content", searchTerm)), allGroupHeadsCollector);
- int[] expectedGroupHeads = createExpectedGroupHeads(searchTerm, groupDocs, sortWithinGroup, sortByScoreOnly, fieldIdToDocID);
- int[] actualGroupHeads = allGroupHeadsCollector.retrieveGroupHeads();
- // The actual group heads contains Lucene ids. Need to change them into our id value.
- for (int i = 0; i < actualGroupHeads.length; i++) {
- actualGroupHeads[i] = docIdToFieldId.get(actualGroupHeads[i]);
+ System.out.println("\n=== Actual: \n");
+ for (int actualDocId : actualGroupHeads) {
+ GroupDoc actualGroupDoc = groupDocs[actualDocId];
+ String actualGroup = actualGroupDoc.group == null ? null : actualGroupDoc.group.utf8ToString();
+ System.out.println(
+ String.format(Locale.ROOT,
+ "Group:%10s score%5f Sort1:%10s Sort2:%10s Sort3:%10s doc:%5d",
+ actualGroup, actualGroupDoc.score, actualGroupDoc.sort1.utf8ToString(),
+ actualGroupDoc.sort2.utf8ToString(), actualGroupDoc.sort3.utf8ToString(), actualDocId
+ )
+ );
}
- // Allows us the easily iterate and assert the actual and expected results.
- Arrays.sort(expectedGroupHeads);
- Arrays.sort(actualGroupHeads);
-
- if (VERBOSE) {
- System.out.println("Collector: " + allGroupHeadsCollector.getClass().getSimpleName());
- System.out.println("Sort within group: " + sortWithinGroup);
- System.out.println("Num group: " + numGroups);
- System.out.println("Num doc: " + numDocs);
- System.out.println("\n=== Expected: \n");
- for (int expectedDocId : expectedGroupHeads) {
- GroupDoc expectedGroupDoc = groupDocs[expectedDocId];
- String expectedGroup = expectedGroupDoc.group == null ? null : expectedGroupDoc.group.utf8ToString();
- System.out.println(
- String.format(Locale.ROOT,
- "Group:%10s score%5f Sort1:%10s Sort2:%10s Sort3:%10s doc:%5d",
- expectedGroup, expectedGroupDoc.score, expectedGroupDoc.sort1.utf8ToString(),
- expectedGroupDoc.sort2.utf8ToString(), expectedGroupDoc.sort3.utf8ToString(), expectedDocId
- )
- );
- }
- System.out.println("\n=== Actual: \n");
- for (int actualDocId : actualGroupHeads) {
- GroupDoc actualGroupDoc = groupDocs[actualDocId];
- String actualGroup = actualGroupDoc.group == null ? null : actualGroupDoc.group.utf8ToString();
- System.out.println(
- String.format(Locale.ROOT,
- "Group:%10s score%5f Sort1:%10s Sort2:%10s Sort3:%10s doc:%5d",
- actualGroup, actualGroupDoc.score, actualGroupDoc.sort1.utf8ToString(),
- actualGroupDoc.sort2.utf8ToString(), actualGroupDoc.sort3.utf8ToString(), actualDocId
- )
- );
- }
- System.out.println("\n===================================================================================");
- }
-
- assertArrayEquals(expectedGroupHeads, actualGroupHeads);
+ System.out.println("\n===================================================================================");
}
- } finally {
- QueryUtils.purgeFieldCache(r);
+
+ assertArrayEquals(expectedGroupHeads, actualGroupHeads);
}
-
+
r.close();
dir.close();
}
@@ -542,14 +528,13 @@ public class AllGroupHeadsCollectorTest
}
private void addGroupField(Document doc, String groupField, String value, DocValuesType valueType) {
- doc.add(new TextField(groupField, value, Field.Store.NO));
Field valuesField = null;
switch(valueType) {
case BINARY:
- valuesField = new BinaryDocValuesField(groupField + "_dv", new BytesRef(value));
+ valuesField = new BinaryDocValuesField(groupField, new BytesRef(value));
break;
case SORTED:
- valuesField = new SortedDocValuesField(groupField + "_dv", new BytesRef(value));
+ valuesField = new SortedDocValuesField(groupField, new BytesRef(value));
break;
default:
fail("unhandled type");
Modified: lucene/dev/branches/lucene5675/lucene/grouping/src/test/org/apache/lucene/search/grouping/DistinctValuesCollectorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/grouping/src/test/org/apache/lucene/search/grouping/DistinctValuesCollectorTest.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/grouping/src/test/org/apache/lucene/search/grouping/DistinctValuesCollectorTest.java (original)
+++ lucene/dev/branches/lucene5675/lucene/grouping/src/test/org/apache/lucene/search/grouping/DistinctValuesCollectorTest.java Mon May 19 10:34:35 2014
@@ -17,16 +17,27 @@ package org.apache.lucene.search.groupin
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.StoredDocument;
import org.apache.lucene.index.Term;
@@ -41,92 +52,71 @@ import org.apache.lucene.search.grouping
import org.apache.lucene.search.grouping.term.TermFirstPassGroupingCollector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueStr;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
+@SuppressCodecs({"Lucene40", "Lucene41", "Lucene42"}) // we need missing support... i think?
public class DistinctValuesCollectorTest extends AbstractGroupingTestCase {
private final static NullComparator nullComparator = new NullComparator();
private final String groupField = "author";
- private final String dvGroupField = "author_dv";
private final String countField = "publisher";
- private final String dvCountField = "publisher_dv";
public void testSimple() throws Exception {
Random random = random();
- DocValuesType[] dvTypes = new DocValuesType[]{
- DocValuesType.NUMERIC,
- DocValuesType.BINARY,
- DocValuesType.SORTED,
- };
Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(
random,
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
- boolean canUseDV = true;
- DocValuesType dvType = canUseDV ? dvTypes[random.nextInt(dvTypes.length)] : null;
-
Document doc = new Document();
- addField(doc, groupField, "1", dvType);
- addField(doc, countField, "1", dvType);
+ addField(doc, groupField, "1");
+ addField(doc, countField, "1");
doc.add(new TextField("content", "random text", Field.Store.NO));
doc.add(new StringField("id", "1", Field.Store.NO));
w.addDocument(doc);
// 1
doc = new Document();
- addField(doc, groupField, "1", dvType);
- addField(doc, countField, "1", dvType);
+ addField(doc, groupField, "1");
+ addField(doc, countField, "1");
doc.add(new TextField("content", "some more random text blob", Field.Store.NO));
doc.add(new StringField("id", "2", Field.Store.NO));
w.addDocument(doc);
// 2
doc = new Document();
- addField(doc, groupField, "1", dvType);
- addField(doc, countField, "2", dvType);
+ addField(doc, groupField, "1");
+ addField(doc, countField, "2");
doc.add(new TextField("content", "some more random textual data", Field.Store.NO));
doc.add(new StringField("id", "3", Field.Store.NO));
w.addDocument(doc);
w.commit(); // To ensure a second segment
- // 3
+ // 3 -- no count field
doc = new Document();
- addField(doc, groupField, "2", dvType);
+ addField(doc, groupField, "2");
doc.add(new TextField("content", "some random text", Field.Store.NO));
doc.add(new StringField("id", "4", Field.Store.NO));
w.addDocument(doc);
// 4
doc = new Document();
- addField(doc, groupField, "3", dvType);
- addField(doc, countField, "1", dvType);
+ addField(doc, groupField, "3");
+ addField(doc, countField, "1");
doc.add(new TextField("content", "some more random text", Field.Store.NO));
doc.add(new StringField("id", "5", Field.Store.NO));
w.addDocument(doc);
// 5
doc = new Document();
- addField(doc, groupField, "3", dvType);
- addField(doc, countField, "1", dvType);
+ addField(doc, groupField, "3");
+ addField(doc, countField, "1");
doc.add(new TextField("content", "random blob", Field.Store.NO));
doc.add(new StringField("id", "6", Field.Store.NO));
w.addDocument(doc);
@@ -134,7 +124,7 @@ public class DistinctValuesCollectorTest
// 6 -- no author field
doc = new Document();
doc.add(new TextField("content", "random word stuck in alot of other text", Field.Store.YES));
- addField(doc, countField, "1", dvType);
+ addField(doc, countField, "1");
doc.add(new StringField("id", "6", Field.Store.NO));
w.addDocument(doc);
@@ -160,13 +150,13 @@ public class DistinctValuesCollectorTest
};
// === Search for content:random
- AbstractFirstPassGroupingCollector<Comparable<Object>> firstCollector = createRandomFirstPassCollector(dvType, new Sort(), groupField, 10);
+ AbstractFirstPassGroupingCollector<Comparable<Object>> firstCollector = createRandomFirstPassCollector(new Sort(), groupField, 10);
indexSearcher.search(new TermQuery(new Term("content", "random")), firstCollector);
AbstractDistinctValuesCollector<? extends AbstractDistinctValuesCollector.GroupCount<Comparable<Object>>> distinctValuesCollector
- = createDistinctCountCollector(firstCollector, groupField, countField, dvType);
+ = createDistinctCountCollector(firstCollector, groupField, countField);
indexSearcher.search(new TermQuery(new Term("content", "random")), distinctValuesCollector);
- List<? extends AbstractDistinctValuesCollector.GroupCount<Comparable<Object>>> gcs = distinctValuesCollector.getGroups();
+ List<? extends AbstractDistinctValuesCollector.GroupCount<Comparable<Object>>> gcs = distinctValuesCollector.getGroups();
Collections.sort(gcs, cmp);
assertEquals(4, gcs.size());
@@ -193,9 +183,9 @@ public class DistinctValuesCollectorTest
compare("1", countValues.get(0));
// === Search for content:some
- firstCollector = createRandomFirstPassCollector(dvType, new Sort(), groupField, 10);
+ firstCollector = createRandomFirstPassCollector(new Sort(), groupField, 10);
indexSearcher.search(new TermQuery(new Term("content", "some")), firstCollector);
- distinctValuesCollector = createDistinctCountCollector(firstCollector, groupField, countField, dvType);
+ distinctValuesCollector = createDistinctCountCollector(firstCollector, groupField, countField);
indexSearcher.search(new TermQuery(new Term("content", "some")), distinctValuesCollector);
gcs = distinctValuesCollector.getGroups();
@@ -220,9 +210,9 @@ public class DistinctValuesCollectorTest
compare("1", countValues.get(0));
// === Search for content:blob
- firstCollector = createRandomFirstPassCollector(dvType, new Sort(), groupField, 10);
+ firstCollector = createRandomFirstPassCollector(new Sort(), groupField, 10);
indexSearcher.search(new TermQuery(new Term("content", "blob")), firstCollector);
- distinctValuesCollector = createDistinctCountCollector(firstCollector, groupField, countField, dvType);
+ distinctValuesCollector = createDistinctCountCollector(firstCollector, groupField, countField);
indexSearcher.search(new TermQuery(new Term("content", "blob")), distinctValuesCollector);
gcs = distinctValuesCollector.getGroups();
@@ -251,18 +241,16 @@ public class DistinctValuesCollectorTest
IndexContext context = createIndexContext();
for (int searchIter = 0; searchIter < 100; searchIter++) {
final IndexSearcher searcher = newSearcher(context.indexReader);
- boolean useDv = context.dvType != null && random.nextBoolean();
- DocValuesType dvType = useDv ? context.dvType : null;
String term = context.contentStrings[random.nextInt(context.contentStrings.length)];
Sort groupSort = new Sort(new SortField("id", SortField.Type.STRING));
int topN = 1 + random.nextInt(10);
List<AbstractDistinctValuesCollector.GroupCount<Comparable<?>>> expectedResult = createExpectedResult(context, term, groupSort, topN);
- AbstractFirstPassGroupingCollector<Comparable<?>> firstCollector = createRandomFirstPassCollector(dvType, groupSort, groupField, topN);
+ AbstractFirstPassGroupingCollector<Comparable<?>> firstCollector = createRandomFirstPassCollector(groupSort, groupField, topN);
searcher.search(new TermQuery(new Term("content", term)), firstCollector);
AbstractDistinctValuesCollector<? extends AbstractDistinctValuesCollector.GroupCount<Comparable<?>>> distinctValuesCollector
- = createDistinctCountCollector(firstCollector, groupField, countField, dvType);
+ = createDistinctCountCollector(firstCollector, groupField, countField);
searcher.search(new TermQuery(new Term("content", term)), distinctValuesCollector);
@SuppressWarnings("unchecked")
List<AbstractDistinctValuesCollector.GroupCount<Comparable<?>>> actualResult = (List<AbstractDistinctValuesCollector.GroupCount<Comparable<?>>>) distinctValuesCollector.getGroups();
@@ -273,7 +261,6 @@ public class DistinctValuesCollectorTest
System.out.println("1st pass collector class name=" + firstCollector.getClass().getName());
System.out.println("2nd pass collector class name=" + distinctValuesCollector.getClass().getName());
System.out.println("Search term=" + term);
- System.out.println("DVType=" + dvType);
System.out.println("1st pass groups=" + firstCollector.getTopGroups(0, false));
System.out.println("Expected:");
printGroups(expectedResult);
@@ -363,33 +350,14 @@ public class DistinctValuesCollectorTest
}
}
- private void addField(Document doc, String field, String value, DocValuesType type) {
- doc.add(new StringField(field, value, Field.Store.YES));
- if (type == null) {
- return;
- }
- String dvField = field + "_dv";
-
- Field valuesField = null;
- switch (type) {
- case NUMERIC:
- valuesField = new NumericDocValuesField(dvField, Integer.parseInt(value));
- break;
- case BINARY:
- valuesField = new BinaryDocValuesField(dvField, new BytesRef(value));
- break;
- case SORTED:
- valuesField = new SortedDocValuesField(dvField, new BytesRef(value));
- break;
- }
- doc.add(valuesField);
+ private void addField(Document doc, String field, String value) {
+ doc.add(new SortedDocValuesField(field, new BytesRef(value)));
}
@SuppressWarnings({"unchecked","rawtypes"})
private <T extends Comparable> AbstractDistinctValuesCollector<AbstractDistinctValuesCollector.GroupCount<T>> createDistinctCountCollector(AbstractFirstPassGroupingCollector<T> firstPassGroupingCollector,
String groupField,
- String countField,
- DocValuesType dvType) {
+ String countField) {
Random random = random();
Collection<SearchGroup<T>> searchGroups = firstPassGroupingCollector.getTopGroups(0, false);
if (FunctionFirstPassGroupingCollector.class.isAssignableFrom(firstPassGroupingCollector.getClass())) {
@@ -400,20 +368,12 @@ public class DistinctValuesCollectorTest
}
@SuppressWarnings({"unchecked","rawtypes"})
- private <T> AbstractFirstPassGroupingCollector<T> createRandomFirstPassCollector(DocValuesType dvType, Sort groupSort, String groupField, int topNGroups) throws IOException {
+ private <T> AbstractFirstPassGroupingCollector<T> createRandomFirstPassCollector(Sort groupSort, String groupField, int topNGroups) throws IOException {
Random random = random();
- if (dvType != null) {
- if (random.nextBoolean()) {
- return (AbstractFirstPassGroupingCollector<T>) new FunctionFirstPassGroupingCollector(new BytesRefFieldSource(groupField), new HashMap<>(), groupSort, topNGroups);
- } else {
- return (AbstractFirstPassGroupingCollector<T>) new TermFirstPassGroupingCollector(groupField, groupSort, topNGroups);
- }
+ if (random.nextBoolean()) {
+ return (AbstractFirstPassGroupingCollector<T>) new FunctionFirstPassGroupingCollector(new BytesRefFieldSource(groupField), new HashMap<>(), groupSort, topNGroups);
} else {
- if (random.nextBoolean()) {
- return (AbstractFirstPassGroupingCollector<T>) new FunctionFirstPassGroupingCollector(new BytesRefFieldSource(groupField), new HashMap<>(), groupSort, topNGroups);
- } else {
- return (AbstractFirstPassGroupingCollector<T>) new TermFirstPassGroupingCollector(groupField, groupSort, topNGroups);
- }
+ return (AbstractFirstPassGroupingCollector<T>) new TermFirstPassGroupingCollector(groupField, groupSort, topNGroups);
}
}
@@ -444,10 +404,6 @@ public class DistinctValuesCollectorTest
private IndexContext createIndexContext() throws Exception {
Random random = random();
- DocValuesType[] dvTypes = new DocValuesType[]{
- DocValuesType.BINARY,
- DocValuesType.SORTED
- };
Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(
@@ -457,9 +413,6 @@ public class DistinctValuesCollectorTest
new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy())
);
- boolean canUseDV = true;
- DocValuesType dvType = canUseDV ? dvTypes[random.nextInt(dvTypes.length)] : null;
-
int numDocs = 86 + random.nextInt(1087) * RANDOM_MULTIPLIER;
String[] groupValues = new String[numDocs / 5];
String[] countValues = new String[numDocs / 10];
@@ -492,10 +445,10 @@ public class DistinctValuesCollectorTest
Document doc = new Document();
doc.add(new StringField("id", String.format(Locale.ROOT, "%09d", i), Field.Store.YES));
if (groupValue != null) {
- addField(doc, groupField, groupValue, dvType);
+ addField(doc, groupField, groupValue);
}
if (countValue != null) {
- addField(doc, countField, countValue, dvType);
+ addField(doc, countField, countValue);
}
doc.add(new TextField("content", content, Field.Store.YES));
w.addDocument(doc);
@@ -510,22 +463,20 @@ public class DistinctValuesCollectorTest
}
w.shutdown();
- return new IndexContext(dir, reader, dvType, searchTermToGroupCounts, contentStrings.toArray(new String[contentStrings.size()]));
+ return new IndexContext(dir, reader, searchTermToGroupCounts, contentStrings.toArray(new String[contentStrings.size()]));
}
private static class IndexContext {
final Directory directory;
final DirectoryReader indexReader;
- final DocValuesType dvType;
final Map<String, Map<String, Set<String>>> searchTermToGroupCounts;
final String[] contentStrings;
- IndexContext(Directory directory, DirectoryReader indexReader, DocValuesType dvType,
+ IndexContext(Directory directory, DirectoryReader indexReader,
Map<String, Map<String, Set<String>>> searchTermToGroupCounts, String[] contentStrings) {
this.directory = directory;
this.indexReader = indexReader;
- this.dvType = dvType;
this.searchTermToGroupCounts = searchTermToGroupCounts;
this.contentStrings = contentStrings;
}
Modified: lucene/dev/branches/lucene5675/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupFacetCollectorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupFacetCollectorTest.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupFacetCollectorTest.java (original)
+++ lucene/dev/branches/lucene5675/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupFacetCollectorTest.java Mon May 19 10:34:35 2014
@@ -17,11 +17,26 @@ package org.apache.lucene.search.groupin
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeSet;
+
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.NoMergePolicy;
@@ -33,22 +48,12 @@ import org.apache.lucene.search.TermQuer
import org.apache.lucene.search.grouping.term.TermGroupFacetCollector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.TestUtil;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.Random;
-import java.util.Set;
-import java.util.TreeSet;
+// Need SSDV, ord=-1 for missing:
+@SuppressCodecs({"Lucene40", "Lucene41", "Lucene42"})
public class GroupFacetCollectorTest extends AbstractGroupingTestCase {
public void testSimple() throws Exception {
@@ -62,7 +67,7 @@ public class GroupFacetCollectorTest ext
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
- boolean useDv = random().nextBoolean();
+ boolean useDv = true;
// 0
Document doc = new Document();
@@ -287,7 +292,7 @@ public class GroupFacetCollectorTest ext
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE));
- boolean useDv = false;
+ boolean useDv = true;
// Cannot assert this since we use NoMergePolicy:
w.setDoRandomForceMergeAssert(false);
@@ -300,7 +305,7 @@ public class GroupFacetCollectorTest ext
// 1
doc = new Document();
addField(doc, groupField, "a", useDv);
- doc.add(new StringField("airport", "ams", Field.Store.NO));
+ doc.add(new SortedSetDocValuesField("airport", new BytesRef("ams")));
w.addDocument(doc);
w.commit();
@@ -309,32 +314,31 @@ public class GroupFacetCollectorTest ext
// 2
doc = new Document();
addField(doc, groupField, "a", useDv);
- doc.add(new StringField("airport", "ams", Field.Store.NO));
+ doc.add(new SortedSetDocValuesField("airport", new BytesRef("ams")));
w.addDocument(doc);
// 3
doc = new Document();
addField(doc, groupField, "a", useDv);
- doc.add(new StringField("airport", "dus", Field.Store.NO));
-
+ doc.add(new SortedSetDocValuesField("airport", new BytesRef("dus")));
w.addDocument(doc);
// 4
doc = new Document();
addField(doc, groupField, "b", useDv);
- doc.add(new StringField("airport", "ams", Field.Store.NO));
+ doc.add(new SortedSetDocValuesField("airport", new BytesRef("ams")));
w.addDocument(doc);
// 5
doc = new Document();
addField(doc, groupField, "b", useDv);
- doc.add(new StringField("airport", "ams", Field.Store.NO));
+ doc.add(new SortedSetDocValuesField("airport", new BytesRef("ams")));
w.addDocument(doc);
// 6
doc = new Document();
addField(doc, groupField, "b", useDv);
- doc.add(new StringField("airport", "ams", Field.Store.NO));
+ doc.add(new SortedSetDocValuesField("airport", new BytesRef("ams")));
w.addDocument(doc);
w.commit();
@@ -346,7 +350,7 @@ public class GroupFacetCollectorTest ext
w.shutdown();
IndexSearcher indexSearcher = newSearcher(DirectoryReader.open(dir));
- AbstractGroupFacetCollector groupedAirportFacetCollector = createRandomCollector(groupField, "airport", null, true);
+ AbstractGroupFacetCollector groupedAirportFacetCollector = createRandomCollector(groupField + "_dv", "airport", null, true);
indexSearcher.search(new MatchAllDocsQuery(), groupedAirportFacetCollector);
TermGroupFacetCollector.GroupedFacetResult airportResult = groupedAirportFacetCollector.mergeSegmentResults(10, 0, false);
assertEquals(3, airportResult.getTotalCount());
@@ -364,10 +368,8 @@ public class GroupFacetCollectorTest ext
}
private void addField(Document doc, String field, String value, boolean canUseIDV) {
- doc.add(new StringField(field, value, Field.Store.NO));
- if (canUseIDV) {
- doc.add(new SortedDocValuesField(field + "_dv", new BytesRef(value)));
- }
+ assert canUseIDV;
+ doc.add(new SortedDocValuesField(field + "_dv", new BytesRef(value)));
}
public void testRandom() throws Exception {
@@ -386,7 +388,6 @@ public class GroupFacetCollectorTest ext
if (VERBOSE) {
System.out.println("TEST: searchIter=" + searchIter);
}
- boolean useDv = !multipleFacetsPerDocument && context.useDV && random.nextBoolean();
String searchTerm = context.contentStrings[random.nextInt(context.contentStrings.length)];
int limit = random.nextInt(context.facetValues.size());
int offset = random.nextInt(context.facetValues.size() - limit);
@@ -409,7 +410,7 @@ public class GroupFacetCollectorTest ext
}
GroupedFacetResult expectedFacetResult = createExpectedFacetResult(searchTerm, context, offset, limit, minCount, orderByCount, facetPrefix);
- AbstractGroupFacetCollector groupFacetCollector = createRandomCollector(useDv ? "group_dv" : "group", useDv ? "facet_dv" : "facet", facetPrefix, multipleFacetsPerDocument);
+ AbstractGroupFacetCollector groupFacetCollector = createRandomCollector("group", "facet", facetPrefix, multipleFacetsPerDocument);
searcher.search(new TermQuery(new Term("content", searchTerm)), groupFacetCollector);
TermGroupFacetCollector.GroupedFacetResult actualFacetResult = groupFacetCollector.mergeSegmentResults(size, minCount, orderByCount);
@@ -417,7 +418,6 @@ public class GroupFacetCollectorTest ext
List<TermGroupFacetCollector.FacetEntry> actualFacetEntries = actualFacetResult.getFacetEntries(offset, limit);
if (VERBOSE) {
- System.out.println("Use DV: " + useDv);
System.out.println("Collector: " + groupFacetCollector.getClass().getSimpleName());
System.out.println("Num group: " + context.numGroups);
System.out.println("Num doc: " + context.numDocs);
@@ -514,35 +514,29 @@ public class GroupFacetCollectorTest ext
new MockAnalyzer(random)
)
);
- boolean canUseDV = true;
- boolean useDv = canUseDV && !multipleFacetValuesPerDocument && random.nextBoolean();
-
Document doc = new Document();
Document docNoGroup = new Document();
Document docNoFacet = new Document();
Document docNoGroupNoFacet = new Document();
Field group = newStringField("group", "", Field.Store.NO);
- Field groupDc = new SortedDocValuesField("group_dv", new BytesRef());
- if (useDv) {
- doc.add(groupDc);
- docNoFacet.add(groupDc);
- }
+ Field groupDc = new SortedDocValuesField("group", new BytesRef());
+ doc.add(groupDc);
+ docNoFacet.add(groupDc);
doc.add(group);
docNoFacet.add(group);
Field[] facetFields;
- if (useDv) {
- assert !multipleFacetValuesPerDocument;
+ if (multipleFacetValuesPerDocument == false) {
facetFields = new Field[2];
facetFields[0] = newStringField("facet", "", Field.Store.NO);
doc.add(facetFields[0]);
docNoGroup.add(facetFields[0]);
- facetFields[1] = new SortedDocValuesField("facet_dv", new BytesRef());
+ facetFields[1] = new SortedDocValuesField("facet", new BytesRef());
doc.add(facetFields[1]);
docNoGroup.add(facetFields[1]);
} else {
facetFields = multipleFacetValuesPerDocument ? new Field[2 + random.nextInt(6)] : new Field[1];
for (int i = 0; i < facetFields.length; i++) {
- facetFields[i] = newStringField("facet", "", Field.Store.NO);
+ facetFields[i] = new SortedSetDocValuesField("facet", new BytesRef());
doc.add(facetFields[i]);
docNoGroup.add(facetFields[i]);
}
@@ -576,11 +570,7 @@ public class GroupFacetCollectorTest ext
if (random.nextInt(24) == 17) {
// So we test the "doc doesn't have the group'd
// field" case:
- if (useDv) {
- groupValue = "";
- } else {
- groupValue = null;
- }
+ groupValue = "";
} else {
groupValue = groups.get(random.nextInt(groups.size()));
}
@@ -592,8 +582,22 @@ public class GroupFacetCollectorTest ext
Map<String, Set<String>> facetToGroups = searchTermToFacetToGroups.get(contentStr);
List<String> facetVals = new ArrayList<>();
- if (useDv || random.nextInt(24) != 18) {
- if (useDv) {
+ if (multipleFacetValuesPerDocument == false) {
+ String facetValue = facetValues.get(random.nextInt(facetValues.size()));
+ uniqueFacetValues.add(facetValue);
+ if (!facetToGroups.containsKey(facetValue)) {
+ facetToGroups.put(facetValue, new HashSet<String>());
+ }
+ Set<String> groupsInFacet = facetToGroups.get(facetValue);
+ groupsInFacet.add(groupValue);
+ if (groupsInFacet.size() > facetWithMostGroups) {
+ facetWithMostGroups = groupsInFacet.size();
+ }
+ facetFields[0].setStringValue(facetValue);
+ facetFields[1].setBytesValue(new BytesRef(facetValue));
+ facetVals.add(facetValue);
+ } else {
+ for (Field facetField : facetFields) {
String facetValue = facetValues.get(random.nextInt(facetValues.size()));
uniqueFacetValues.add(facetValue);
if (!facetToGroups.containsKey(facetValue)) {
@@ -604,34 +608,8 @@ public class GroupFacetCollectorTest ext
if (groupsInFacet.size() > facetWithMostGroups) {
facetWithMostGroups = groupsInFacet.size();
}
- facetFields[0].setStringValue(facetValue);
- facetFields[1].setBytesValue(new BytesRef(facetValue));
+ facetField.setBytesValue(new BytesRef(facetValue));
facetVals.add(facetValue);
- } else {
- for (Field facetField : facetFields) {
- String facetValue = facetValues.get(random.nextInt(facetValues.size()));
- uniqueFacetValues.add(facetValue);
- if (!facetToGroups.containsKey(facetValue)) {
- facetToGroups.put(facetValue, new HashSet<String>());
- }
- Set<String> groupsInFacet = facetToGroups.get(facetValue);
- groupsInFacet.add(groupValue);
- if (groupsInFacet.size() > facetWithMostGroups) {
- facetWithMostGroups = groupsInFacet.size();
- }
- facetField.setStringValue(facetValue);
- facetVals.add(facetValue);
- }
- }
- } else {
- uniqueFacetValues.add(null);
- if (!facetToGroups.containsKey(null)) {
- facetToGroups.put(null, new HashSet<String>());
- }
- Set<String> groupsInFacet = facetToGroups.get(null);
- groupsInFacet.add(groupValue);
- if (groupsInFacet.size() > facetWithMostGroups) {
- facetWithMostGroups = groupsInFacet.size();
}
}
@@ -640,11 +618,10 @@ public class GroupFacetCollectorTest ext
}
if (groupValue != null) {
- if (useDv) {
- groupDc.setBytesValue(new BytesRef(groupValue));
- }
+ groupDc.setBytesValue(new BytesRef(groupValue));
group.setStringValue(groupValue);
- } else if (useDv) {
+ } else {
+ // TODO: not true
// DV cannot have missing values:
groupDc.setBytesValue(new BytesRef());
}
@@ -663,7 +640,7 @@ public class GroupFacetCollectorTest ext
DirectoryReader reader = writer.getReader();
writer.shutdown();
- return new IndexContext(searchTermToFacetToGroups, reader, numDocs, dir, facetWithMostGroups, numGroups, contentBrs, uniqueFacetValues, useDv);
+ return new IndexContext(searchTermToFacetToGroups, reader, numDocs, dir, facetWithMostGroups, numGroups, contentBrs, uniqueFacetValues);
}
private GroupedFacetResult createExpectedFacetResult(String searchTerm, IndexContext context, int offset, int limit, int minCount, final boolean orderByCount, String facetPrefix) {
@@ -738,8 +715,6 @@ public class GroupFacetCollectorTest ext
private AbstractGroupFacetCollector createRandomCollector(String groupField, String facetField, String facetPrefix, boolean multipleFacetsPerDocument) {
BytesRef facetPrefixBR = facetPrefix == null ? null : new BytesRef(facetPrefix);
- // DocValues cannot be multi-valued:
- assert !multipleFacetsPerDocument || !groupField.endsWith("_dv");
return TermGroupFacetCollector.createTermGroupFacetCollector(groupField, facetField, multipleFacetsPerDocument, facetPrefixBR, random().nextInt(1024));
}
@@ -764,10 +739,9 @@ public class GroupFacetCollectorTest ext
final int facetWithMostGroups;
final int numGroups;
final String[] contentStrings;
- final boolean useDV;
public IndexContext(Map<String, Map<String, Set<String>>> searchTermToFacetGroups, DirectoryReader r,
- int numDocs, Directory dir, int facetWithMostGroups, int numGroups, String[] contentStrings, NavigableSet<String> facetValues, boolean useDV) {
+ int numDocs, Directory dir, int facetWithMostGroups, int numGroups, String[] contentStrings, NavigableSet<String> facetValues) {
this.searchTermToFacetGroups = searchTermToFacetGroups;
this.indexReader = r;
this.numDocs = numDocs;
@@ -776,7 +750,6 @@ public class GroupFacetCollectorTest ext
this.numGroups = numGroups;
this.contentStrings = contentStrings;
this.facetValues = facetValues;
- this.useDV = useDV;
}
}