You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/05/19 12:34:40 UTC
svn commit: r1595824 [6/9] - in /lucene/dev/branches/lucene5675: ./ lucene/
lucene/analysis/ lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/collation/
lucene/analysis/common/src/test/org/apache/lucene/collation/ lucene/analys...
Modified: lucene/dev/branches/lucene5675/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java (original)
+++ lucene/dev/branches/lucene5675/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java Mon May 19 10:34:35 2014
@@ -21,6 +21,8 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.document.*;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.RandomIndexWriter;
@@ -39,6 +41,7 @@ import org.apache.lucene.search.grouping
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueStr;
@@ -52,6 +55,7 @@ import java.util.*;
// - test ties
// - test compound sort
+@SuppressCodecs({"Lucene40", "Lucene41", "Lucene42"}) // we need missing support... i think?
public class TestGrouping extends LuceneTestCase {
public void testBasic() throws Exception {
@@ -120,10 +124,6 @@ public class TestGrouping extends Lucene
final Sort groupSort = Sort.RELEVANCE;
- if (random().nextBoolean()) {
- groupField += "_dv";
- }
-
final AbstractFirstPassGroupingCollector<?> c1 = createRandomFirstPassCollector(groupField, groupSort, 10);
indexSearcher.search(new TermQuery(new Term("content", "random")), c1);
@@ -172,8 +172,7 @@ public class TestGrouping extends Lucene
}
private void addGroupField(Document doc, String groupField, String value) {
- doc.add(new TextField(groupField, value, Field.Store.YES));
- doc.add(new SortedDocValuesField(groupField + "_dv", new BytesRef(value)));
+ doc.add(new SortedDocValuesField(groupField, new BytesRef(value)));
}
private AbstractFirstPassGroupingCollector<?> createRandomFirstPassCollector(String groupField, Sort groupSort, int topDocs) throws IOException {
@@ -569,10 +568,14 @@ public class TestGrouping extends Lucene
docs.add(doc);
if (groupValue.group != null) {
doc.add(newStringField("group", groupValue.group.utf8ToString(), Field.Store.YES));
+ doc.add(new SortedDocValuesField("group", BytesRef.deepCopyOf(groupValue.group)));
}
doc.add(newStringField("sort1", groupValue.sort1.utf8ToString(), Field.Store.NO));
+ doc.add(new SortedDocValuesField("sort1", BytesRef.deepCopyOf(groupValue.sort1)));
doc.add(newStringField("sort2", groupValue.sort2.utf8ToString(), Field.Store.NO));
+ doc.add(new SortedDocValuesField("sort2", BytesRef.deepCopyOf(groupValue.sort2)));
doc.add(new IntField("id", groupValue.id, Field.Store.NO));
+ doc.add(new NumericDocValuesField("id", groupValue.id));
doc.add(newTextField("content", groupValue.content, Field.Store.NO));
//System.out.println("TEST: doc content=" + groupValue.content + " group=" + (groupValue.group == null ? "null" : groupValue.group.utf8ToString()) + " sort1=" + groupValue.sort1.utf8ToString() + " id=" + groupValue.id);
}
@@ -642,7 +645,7 @@ public class TestGrouping extends Lucene
// For that reason we don't generate empty string
// groups.
randomValue = TestUtil.randomRealisticUnicodeString(random());
- //randomValue = _TestUtil.randomSimpleString(random());
+ //randomValue = TestUtil.randomSimpleString(random());
} while ("".equals(randomValue));
groups.add(new BytesRef(randomValue));
@@ -670,22 +673,18 @@ public class TestGrouping extends Lucene
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random())));
- boolean canUseIDV = true;
-
Document doc = new Document();
Document docNoGroup = new Document();
- Field idvGroupField = new SortedDocValuesField("group_dv", new BytesRef());
- if (canUseIDV) {
- doc.add(idvGroupField);
- docNoGroup.add(idvGroupField);
- }
+ Field idvGroupField = new SortedDocValuesField("group", new BytesRef());
+ doc.add(idvGroupField);
+ docNoGroup.add(idvGroupField);
Field group = newStringField("group", "", Field.Store.NO);
doc.add(group);
- Field sort1 = newStringField("sort1", "", Field.Store.NO);
+ Field sort1 = new SortedDocValuesField("sort1", new BytesRef());
doc.add(sort1);
docNoGroup.add(sort1);
- Field sort2 = newStringField("sort2", "", Field.Store.NO);
+ Field sort2 = new SortedDocValuesField("sort2", new BytesRef());
doc.add(sort2);
docNoGroup.add(sort2);
Field content = newTextField("content", "", Field.Store.NO);
@@ -693,7 +692,10 @@ public class TestGrouping extends Lucene
docNoGroup.add(content);
IntField id = new IntField("id", 0, Field.Store.NO);
doc.add(id);
+ NumericDocValuesField idDV = new NumericDocValuesField("id", 0);
+ doc.add(idDV);
docNoGroup.add(id);
+ docNoGroup.add(idDV);
final GroupDoc[] groupDocs = new GroupDoc[numDocs];
for(int i=0;i<numDocs;i++) {
final BytesRef groupValue;
@@ -716,19 +718,19 @@ public class TestGrouping extends Lucene
groupDocs[i] = groupDoc;
if (groupDoc.group != null) {
group.setStringValue(groupDoc.group.utf8ToString());
- if (canUseIDV) {
- idvGroupField.setBytesValue(BytesRef.deepCopyOf(groupDoc.group));
- }
- } else if (canUseIDV) {
+ idvGroupField.setBytesValue(BytesRef.deepCopyOf(groupDoc.group));
+ } else {
+ // TODO: not true
// Must explicitly set empty string, else eg if
// the segment has all docs missing the field then
// we get null back instead of empty BytesRef:
idvGroupField.setBytesValue(new BytesRef());
}
- sort1.setStringValue(groupDoc.sort1.utf8ToString());
- sort2.setStringValue(groupDoc.sort2.utf8ToString());
+ sort1.setBytesValue(BytesRef.deepCopyOf(groupDoc.sort1));
+ sort2.setBytesValue(BytesRef.deepCopyOf(groupDoc.sort2));
content.setStringValue(groupDoc.content);
id.setIntValue(groupDoc.id);
+ idDV.setLongValue(groupDoc.id);
if (groupDoc.group == null) {
w.addDocument(docNoGroup);
} else {
@@ -742,405 +744,387 @@ public class TestGrouping extends Lucene
final DirectoryReader r = w.getReader();
w.shutdown();
- // NOTE: intentional but temporary field cache insanity!
- final FieldCache.Ints docIDToID = FieldCache.DEFAULT.getInts(SlowCompositeReaderWrapper.wrap(r), "id", false);
+ final NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
DirectoryReader rBlocks = null;
Directory dirBlocks = null;
- try {
- final IndexSearcher s = newSearcher(r);
+ final IndexSearcher s = newSearcher(r);
+ if (VERBOSE) {
+ System.out.println("\nTEST: searcher=" + s);
+ }
+
+ final ShardState shards = new ShardState(s);
+
+ for(int contentID=0;contentID<3;contentID++) {
+ final ScoreDoc[] hits = s.search(new TermQuery(new Term("content", "real"+contentID)), numDocs).scoreDocs;
+ for(ScoreDoc hit : hits) {
+ final GroupDoc gd = groupDocs[(int) docIDToID.get(hit.doc)];
+ assertTrue(gd.score == 0.0);
+ gd.score = hit.score;
+ assertEquals(gd.id, docIDToID.get(hit.doc));
+ }
+ }
+
+ for(GroupDoc gd : groupDocs) {
+ assertTrue(gd.score != 0.0);
+ }
+
+ // Build 2nd index, where docs are added in blocks by
+ // group, so we can use single pass collector
+ dirBlocks = newDirectory();
+ rBlocks = getDocBlockReader(dirBlocks, groupDocs);
+ final Filter lastDocInBlock = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("groupend", "x"))));
+ final NumericDocValues docIDToIDBlocks = MultiDocValues.getNumericValues(rBlocks, "id");
+ assertNotNull(docIDToIDBlocks);
+
+ final IndexSearcher sBlocks = newSearcher(rBlocks);
+ final ShardState shardsBlocks = new ShardState(sBlocks);
+
+ // ReaderBlocks only increases maxDoc() vs reader, which
+ // means a monotonic shift in scores, so we can
+ // reliably remap them w/ Map:
+ final Map<String,Map<Float,Float>> scoreMap = new HashMap<>();
+
+ // Tricky: must separately set .score2, because the doc
+ // block index was created with possible deletions!
+ //System.out.println("fixup score2");
+ for(int contentID=0;contentID<3;contentID++) {
+ //System.out.println(" term=real" + contentID);
+ final Map<Float,Float> termScoreMap = new HashMap<>();
+ scoreMap.put("real"+contentID, termScoreMap);
+ //System.out.println("term=real" + contentID + " dfold=" + s.docFreq(new Term("content", "real"+contentID)) +
+ //" dfnew=" + sBlocks.docFreq(new Term("content", "real"+contentID)));
+ final ScoreDoc[] hits = sBlocks.search(new TermQuery(new Term("content", "real"+contentID)), numDocs).scoreDocs;
+ for(ScoreDoc hit : hits) {
+ final GroupDoc gd = groupDocsByID[(int) docIDToIDBlocks.get(hit.doc)];
+ assertTrue(gd.score2 == 0.0);
+ gd.score2 = hit.score;
+ assertEquals(gd.id, docIDToIDBlocks.get(hit.doc));
+ //System.out.println(" score=" + gd.score + " score2=" + hit.score + " id=" + docIDToIDBlocks.get(hit.doc));
+ termScoreMap.put(gd.score, gd.score2);
+ }
+ }
+
+ for(int searchIter=0;searchIter<100;searchIter++) {
+
if (VERBOSE) {
- System.out.println("\nTEST: searcher=" + s);
+ System.out.println("\nTEST: searchIter=" + searchIter);
}
-
- if (SlowCompositeReaderWrapper.class.isAssignableFrom(s.getIndexReader().getClass())) {
- canUseIDV = false;
- } else {
- canUseIDV = true;
- }
- final ShardState shards = new ShardState(s);
-
- for(int contentID=0;contentID<3;contentID++) {
- final ScoreDoc[] hits = s.search(new TermQuery(new Term("content", "real"+contentID)), numDocs).scoreDocs;
- for(ScoreDoc hit : hits) {
- final GroupDoc gd = groupDocs[docIDToID.get(hit.doc)];
- assertTrue(gd.score == 0.0);
- gd.score = hit.score;
- assertEquals(gd.id, docIDToID.get(hit.doc));
+
+ final String searchTerm = "real" + random().nextInt(3);
+ final boolean fillFields = random().nextBoolean();
+ boolean getScores = random().nextBoolean();
+ final boolean getMaxScores = random().nextBoolean();
+ final Sort groupSort = getRandomSort();
+ //final Sort groupSort = new Sort(new SortField[] {new SortField("sort1", SortField.STRING), new SortField("id", SortField.INT)});
+ // TODO: also test null (= sort by relevance)
+ final Sort docSort = getRandomSort();
+
+ for(SortField sf : docSort.getSort()) {
+ if (sf.getType() == SortField.Type.SCORE) {
+ getScores = true;
+ break;
}
}
-
- for(GroupDoc gd : groupDocs) {
- assertTrue(gd.score != 0.0);
- }
-
- // Build 2nd index, where docs are added in blocks by
- // group, so we can use single pass collector
- dirBlocks = newDirectory();
- rBlocks = getDocBlockReader(dirBlocks, groupDocs);
- final Filter lastDocInBlock = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("groupend", "x"))));
- final FieldCache.Ints docIDToIDBlocks = FieldCache.DEFAULT.getInts(SlowCompositeReaderWrapper.wrap(rBlocks), "id", false);
-
- final IndexSearcher sBlocks = newSearcher(rBlocks);
- final ShardState shardsBlocks = new ShardState(sBlocks);
-
- // ReaderBlocks only increases maxDoc() vs reader, which
- // means a monotonic shift in scores, so we can
- // reliably remap them w/ Map:
- final Map<String,Map<Float,Float>> scoreMap = new HashMap<>();
-
- // Tricky: must separately set .score2, because the doc
- // block index was created with possible deletions!
- //System.out.println("fixup score2");
- for(int contentID=0;contentID<3;contentID++) {
- //System.out.println(" term=real" + contentID);
- final Map<Float,Float> termScoreMap = new HashMap<>();
- scoreMap.put("real"+contentID, termScoreMap);
- //System.out.println("term=real" + contentID + " dfold=" + s.docFreq(new Term("content", "real"+contentID)) +
- //" dfnew=" + sBlocks.docFreq(new Term("content", "real"+contentID)));
- final ScoreDoc[] hits = sBlocks.search(new TermQuery(new Term("content", "real"+contentID)), numDocs).scoreDocs;
- for(ScoreDoc hit : hits) {
- final GroupDoc gd = groupDocsByID[docIDToIDBlocks.get(hit.doc)];
- assertTrue(gd.score2 == 0.0);
- gd.score2 = hit.score;
- assertEquals(gd.id, docIDToIDBlocks.get(hit.doc));
- //System.out.println(" score=" + gd.score + " score2=" + hit.score + " id=" + docIDToIDBlocks.get(hit.doc));
- termScoreMap.put(gd.score, gd.score2);
+
+ for(SortField sf : groupSort.getSort()) {
+ if (sf.getType() == SortField.Type.SCORE) {
+ getScores = true;
+ break;
}
}
-
- for(int searchIter=0;searchIter<100;searchIter++) {
-
+
+ final int topNGroups = TestUtil.nextInt(random(), 1, 30);
+ //final int topNGroups = 10;
+ final int docsPerGroup = TestUtil.nextInt(random(), 1, 50);
+
+ final int groupOffset = TestUtil.nextInt(random(), 0, (topNGroups - 1) / 2);
+ //final int groupOffset = 0;
+
+ final int docOffset = TestUtil.nextInt(random(), 0, docsPerGroup - 1);
+ //final int docOffset = 0;
+
+ final boolean doCache = random().nextBoolean();
+ final boolean doAllGroups = random().nextBoolean();
+ if (VERBOSE) {
+ System.out.println("TEST: groupSort=" + groupSort + " docSort=" + docSort + " searchTerm=" + searchTerm + " dF=" + r.docFreq(new Term("content", searchTerm)) +" dFBlock=" + rBlocks.docFreq(new Term("content", searchTerm)) + " topNGroups=" + topNGroups + " groupOffset=" + groupOffset + " docOffset=" + docOffset + " doCache=" + doCache + " docsPerGroup=" + docsPerGroup + " doAllGroups=" + doAllGroups + " getScores=" + getScores + " getMaxScores=" + getMaxScores);
+ }
+
+ String groupField = "group";
+ if (VERBOSE) {
+ System.out.println(" groupField=" + groupField);
+ }
+ final AbstractFirstPassGroupingCollector<?> c1 = createRandomFirstPassCollector(groupField, groupSort, groupOffset+topNGroups);
+ final CachingCollector cCache;
+ final Collector c;
+
+ final AbstractAllGroupsCollector<?> allGroupsCollector;
+ if (doAllGroups) {
+ allGroupsCollector = createAllGroupsCollector(c1, groupField);
+ } else {
+ allGroupsCollector = null;
+ }
+
+ final boolean useWrappingCollector = random().nextBoolean();
+
+ if (doCache) {
+ final double maxCacheMB = random().nextDouble();
if (VERBOSE) {
- System.out.println("\nTEST: searchIter=" + searchIter);
+ System.out.println("TEST: maxCacheMB=" + maxCacheMB);
}
-
- final String searchTerm = "real" + random().nextInt(3);
- final boolean fillFields = random().nextBoolean();
- boolean getScores = random().nextBoolean();
- final boolean getMaxScores = random().nextBoolean();
- final Sort groupSort = getRandomSort();
- //final Sort groupSort = new Sort(new SortField[] {new SortField("sort1", SortField.STRING), new SortField("id", SortField.INT)});
- // TODO: also test null (= sort by relevance)
- final Sort docSort = getRandomSort();
-
- for(SortField sf : docSort.getSort()) {
- if (sf.getType() == SortField.Type.SCORE) {
- getScores = true;
- break;
- }
- }
-
- for(SortField sf : groupSort.getSort()) {
- if (sf.getType() == SortField.Type.SCORE) {
- getScores = true;
- break;
+
+ if (useWrappingCollector) {
+ if (doAllGroups) {
+ cCache = CachingCollector.create(c1, true, maxCacheMB);
+ c = MultiCollector.wrap(cCache, allGroupsCollector);
+ } else {
+ c = cCache = CachingCollector.create(c1, true, maxCacheMB);
}
+ } else {
+ // Collect only into cache, then replay multiple times:
+ c = cCache = CachingCollector.create(false, true, maxCacheMB);
}
-
- final int topNGroups = TestUtil.nextInt(random(), 1, 30);
- //final int topNGroups = 10;
- final int docsPerGroup = TestUtil.nextInt(random(), 1, 50);
-
- final int groupOffset = TestUtil.nextInt(random(), 0, (topNGroups - 1) / 2);
- //final int groupOffset = 0;
-
- final int docOffset = TestUtil.nextInt(random(), 0, docsPerGroup - 1);
- //final int docOffset = 0;
-
- final boolean doCache = random().nextBoolean();
- final boolean doAllGroups = random().nextBoolean();
- if (VERBOSE) {
- System.out.println("TEST: groupSort=" + groupSort + " docSort=" + docSort + " searchTerm=" + searchTerm + " dF=" + r.docFreq(new Term("content", searchTerm)) +" dFBlock=" + rBlocks.docFreq(new Term("content", searchTerm)) + " topNGroups=" + topNGroups + " groupOffset=" + groupOffset + " docOffset=" + docOffset + " doCache=" + doCache + " docsPerGroup=" + docsPerGroup + " doAllGroups=" + doAllGroups + " getScores=" + getScores + " getMaxScores=" + getMaxScores);
- }
-
- String groupField = "group";
- if (canUseIDV && random().nextBoolean()) {
- groupField += "_dv";
- }
- if (VERBOSE) {
- System.out.println(" groupField=" + groupField);
- }
- final AbstractFirstPassGroupingCollector<?> c1 = createRandomFirstPassCollector(groupField, groupSort, groupOffset+topNGroups);
- final CachingCollector cCache;
- final Collector c;
-
- final AbstractAllGroupsCollector<?> allGroupsCollector;
+ } else {
+ cCache = null;
if (doAllGroups) {
- allGroupsCollector = createAllGroupsCollector(c1, groupField);
+ c = MultiCollector.wrap(c1, allGroupsCollector);
} else {
- allGroupsCollector = null;
+ c = c1;
}
-
- final boolean useWrappingCollector = random().nextBoolean();
-
- if (doCache) {
- final double maxCacheMB = random().nextDouble();
- if (VERBOSE) {
- System.out.println("TEST: maxCacheMB=" + maxCacheMB);
- }
-
- if (useWrappingCollector) {
- if (doAllGroups) {
- cCache = CachingCollector.create(c1, true, maxCacheMB);
- c = MultiCollector.wrap(cCache, allGroupsCollector);
- } else {
- c = cCache = CachingCollector.create(c1, true, maxCacheMB);
- }
- } else {
- // Collect only into cache, then replay multiple times:
- c = cCache = CachingCollector.create(false, true, maxCacheMB);
+ }
+
+ // Search top reader:
+ final Query query = new TermQuery(new Term("content", searchTerm));
+
+ s.search(query, c);
+
+ if (doCache && !useWrappingCollector) {
+ if (cCache.isCached()) {
+ // Replay for first-pass grouping
+ cCache.replay(c1);
+ if (doAllGroups) {
+ // Replay for all groups:
+ cCache.replay(allGroupsCollector);
}
} else {
- cCache = null;
+ // Replay by re-running search:
+ s.search(query, c1);
if (doAllGroups) {
- c = MultiCollector.wrap(c1, allGroupsCollector);
- } else {
- c = c1;
+ s.search(query, allGroupsCollector);
}
}
-
- // Search top reader:
- final Query query = new TermQuery(new Term("content", searchTerm));
-
- s.search(query, c);
-
- if (doCache && !useWrappingCollector) {
- if (cCache.isCached()) {
- // Replay for first-pass grouping
- cCache.replay(c1);
- if (doAllGroups) {
- // Replay for all groups:
- cCache.replay(allGroupsCollector);
- }
- } else {
- // Replay by re-running search:
- s.search(query, c1);
- if (doAllGroups) {
- s.search(query, allGroupsCollector);
- }
+ }
+
+ // Get 1st pass top groups
+ final Collection<SearchGroup<BytesRef>> topGroups = getSearchGroups(c1, groupOffset, fillFields);
+ final TopGroups<BytesRef> groupsResult;
+ if (VERBOSE) {
+ System.out.println("TEST: first pass topGroups");
+ if (topGroups == null) {
+ System.out.println(" null");
+ } else {
+ for (SearchGroup<BytesRef> searchGroup : topGroups) {
+ System.out.println(" " + (searchGroup.groupValue == null ? "null" : searchGroup.groupValue) + ": " + Arrays.deepToString(searchGroup.sortValues));
}
}
-
- // Get 1st pass top groups
- final Collection<SearchGroup<BytesRef>> topGroups = getSearchGroups(c1, groupOffset, fillFields);
- final TopGroups<BytesRef> groupsResult;
+ }
+
+ // Get 1st pass top groups using shards
+
+ final TopGroups<BytesRef> topGroupsShards = searchShards(s, shards.subSearchers, query, groupSort, docSort,
+ groupOffset, topNGroups, docOffset, docsPerGroup, getScores, getMaxScores, true, false);
+ final AbstractSecondPassGroupingCollector<?> c2;
+ if (topGroups != null) {
+
if (VERBOSE) {
- System.out.println("TEST: first pass topGroups");
- if (topGroups == null) {
- System.out.println(" null");
- } else {
- for (SearchGroup<BytesRef> searchGroup : topGroups) {
- System.out.println(" " + (searchGroup.groupValue == null ? "null" : searchGroup.groupValue) + ": " + Arrays.deepToString(searchGroup.sortValues));
- }
+ System.out.println("TEST: topGroups");
+ for (SearchGroup<BytesRef> searchGroup : topGroups) {
+ System.out.println(" " + (searchGroup.groupValue == null ? "null" : searchGroup.groupValue.utf8ToString()) + ": " + Arrays.deepToString(searchGroup.sortValues));
}
}
-
- // Get 1st pass top groups using shards
-
- ValueHolder<Boolean> idvBasedImplsUsedSharded = new ValueHolder<>(false);
- final TopGroups<BytesRef> topGroupsShards = searchShards(s, shards.subSearchers, query, groupSort, docSort,
- groupOffset, topNGroups, docOffset, docsPerGroup, getScores, getMaxScores, canUseIDV, false, idvBasedImplsUsedSharded);
- final AbstractSecondPassGroupingCollector<?> c2;
- if (topGroups != null) {
-
- if (VERBOSE) {
- System.out.println("TEST: topGroups");
- for (SearchGroup<BytesRef> searchGroup : topGroups) {
- System.out.println(" " + (searchGroup.groupValue == null ? "null" : searchGroup.groupValue.utf8ToString()) + ": " + Arrays.deepToString(searchGroup.sortValues));
- }
- }
-
- c2 = createSecondPassCollector(c1, groupField, groupSort, docSort, groupOffset, docOffset + docsPerGroup, getScores, getMaxScores, fillFields);
- if (doCache) {
- if (cCache.isCached()) {
- if (VERBOSE) {
- System.out.println("TEST: cache is intact");
- }
- cCache.replay(c2);
- } else {
- if (VERBOSE) {
- System.out.println("TEST: cache was too large");
- }
- s.search(query, c2);
+
+ c2 = createSecondPassCollector(c1, groupField, groupSort, docSort, groupOffset, docOffset + docsPerGroup, getScores, getMaxScores, fillFields);
+ if (doCache) {
+ if (cCache.isCached()) {
+ if (VERBOSE) {
+ System.out.println("TEST: cache is intact");
}
+ cCache.replay(c2);
} else {
+ if (VERBOSE) {
+ System.out.println("TEST: cache was too large");
+ }
s.search(query, c2);
}
-
- if (doAllGroups) {
- TopGroups<BytesRef> tempTopGroups = getTopGroups(c2, docOffset);
- groupsResult = new TopGroups<>(tempTopGroups, allGroupsCollector.getGroupCount());
- } else {
- groupsResult = getTopGroups(c2, docOffset);
- }
} else {
- c2 = null;
- groupsResult = null;
- if (VERBOSE) {
- System.out.println("TEST: no results");
- }
+ s.search(query, c2);
}
-
- final TopGroups<BytesRef> expectedGroups = slowGrouping(groupDocs, searchTerm, fillFields, getScores, getMaxScores, doAllGroups, groupSort, docSort, topNGroups, docsPerGroup, groupOffset, docOffset);
-
+
+ if (doAllGroups) {
+ TopGroups<BytesRef> tempTopGroups = getTopGroups(c2, docOffset);
+ groupsResult = new TopGroups<>(tempTopGroups, allGroupsCollector.getGroupCount());
+ } else {
+ groupsResult = getTopGroups(c2, docOffset);
+ }
+ } else {
+ c2 = null;
+ groupsResult = null;
if (VERBOSE) {
- if (expectedGroups == null) {
- System.out.println("TEST: no expected groups");
- } else {
- System.out.println("TEST: expected groups totalGroupedHitCount=" + expectedGroups.totalGroupedHitCount);
- for(GroupDocs<BytesRef> gd : expectedGroups.groups) {
- System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue) + " totalHits=" + gd.totalHits + " scoreDocs.len=" + gd.scoreDocs.length);
- for(ScoreDoc sd : gd.scoreDocs) {
- System.out.println(" id=" + sd.doc + " score=" + sd.score);
- }
+ System.out.println("TEST: no results");
+ }
+ }
+
+ final TopGroups<BytesRef> expectedGroups = slowGrouping(groupDocs, searchTerm, fillFields, getScores, getMaxScores, doAllGroups, groupSort, docSort, topNGroups, docsPerGroup, groupOffset, docOffset);
+
+ if (VERBOSE) {
+ if (expectedGroups == null) {
+ System.out.println("TEST: no expected groups");
+ } else {
+ System.out.println("TEST: expected groups totalGroupedHitCount=" + expectedGroups.totalGroupedHitCount);
+ for(GroupDocs<BytesRef> gd : expectedGroups.groups) {
+ System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue) + " totalHits=" + gd.totalHits + " scoreDocs.len=" + gd.scoreDocs.length);
+ for(ScoreDoc sd : gd.scoreDocs) {
+ System.out.println(" id=" + sd.doc + " score=" + sd.score);
}
}
-
- if (groupsResult == null) {
- System.out.println("TEST: no matched groups");
- } else {
- System.out.println("TEST: matched groups totalGroupedHitCount=" + groupsResult.totalGroupedHitCount);
- for(GroupDocs<BytesRef> gd : groupsResult.groups) {
- System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue) + " totalHits=" + gd.totalHits);
- for(ScoreDoc sd : gd.scoreDocs) {
- System.out.println(" id=" + docIDToID.get(sd.doc) + " score=" + sd.score);
- }
- }
-
- if (searchIter == 14) {
- for(int docIDX=0;docIDX<s.getIndexReader().maxDoc();docIDX++) {
- System.out.println("ID=" + docIDToID.get(docIDX) + " explain=" + s.explain(query, docIDX));
- }
+ }
+
+ if (groupsResult == null) {
+ System.out.println("TEST: no matched groups");
+ } else {
+ System.out.println("TEST: matched groups totalGroupedHitCount=" + groupsResult.totalGroupedHitCount);
+ for(GroupDocs<BytesRef> gd : groupsResult.groups) {
+ System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue) + " totalHits=" + gd.totalHits);
+ for(ScoreDoc sd : gd.scoreDocs) {
+ System.out.println(" id=" + docIDToID.get(sd.doc) + " score=" + sd.score);
}
}
-
- if (topGroupsShards == null) {
- System.out.println("TEST: no matched-merged groups");
- } else {
- System.out.println("TEST: matched-merged groups totalGroupedHitCount=" + topGroupsShards.totalGroupedHitCount);
- for(GroupDocs<BytesRef> gd : topGroupsShards.groups) {
- System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue) + " totalHits=" + gd.totalHits);
- for(ScoreDoc sd : gd.scoreDocs) {
- System.out.println(" id=" + docIDToID.get(sd.doc) + " score=" + sd.score);
- }
+
+ if (searchIter == 14) {
+ for(int docIDX=0;docIDX<s.getIndexReader().maxDoc();docIDX++) {
+ System.out.println("ID=" + docIDToID.get(docIDX) + " explain=" + s.explain(query, docIDX));
}
}
}
-
- assertEquals(docIDToID, expectedGroups, groupsResult, true, true, true, getScores, groupField.endsWith("_dv"));
-
- // Confirm merged shards match:
- assertEquals(docIDToID, expectedGroups, topGroupsShards, true, false, fillFields, getScores, idvBasedImplsUsedSharded.value);
- if (topGroupsShards != null) {
- verifyShards(shards.docStarts, topGroupsShards);
- }
-
- final boolean needsScores = getScores || getMaxScores || docSort == null;
- final BlockGroupingCollector c3 = new BlockGroupingCollector(groupSort, groupOffset+topNGroups, needsScores, lastDocInBlock);
- final TermAllGroupsCollector allGroupsCollector2;
- final Collector c4;
- if (doAllGroups) {
- // NOTE: must be "group" and not "group_dv"
- // (groupField) because we didn't index doc
- // values in the block index:
- allGroupsCollector2 = new TermAllGroupsCollector("group");
- c4 = MultiCollector.wrap(c3, allGroupsCollector2);
+
+ if (topGroupsShards == null) {
+ System.out.println("TEST: no matched-merged groups");
} else {
- allGroupsCollector2 = null;
- c4 = c3;
+ System.out.println("TEST: matched-merged groups totalGroupedHitCount=" + topGroupsShards.totalGroupedHitCount);
+ for(GroupDocs<BytesRef> gd : topGroupsShards.groups) {
+ System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue) + " totalHits=" + gd.totalHits);
+ for(ScoreDoc sd : gd.scoreDocs) {
+ System.out.println(" id=" + docIDToID.get(sd.doc) + " score=" + sd.score);
+ }
+ }
}
- // Get block grouping result:
- sBlocks.search(query, c4);
- @SuppressWarnings({"unchecked","rawtypes"})
- final TopGroups<BytesRef> tempTopGroupsBlocks = (TopGroups<BytesRef>) c3.getTopGroups(docSort, groupOffset, docOffset, docOffset+docsPerGroup, fillFields);
- final TopGroups<BytesRef> groupsResultBlocks;
- if (doAllGroups && tempTopGroupsBlocks != null) {
- assertEquals((int) tempTopGroupsBlocks.totalGroupCount, allGroupsCollector2.getGroupCount());
- groupsResultBlocks = new TopGroups<>(tempTopGroupsBlocks, allGroupsCollector2.getGroupCount());
+ }
+
+ assertEquals(docIDToID, expectedGroups, groupsResult, true, true, true, getScores, true);
+
+ // Confirm merged shards match:
+ assertEquals(docIDToID, expectedGroups, topGroupsShards, true, false, fillFields, getScores, true);
+ if (topGroupsShards != null) {
+ verifyShards(shards.docStarts, topGroupsShards);
+ }
+
+ final boolean needsScores = getScores || getMaxScores || docSort == null;
+ final BlockGroupingCollector c3 = new BlockGroupingCollector(groupSort, groupOffset+topNGroups, needsScores, lastDocInBlock);
+ final TermAllGroupsCollector allGroupsCollector2;
+ final Collector c4;
+ if (doAllGroups) {
+ // NOTE: must be "group" and not "group_dv"
+ // (groupField) because we didn't index doc
+ // values in the block index:
+ allGroupsCollector2 = new TermAllGroupsCollector("group");
+ c4 = MultiCollector.wrap(c3, allGroupsCollector2);
+ } else {
+ allGroupsCollector2 = null;
+ c4 = c3;
+ }
+ // Get block grouping result:
+ sBlocks.search(query, c4);
+ @SuppressWarnings({"unchecked","rawtypes"})
+ final TopGroups<BytesRef> tempTopGroupsBlocks = (TopGroups<BytesRef>) c3.getTopGroups(docSort, groupOffset, docOffset, docOffset+docsPerGroup, fillFields);
+ final TopGroups<BytesRef> groupsResultBlocks;
+ if (doAllGroups && tempTopGroupsBlocks != null) {
+ assertEquals((int) tempTopGroupsBlocks.totalGroupCount, allGroupsCollector2.getGroupCount());
+ groupsResultBlocks = new TopGroups<>(tempTopGroupsBlocks, allGroupsCollector2.getGroupCount());
+ } else {
+ groupsResultBlocks = tempTopGroupsBlocks;
+ }
+
+ if (VERBOSE) {
+ if (groupsResultBlocks == null) {
+ System.out.println("TEST: no block groups");
} else {
- groupsResultBlocks = tempTopGroupsBlocks;
- }
-
- if (VERBOSE) {
- if (groupsResultBlocks == null) {
- System.out.println("TEST: no block groups");
- } else {
- System.out.println("TEST: block groups totalGroupedHitCount=" + groupsResultBlocks.totalGroupedHitCount);
- boolean first = true;
- for(GroupDocs<BytesRef> gd : groupsResultBlocks.groups) {
- System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue.utf8ToString()) + " totalHits=" + gd.totalHits);
- for(ScoreDoc sd : gd.scoreDocs) {
- System.out.println(" id=" + docIDToIDBlocks.get(sd.doc) + " score=" + sd.score);
- if (first) {
- System.out.println("explain: " + sBlocks.explain(query, sd.doc));
- first = false;
- }
+ System.out.println("TEST: block groups totalGroupedHitCount=" + groupsResultBlocks.totalGroupedHitCount);
+ boolean first = true;
+ for(GroupDocs<BytesRef> gd : groupsResultBlocks.groups) {
+ System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue.utf8ToString()) + " totalHits=" + gd.totalHits);
+ for(ScoreDoc sd : gd.scoreDocs) {
+ System.out.println(" id=" + docIDToIDBlocks.get(sd.doc) + " score=" + sd.score);
+ if (first) {
+ System.out.println("explain: " + sBlocks.explain(query, sd.doc));
+ first = false;
}
}
}
}
-
- // Get shard'd block grouping result:
- // Block index does not index DocValues so we pass
- // false for canUseIDV:
- final TopGroups<BytesRef> topGroupsBlockShards = searchShards(sBlocks, shardsBlocks.subSearchers, query,
- groupSort, docSort, groupOffset, topNGroups, docOffset, docsPerGroup, getScores, getMaxScores, false, false, new ValueHolder<>(false));
-
- if (expectedGroups != null) {
- // Fixup scores for reader2
- for (GroupDocs<?> groupDocsHits : expectedGroups.groups) {
- for(ScoreDoc hit : groupDocsHits.scoreDocs) {
- final GroupDoc gd = groupDocsByID[hit.doc];
- assertEquals(gd.id, hit.doc);
- //System.out.println("fixup score " + hit.score + " to " + gd.score2 + " vs " + gd.score);
- hit.score = gd.score2;
- }
+ }
+
+ // Get shard'd block grouping result:
+ final TopGroups<BytesRef> topGroupsBlockShards = searchShards(sBlocks, shardsBlocks.subSearchers, query,
+ groupSort, docSort, groupOffset, topNGroups, docOffset, docsPerGroup, getScores, getMaxScores, false, false);
+
+ if (expectedGroups != null) {
+ // Fixup scores for reader2
+ for (GroupDocs<?> groupDocsHits : expectedGroups.groups) {
+ for(ScoreDoc hit : groupDocsHits.scoreDocs) {
+ final GroupDoc gd = groupDocsByID[hit.doc];
+ assertEquals(gd.id, hit.doc);
+ //System.out.println("fixup score " + hit.score + " to " + gd.score2 + " vs " + gd.score);
+ hit.score = gd.score2;
}
-
- final SortField[] sortFields = groupSort.getSort();
- final Map<Float,Float> termScoreMap = scoreMap.get(searchTerm);
- for(int groupSortIDX=0;groupSortIDX<sortFields.length;groupSortIDX++) {
- if (sortFields[groupSortIDX].getType() == SortField.Type.SCORE) {
- for (GroupDocs<?> groupDocsHits : expectedGroups.groups) {
- if (groupDocsHits.groupSortValues != null) {
- //System.out.println("remap " + groupDocsHits.groupSortValues[groupSortIDX] + " to " + termScoreMap.get(groupDocsHits.groupSortValues[groupSortIDX]));
- groupDocsHits.groupSortValues[groupSortIDX] = termScoreMap.get(groupDocsHits.groupSortValues[groupSortIDX]);
- assertNotNull(groupDocsHits.groupSortValues[groupSortIDX]);
- }
+ }
+
+ final SortField[] sortFields = groupSort.getSort();
+ final Map<Float,Float> termScoreMap = scoreMap.get(searchTerm);
+ for(int groupSortIDX=0;groupSortIDX<sortFields.length;groupSortIDX++) {
+ if (sortFields[groupSortIDX].getType() == SortField.Type.SCORE) {
+ for (GroupDocs<?> groupDocsHits : expectedGroups.groups) {
+ if (groupDocsHits.groupSortValues != null) {
+ //System.out.println("remap " + groupDocsHits.groupSortValues[groupSortIDX] + " to " + termScoreMap.get(groupDocsHits.groupSortValues[groupSortIDX]));
+ groupDocsHits.groupSortValues[groupSortIDX] = termScoreMap.get(groupDocsHits.groupSortValues[groupSortIDX]);
+ assertNotNull(groupDocsHits.groupSortValues[groupSortIDX]);
}
}
}
-
- final SortField[] docSortFields = docSort.getSort();
- for(int docSortIDX=0;docSortIDX<docSortFields.length;docSortIDX++) {
- if (docSortFields[docSortIDX].getType() == SortField.Type.SCORE) {
- for (GroupDocs<?> groupDocsHits : expectedGroups.groups) {
- for(ScoreDoc _hit : groupDocsHits.scoreDocs) {
- FieldDoc hit = (FieldDoc) _hit;
- if (hit.fields != null) {
- hit.fields[docSortIDX] = termScoreMap.get(hit.fields[docSortIDX]);
- assertNotNull(hit.fields[docSortIDX]);
- }
+ }
+
+ final SortField[] docSortFields = docSort.getSort();
+ for(int docSortIDX=0;docSortIDX<docSortFields.length;docSortIDX++) {
+ if (docSortFields[docSortIDX].getType() == SortField.Type.SCORE) {
+ for (GroupDocs<?> groupDocsHits : expectedGroups.groups) {
+ for(ScoreDoc _hit : groupDocsHits.scoreDocs) {
+ FieldDoc hit = (FieldDoc) _hit;
+ if (hit.fields != null) {
+ hit.fields[docSortIDX] = termScoreMap.get(hit.fields[docSortIDX]);
+ assertNotNull(hit.fields[docSortIDX]);
}
}
}
}
}
-
- assertEquals(docIDToIDBlocks, expectedGroups, groupsResultBlocks, false, true, true, getScores, false);
- assertEquals(docIDToIDBlocks, expectedGroups, topGroupsBlockShards, false, false, fillFields, getScores, false);
- }
- } finally {
- QueryUtils.purgeFieldCache(r);
- if (rBlocks != null) {
- QueryUtils.purgeFieldCache(rBlocks);
}
+
+ assertEquals(docIDToIDBlocks, expectedGroups, groupsResultBlocks, false, true, true, getScores, false);
+ assertEquals(docIDToIDBlocks, expectedGroups, topGroupsBlockShards, false, false, fillFields, getScores, false);
}
-
+
r.close();
dir.close();
-
+
rBlocks.close();
dirBlocks.close();
}
@@ -1158,7 +1142,7 @@ public class TestGrouping extends Lucene
}
private TopGroups<BytesRef> searchShards(IndexSearcher topSearcher, ShardSearcher[] subSearchers, Query query, Sort groupSort, Sort docSort, int groupOffset, int topNGroups, int docOffset,
- int topNDocs, boolean getScores, boolean getMaxScores, boolean canUseIDV, boolean preFlex, ValueHolder<Boolean> usedIdvBasedImpl) throws Exception {
+ int topNDocs, boolean getScores, boolean getMaxScores, boolean canUseIDV, boolean preFlex) throws Exception {
// TODO: swap in caching, all groups collector hereassertEquals(expected.totalHitCount, actual.totalHitCount);
// too...
@@ -1182,10 +1166,6 @@ public class TestGrouping extends Lucene
}
String groupField = "group";
- if (shardsCanUseIDV && random().nextBoolean()) {
- groupField += "_dv";
- usedIdvBasedImpl.value = true;
- }
for(int shardIDX=0;shardIDX<subSearchers.length;shardIDX++) {
@@ -1257,7 +1237,7 @@ public class TestGrouping extends Lucene
}
}
- private void assertEquals(FieldCache.Ints docIDtoID, TopGroups<BytesRef> expected, TopGroups<BytesRef> actual, boolean verifyGroupValues, boolean verifyTotalGroupCount, boolean verifySortValues, boolean testScores, boolean idvBasedImplsUsed) {
+ private void assertEquals(NumericDocValues docIDtoID, TopGroups<BytesRef> expected, TopGroups<BytesRef> actual, boolean verifyGroupValues, boolean verifyTotalGroupCount, boolean verifySortValues, boolean testScores, boolean idvBasedImplsUsed) {
if (expected == null) {
assertNull(actual);
return;
Modified: lucene/dev/branches/lucene5675/lucene/join/src/java/org/apache/lucene/search/join/TermsCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/join/src/java/org/apache/lucene/search/join/TermsCollector.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/join/src/java/org/apache/lucene/search/join/TermsCollector.java (original)
+++ lucene/dev/branches/lucene5675/lucene/join/src/java/org/apache/lucene/search/join/TermsCollector.java Mon May 19 10:34:35 2014
@@ -21,10 +21,10 @@ import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.SimpleCollector;
import org.apache.lucene.util.BytesRef;
@@ -85,7 +85,7 @@ abstract class TermsCollector extends Si
@Override
protected void doSetNextReader(AtomicReaderContext context) throws IOException {
- docTermOrds = FieldCache.DEFAULT.getDocTermOrds(context.reader(), field);
+ docTermOrds = DocValues.getSortedSet(context.reader(), field);
}
}
@@ -107,7 +107,7 @@ abstract class TermsCollector extends Si
@Override
protected void doSetNextReader(AtomicReaderContext context) throws IOException {
- fromDocTerms = FieldCache.DEFAULT.getTerms(context.reader(), field, false);
+ fromDocTerms = DocValues.getBinary(context.reader(), field);
}
}
Modified: lucene/dev/branches/lucene5675/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java (original)
+++ lucene/dev/branches/lucene5675/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java Mon May 19 10:34:35 2014
@@ -21,10 +21,10 @@ import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.SimpleCollector;
import org.apache.lucene.util.ArrayUtil;
@@ -131,7 +131,7 @@ abstract class TermsWithScoreCollector e
@Override
protected void doSetNextReader(AtomicReaderContext context) throws IOException {
- fromDocTerms = FieldCache.DEFAULT.getTerms(context.reader(), field, false);
+ fromDocTerms = DocValues.getBinary(context.reader(), field);
}
static class Avg extends SV {
@@ -217,7 +217,7 @@ abstract class TermsWithScoreCollector e
@Override
protected void doSetNextReader(AtomicReaderContext context) throws IOException {
- fromDocTermOrds = FieldCache.DEFAULT.getDocTermOrds(context.reader(), field);
+ fromDocTermOrds = DocValues.getSortedSet(context.reader(), field);
}
static class Avg extends MV {
Modified: lucene/dev/branches/lucene5675/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java (original)
+++ lucene/dev/branches/lucene5675/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java Mon May 19 10:34:35 2014
@@ -17,6 +17,13 @@ package org.apache.lucene.search.join;
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.*;
import org.apache.lucene.index.*;
@@ -26,14 +33,9 @@ import org.apache.lucene.search.grouping
import org.apache.lucene.search.grouping.TopGroups;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.*;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-
+@SuppressCodecs({"Lucene40", "Lucene41", "Lucene42"}) // needs working "missing"
public class TestBlockJoin extends LuceneTestCase {
// One resume...
@@ -451,7 +453,7 @@ public class TestBlockJoin extends Lucen
final String[] values = fields[fieldID] = new String[valueCount];
for(int i=0;i<valueCount;i++) {
values[i] = TestUtil.randomRealisticUnicodeString(random());
- //values[i] = _TestUtil.randomSimpleString(random);
+ //values[i] = TestUtil.randomSimpleString(random());
}
}
@@ -511,9 +513,18 @@ public class TestBlockJoin extends Lucen
parentDoc.add(id);
parentJoinDoc.add(id);
parentJoinDoc.add(newStringField("isParent", "x", Field.Store.NO));
+ id = new NumericDocValuesField("parentID", parentDocID);
+ parentDoc.add(id);
+ parentJoinDoc.add(id);
+ parentJoinDoc.add(newStringField("isParent", "x", Field.Store.NO));
for(int field=0;field<parentFields.length;field++) {
if (random().nextDouble() < 0.9) {
- Field f = newStringField("parent" + field, parentFields[field][random().nextInt(parentFields[field].length)], Field.Store.NO);
+ String s = parentFields[field][random().nextInt(parentFields[field].length)];
+ Field f = newStringField("parent" + field, s, Field.Store.NO);
+ parentDoc.add(f);
+ parentJoinDoc.add(f);
+
+ f = new SortedDocValuesField("parent" + field, new BytesRef(s));
parentDoc.add(f);
parentJoinDoc.add(f);
}
@@ -548,10 +559,18 @@ public class TestBlockJoin extends Lucen
Field childID = new IntField("childID", childDocID, Field.Store.YES);
childDoc.add(childID);
joinChildDoc.add(childID);
+ childID = new NumericDocValuesField("childID", childDocID);
+ childDoc.add(childID);
+ joinChildDoc.add(childID);
for(int childFieldID=0;childFieldID<childFields.length;childFieldID++) {
if (random().nextDouble() < 0.9) {
- Field f = newStringField("child" + childFieldID, childFields[childFieldID][random().nextInt(childFields[childFieldID].length)], Field.Store.NO);
+ String s = childFields[childFieldID][random().nextInt(childFields[childFieldID].length)];
+ Field f = newStringField("child" + childFieldID, s, Field.Store.NO);
+ childDoc.add(f);
+ joinChildDoc.add(f);
+
+ f = new SortedDocValuesField("child" + childFieldID, new BytesRef(s));
childDoc.add(f);
joinChildDoc.add(f);
}
@@ -727,7 +746,7 @@ public class TestBlockJoin extends Lucen
parentAndChildSort);
if (VERBOSE) {
- System.out.println("\nTEST: normal index gets " + results.totalHits + " hits");
+ System.out.println("\nTEST: normal index gets " + results.totalHits + " hits; sort=" + parentAndChildSort);
final ScoreDoc[] hits = results.scoreDocs;
for(int hitIDX=0;hitIDX<hits.length;hitIDX++) {
final StoredDocument doc = s.doc(hits[hitIDX].doc);
@@ -735,7 +754,7 @@ public class TestBlockJoin extends Lucen
System.out.println(" parentID=" + doc.get("parentID") + " childID=" + doc.get("childID") + " (docID=" + hits[hitIDX].doc + ")");
FieldDoc fd = (FieldDoc) hits[hitIDX];
if (fd.fields != null) {
- System.out.print(" ");
+ System.out.print(" " + fd.fields.length + " sort values: ");
for(Object o : fd.fields) {
if (o instanceof BytesRef) {
System.out.print(((BytesRef) o).utf8ToString() + " ");
Modified: lucene/dev/branches/lucene5675/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinSorting.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinSorting.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinSorting.java (original)
+++ lucene/dev/branches/lucene5675/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinSorting.java Mon May 19 10:34:35 2014
@@ -20,6 +20,7 @@ package org.apache.lucene.search.join;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.NoMergePolicy;
@@ -58,14 +59,17 @@ public class TestBlockJoinSorting extend
List<Document> docs = new ArrayList<>();
Document document = new Document();
document.add(new StringField("field2", "a", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("a")));
document.add(new StringField("filter_1", "T", Field.Store.NO));
docs.add(document);
document = new Document();
document.add(new StringField("field2", "b", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("b")));
document.add(new StringField("filter_1", "T", Field.Store.NO));
docs.add(document);
document = new Document();
document.add(new StringField("field2", "c", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("c")));
document.add(new StringField("filter_1", "T", Field.Store.NO));
docs.add(document);
document = new Document();
@@ -78,14 +82,17 @@ public class TestBlockJoinSorting extend
docs.clear();
document = new Document();
document.add(new StringField("field2", "c", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("c")));
document.add(new StringField("filter_1", "T", Field.Store.NO));
docs.add(document);
document = new Document();
document.add(new StringField("field2", "d", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("d")));
document.add(new StringField("filter_1", "T", Field.Store.NO));
docs.add(document);
document = new Document();
document.add(new StringField("field2", "e", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("e")));
document.add(new StringField("filter_1", "T", Field.Store.NO));
docs.add(document);
document = new Document();
@@ -97,14 +104,17 @@ public class TestBlockJoinSorting extend
docs.clear();
document = new Document();
document.add(new StringField("field2", "e", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("e")));
document.add(new StringField("filter_1", "T", Field.Store.NO));
docs.add(document);
document = new Document();
document.add(new StringField("field2", "f", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("f")));
document.add(new StringField("filter_1", "T", Field.Store.NO));
docs.add(document);
document = new Document();
document.add(new StringField("field2", "g", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("g")));
document.add(new StringField("filter_1", "T", Field.Store.NO));
docs.add(document);
document = new Document();
@@ -116,14 +126,17 @@ public class TestBlockJoinSorting extend
docs.clear();
document = new Document();
document.add(new StringField("field2", "g", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("g")));
document.add(new StringField("filter_1", "T", Field.Store.NO));
docs.add(document);
document = new Document();
document.add(new StringField("field2", "h", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("h")));
document.add(new StringField("filter_1", "F", Field.Store.NO));
docs.add(document);
document = new Document();
document.add(new StringField("field2", "i", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("i")));
document.add(new StringField("filter_1", "F", Field.Store.NO));
docs.add(document);
document = new Document();
@@ -136,14 +149,17 @@ public class TestBlockJoinSorting extend
docs.clear();
document = new Document();
document.add(new StringField("field2", "i", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("i")));
document.add(new StringField("filter_1", "F", Field.Store.NO));
docs.add(document);
document = new Document();
document.add(new StringField("field2", "j", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("j")));
document.add(new StringField("filter_1", "F", Field.Store.NO));
docs.add(document);
document = new Document();
document.add(new StringField("field2", "k", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("k")));
document.add(new StringField("filter_1", "F", Field.Store.NO));
docs.add(document);
document = new Document();
@@ -155,14 +171,17 @@ public class TestBlockJoinSorting extend
docs.clear();
document = new Document();
document.add(new StringField("field2", "k", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("k")));
document.add(new StringField("filter_1", "T", Field.Store.NO));
docs.add(document);
document = new Document();
document.add(new StringField("field2", "l", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("l")));
document.add(new StringField("filter_1", "T", Field.Store.NO));
docs.add(document);
document = new Document();
document.add(new StringField("field2", "m", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("m")));
document.add(new StringField("filter_1", "T", Field.Store.NO));
docs.add(document);
document = new Document();
@@ -180,14 +199,17 @@ public class TestBlockJoinSorting extend
docs.clear();
document = new Document();
document.add(new StringField("field2", "m", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("m")));
document.add(new StringField("filter_1", "T", Field.Store.NO));
docs.add(document);
document = new Document();
document.add(new StringField("field2", "n", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("n")));
document.add(new StringField("filter_1", "F", Field.Store.NO));
docs.add(document);
document = new Document();
document.add(new StringField("field2", "o", Field.Store.NO));
+ document.add(new SortedDocValuesField("field2", new BytesRef("o")));
document.add(new StringField("filter_1", "F", Field.Store.NO));
docs.add(document);
document = new Document();
Modified: lucene/dev/branches/lucene5675/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java (original)
+++ lucene/dev/branches/lucene5675/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java Mon May 19 10:34:35 2014
@@ -34,10 +34,13 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiFields;
@@ -53,7 +56,6 @@ import org.apache.lucene.search.BooleanQ
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
@@ -68,9 +70,11 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.TestUtil;
import org.junit.Test;
+@SuppressCodecs({"Lucene40", "Lucene41", "Lucene42"}) // we need SortedSet, docsWithField
public class TestJoinUtil extends LuceneTestCase {
public void testSimple() throws Exception {
@@ -89,20 +93,25 @@ public class TestJoinUtil extends Lucene
doc.add(new TextField("description", "random text", Field.Store.NO));
doc.add(new TextField("name", "name1", Field.Store.NO));
doc.add(new TextField(idField, "1", Field.Store.NO));
+ doc.add(new SortedDocValuesField(idField, new BytesRef("1")));
w.addDocument(doc);
// 1
doc = new Document();
doc.add(new TextField("price", "10.0", Field.Store.NO));
doc.add(new TextField(idField, "2", Field.Store.NO));
+ doc.add(new SortedDocValuesField(idField, new BytesRef("2")));
doc.add(new TextField(toField, "1", Field.Store.NO));
+ doc.add(new SortedDocValuesField(toField, new BytesRef("1")));
w.addDocument(doc);
// 2
doc = new Document();
doc.add(new TextField("price", "20.0", Field.Store.NO));
doc.add(new TextField(idField, "3", Field.Store.NO));
+ doc.add(new SortedDocValuesField(idField, new BytesRef("3")));
doc.add(new TextField(toField, "1", Field.Store.NO));
+ doc.add(new SortedDocValuesField(toField, new BytesRef("1")));
w.addDocument(doc);
// 3
@@ -110,6 +119,7 @@ public class TestJoinUtil extends Lucene
doc.add(new TextField("description", "more random text", Field.Store.NO));
doc.add(new TextField("name", "name2", Field.Store.NO));
doc.add(new TextField(idField, "4", Field.Store.NO));
+ doc.add(new SortedDocValuesField(idField, new BytesRef("4")));
w.addDocument(doc);
w.commit();
@@ -117,14 +127,18 @@ public class TestJoinUtil extends Lucene
doc = new Document();
doc.add(new TextField("price", "10.0", Field.Store.NO));
doc.add(new TextField(idField, "5", Field.Store.NO));
+ doc.add(new SortedDocValuesField(idField, new BytesRef("5")));
doc.add(new TextField(toField, "4", Field.Store.NO));
+ doc.add(new SortedDocValuesField(toField, new BytesRef("4")));
w.addDocument(doc);
// 5
doc = new Document();
doc.add(new TextField("price", "20.0", Field.Store.NO));
doc.add(new TextField(idField, "6", Field.Store.NO));
+ doc.add(new SortedDocValuesField(idField, new BytesRef("6")));
doc.add(new TextField(toField, "4", Field.Store.NO));
+ doc.add(new SortedDocValuesField(toField, new BytesRef("4")));
w.addDocument(doc);
IndexSearcher indexSearcher = new IndexSearcher(w.getReader());
@@ -180,16 +194,18 @@ public class TestJoinUtil extends Lucene
doc.add(new TextField("description", "random text", Field.Store.NO));
doc.add(new TextField("name", "name1", Field.Store.NO));
doc.add(new TextField(idField, "0", Field.Store.NO));
+ doc.add(new SortedDocValuesField(idField, new BytesRef("0")));
w.addDocument(doc);
doc = new Document();
doc.add(new TextField("price", "10.0", Field.Store.NO));
- for(int i=0;i<300;i++){
- doc.add(new TextField(toField, ""+i, Field.Store.NO));
- if(!multipleValues){
- w.addDocument(doc);
- doc.removeFields(toField);
+
+ if (multipleValues) {
+ for(int i=0;i<300;i++) {
+ doc.add(new SortedSetDocValuesField(toField, new BytesRef(""+i)));
}
+ } else {
+ doc.add(new SortedDocValuesField(toField, new BytesRef("0")));
}
w.addDocument(doc);
@@ -317,20 +333,25 @@ public class TestJoinUtil extends Lucene
doc.add(new TextField("description", "A random movie", Field.Store.NO));
doc.add(new TextField("name", "Movie 1", Field.Store.NO));
doc.add(new TextField(idField, "1", Field.Store.NO));
+ doc.add(new SortedDocValuesField(idField, new BytesRef("1")));
w.addDocument(doc);
// 1
doc = new Document();
doc.add(new TextField("subtitle", "The first subtitle of this movie", Field.Store.NO));
doc.add(new TextField(idField, "2", Field.Store.NO));
+ doc.add(new SortedDocValuesField(idField, new BytesRef("2")));
doc.add(new TextField(toField, "1", Field.Store.NO));
+ doc.add(new SortedDocValuesField(toField, new BytesRef("1")));
w.addDocument(doc);
// 2
doc = new Document();
doc.add(new TextField("subtitle", "random subtitle; random event movie", Field.Store.NO));
doc.add(new TextField(idField, "3", Field.Store.NO));
+ doc.add(new SortedDocValuesField(idField, new BytesRef("3")));
doc.add(new TextField(toField, "1", Field.Store.NO));
+ doc.add(new SortedDocValuesField(toField, new BytesRef("1")));
w.addDocument(doc);
// 3
@@ -338,6 +359,7 @@ public class TestJoinUtil extends Lucene
doc.add(new TextField("description", "A second random movie", Field.Store.NO));
doc.add(new TextField("name", "Movie 2", Field.Store.NO));
doc.add(new TextField(idField, "4", Field.Store.NO));
+ doc.add(new SortedDocValuesField(idField, new BytesRef("4")));
w.addDocument(doc);
w.commit();
@@ -345,14 +367,18 @@ public class TestJoinUtil extends Lucene
doc = new Document();
doc.add(new TextField("subtitle", "a very random event happened during christmas night", Field.Store.NO));
doc.add(new TextField(idField, "5", Field.Store.NO));
+ doc.add(new SortedDocValuesField(idField, new BytesRef("5")));
doc.add(new TextField(toField, "4", Field.Store.NO));
+ doc.add(new SortedDocValuesField(toField, new BytesRef("4")));
w.addDocument(doc);
// 5
doc = new Document();
doc.add(new TextField("subtitle", "movie end movie test 123 test 123 random", Field.Store.NO));
doc.add(new TextField(idField, "6", Field.Store.NO));
+ doc.add(new SortedDocValuesField(idField, new BytesRef("6")));
doc.add(new TextField(toField, "4", Field.Store.NO));
+ doc.add(new SortedDocValuesField(toField, new BytesRef("4")));
w.addDocument(doc);
IndexSearcher indexSearcher = new IndexSearcher(w.getReader());
@@ -572,6 +598,11 @@ public class TestJoinUtil extends Lucene
context.fromDocuments.get(linkValue).add(docs[i]);
context.randomValueFromDocs.get(value).add(docs[i]);
document.add(newTextField(random(), "from", linkValue, Field.Store.NO));
+ if (multipleValuesPerDocument) {
+ document.add(new SortedSetDocValuesField("from", new BytesRef(linkValue)));
+ } else {
+ document.add(new SortedDocValuesField("from", new BytesRef(linkValue)));
+ }
} else {
if (!context.toDocuments.containsKey(linkValue)) {
context.toDocuments.put(linkValue, new ArrayList<RandomDoc>());
@@ -583,6 +614,11 @@ public class TestJoinUtil extends Lucene
context.toDocuments.get(linkValue).add(docs[i]);
context.randomValueToDocs.get(value).add(docs[i]);
document.add(newTextField(random(), "to", linkValue, Field.Store.NO));
+ if (multipleValuesPerDocument) {
+ document.add(new SortedSetDocValuesField("to", new BytesRef(linkValue)));
+ } else {
+ document.add(new SortedDocValuesField("to", new BytesRef(linkValue)));
+ }
}
}
@@ -644,7 +680,7 @@ public class TestJoinUtil extends Lucene
@Override
protected void doSetNextReader(AtomicReaderContext context) throws IOException {
- docTermOrds = FieldCache.DEFAULT.getDocTermOrds(context.reader(), fromField);
+ docTermOrds = DocValues.getSortedSet(context.reader(), fromField);
}
@Override
@@ -682,8 +718,8 @@ public class TestJoinUtil extends Lucene
@Override
protected void doSetNextReader(AtomicReaderContext context) throws IOException {
- terms = FieldCache.DEFAULT.getTerms(context.reader(), fromField, true);
- docsWithField = FieldCache.DEFAULT.getDocsWithField(context.reader(), fromField);
+ terms = DocValues.getBinary(context.reader(), fromField);
+ docsWithField = DocValues.getDocsWithField(context.reader(), fromField);
}
@Override
@@ -753,7 +789,7 @@ public class TestJoinUtil extends Lucene
@Override
protected void doSetNextReader(AtomicReaderContext context) throws IOException {
docBase = context.docBase;
- docTermOrds = FieldCache.DEFAULT.getDocTermOrds(context.reader(), toField);
+ docTermOrds = DocValues.getSortedSet(context.reader(), toField);
}
@Override
@@ -781,7 +817,7 @@ public class TestJoinUtil extends Lucene
@Override
protected void doSetNextReader(AtomicReaderContext context) throws IOException {
- terms = FieldCache.DEFAULT.getTerms(context.reader(), toField, false);
+ terms = DocValues.getBinary(context.reader(), toField);
docBase = context.docBase;
}
Modified: lucene/dev/branches/lucene5675/lucene/misc/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/misc/build.xml?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/misc/build.xml (original)
+++ lucene/dev/branches/lucene5675/lucene/misc/build.xml Mon May 19 10:34:35 2014
@@ -32,6 +32,10 @@
org/apache/lucene/misc/IndexMergeTool.class
"/>
+ <property name="forbidden-rue-excludes" value="
+ org/apache/lucene/uninverting/FieldCache$CacheEntry.class
+ "/>
+
<import file="../module-build.xml"/>
<target name="install-cpptasks" unless="cpptasks.uptodate" depends="ivy-availability-check,ivy-fail,ivy-configure">
Modified: lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java (original)
+++ lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java Mon May 19 10:34:35 2014
@@ -406,5 +406,11 @@ public class NativeUnixDirectory extends
throw new RuntimeException("IOException during clone: " + this, ioe);
}
}
+
+ @Override
+ public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
+ // TODO: is this the right thing to do?
+ return BufferedIndexInput.wrap(sliceDescription, this, offset, length);
+ }
}
}
Modified: lucene/dev/branches/lucene5675/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreProvider.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreProvider.java (original)
+++ lucene/dev/branches/lucene5675/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreProvider.java Mon May 19 10:34:35 2014
@@ -23,7 +23,6 @@ import org.apache.lucene.index.AtomicRea
import org.apache.lucene.index.IndexReader; // for javadocs
import org.apache.lucene.queries.function.FunctionQuery;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.FieldCache; // for javadocs
/**
* An instance of this subclass should be returned by
@@ -32,7 +31,7 @@ import org.apache.lucene.search.FieldCac
* <p>Since Lucene 2.9, queries operate on each segment of an index separately,
* so the protected {@link #context} field can be used to resolve doc IDs,
* as the supplied <code>doc</code> ID is per-segment and without knowledge
- * of the IndexReader you cannot access the document or {@link FieldCache}.
+ * of the IndexReader you cannot access the document or DocValues.
*
* @lucene.experimental
* @since 2.9.2
Modified: lucene/dev/branches/lucene5675/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java (original)
+++ lucene/dev/branches/lucene5675/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java Mon May 19 10:34:35 2014
@@ -20,12 +20,12 @@ package org.apache.lucene.queries.functi
import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.ValueSourceScorer;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.UnicodeUtil;
@@ -44,12 +44,12 @@ public abstract class DocTermsIndexDocVa
protected final CharsRef spareChars = new CharsRef();
public DocTermsIndexDocValues(ValueSource vs, AtomicReaderContext context, String field) throws IOException {
- try {
- termsIndex = FieldCache.DEFAULT.getTermsIndex(context.reader(), field);
- } catch (RuntimeException e) {
- throw new DocTermsIndexException(field, e);
- }
+ this(vs, open(context, field));
+ }
+
+ protected DocTermsIndexDocValues(ValueSource vs, SortedDocValues termsIndex) {
this.vs = vs;
+ this.termsIndex = termsIndex;
}
protected abstract String toTerm(String readableValue);
@@ -162,6 +162,15 @@ public abstract class DocTermsIndexDocVa
};
}
+ // TODO: why?
+ static SortedDocValues open(AtomicReaderContext context, String field) throws IOException {
+ try {
+ return DocValues.getSorted(context.reader(), field);
+ } catch (RuntimeException e) {
+ throw new DocTermsIndexException(field, e);
+ }
+ }
+
/**
* Custom Exception to be thrown when the DocTermsIndex for a field cannot be generated
*/
Modified: lucene/dev/branches/lucene5675/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java (original)
+++ lucene/dev/branches/lucene5675/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java Mon May 19 10:34:35 2014
@@ -22,13 +22,15 @@ import java.util.Map;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.docvalues.DocTermsIndexDocValues;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.mutable.MutableValue;
+import org.apache.lucene.util.mutable.MutableValueStr;
/**
* An implementation for retrieving {@link FunctionValues} instances for string based fields.
@@ -42,11 +44,12 @@ public class BytesRefFieldSource extends
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final FieldInfo fieldInfo = readerContext.reader().getFieldInfos().fieldInfo(field);
+
// To be sorted or not to be sorted, that is the question
// TODO: do it cleaner?
if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.BINARY) {
- final BinaryDocValues binaryValues = FieldCache.DEFAULT.getTerms(readerContext.reader(), field, true);
- final Bits docsWithField = FieldCache.DEFAULT.getDocsWithField(readerContext.reader(), field);
+ final BinaryDocValues binaryValues = DocValues.getBinary(readerContext.reader(), field);
+ final Bits docsWithField = DocValues.getDocsWithField(readerContext.reader(), field);
return new FunctionValues() {
@Override
@@ -76,6 +79,31 @@ public class BytesRefFieldSource extends
public String toString(int doc) {
return description() + '=' + strVal(doc);
}
+
+ @Override
+ public ValueFiller getValueFiller() {
+ return new ValueFiller() {
+ private final MutableValueStr mval = new MutableValueStr();
+
+ @Override
+ public MutableValue getValue() {
+ return mval;
+ }
+
+ @Override
+ public void fillValue(int doc) {
+ mval.exists = docsWithField.get(doc);
+ if (mval.exists) {
+ binaryValues.get(doc, mval.value);
+ } else {
+ mval.value.bytes = BytesRef.EMPTY_BYTES;
+ mval.value.offset = 0;
+ mval.value.length = 0;
+ }
+ }
+ };
+ }
+
};
} else {
return new DocTermsIndexDocValues(this, readerContext, field) {
Modified: lucene/dev/branches/lucene5675/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java?rev=1595824&r1=1595823&r2=1595824&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java (original)
+++ lucene/dev/branches/lucene5675/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java Mon May 19 10:34:35 2014
@@ -20,31 +20,24 @@ package org.apache.lucene.queries.functi
import java.io.IOException;
import java.util.Map;
+import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSourceScorer;
import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueDouble;
/**
- * Obtains double field values from {@link FieldCache#getDoubles} and makes
+ * Obtains double field values from {@link AtomicReader#getNumericDocValues} and makes
* those values available as other numeric types, casting as needed.
*/
public class DoubleFieldSource extends FieldCacheSource {
- protected final FieldCache.DoubleParser parser;
-
public DoubleFieldSource(String field) {
- this(field, null);
- }
-
- public DoubleFieldSource(String field, FieldCache.DoubleParser parser) {
super(field);
- this.parser = parser;
}
@Override
@@ -54,12 +47,12 @@ public class DoubleFieldSource extends F
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
- final FieldCache.Doubles arr = cache.getDoubles(readerContext.reader(), field, parser, true);
- final Bits valid = cache.getDocsWithField(readerContext.reader(), field);
+ final NumericDocValues arr = DocValues.getNumeric(readerContext.reader(), field);
+ final Bits valid = DocValues.getDocsWithField(readerContext.reader(), field);
return new DoubleDocValues(this) {
@Override
public double doubleVal(int doc) {
- return arr.get(doc);
+ return Double.longBitsToDouble(arr.get(doc));
}
@Override
@@ -79,29 +72,24 @@ public class DoubleFieldSource extends F
@Override
public void fillValue(int doc) {
- mval.value = arr.get(doc);
+ mval.value = doubleVal(doc);
mval.exists = mval.value != 0 || valid.get(doc);
}
};
}
-
-
- };
-
+ };
}
@Override
public boolean equals(Object o) {
if (o.getClass() != DoubleFieldSource.class) return false;
DoubleFieldSource other = (DoubleFieldSource) o;
- return super.equals(other)
- && (this.parser == null ? other.parser == null :
- this.parser.getClass() == other.parser.getClass());
+ return super.equals(other);
}
@Override
public int hashCode() {
- int h = parser == null ? Double.class.hashCode() : parser.getClass().hashCode();
+ int h = Double.class.hashCode();
h += super.hashCode();
return h;
}