You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/11/24 00:27:19 UTC
svn commit: r1544892 [2/2] - in /lucene/dev/branches/lucene5339: ./
lucene/facet/src/java/org/apache/lucene/facet/simple/
lucene/facet/src/java/org/apache/lucene/facet/taxonomy/
lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/ lucene/f...
Modified: lucene/dev/branches/lucene5339/lucene/facet/src/test/org/apache/lucene/facet/simple/TestTaxonomyFacetSumValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/lucene/facet/src/test/org/apache/lucene/facet/simple/TestTaxonomyFacetSumValueSource.java?rev=1544892&r1=1544891&r2=1544892&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/lucene/facet/src/test/org/apache/lucene/facet/simple/TestTaxonomyFacetSumValueSource.java (original)
+++ lucene/dev/branches/lucene5339/lucene/facet/src/test/org/apache/lucene/facet/simple/TestTaxonomyFacetSumValueSource.java Sat Nov 23 23:27:18 2013
@@ -24,12 +24,15 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.StringField;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.taxonomy.FacetLabel;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
@@ -37,13 +40,26 @@ import org.apache.lucene.facet.taxonomy.
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
import org.apache.lucene.facet.util.PrintTaxonomyStats;
+import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.queries.function.FunctionQuery;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
import org.apache.lucene.queries.function.valuesource.IntFieldSource;
+import org.apache.lucene.queries.function.valuesource.LongFieldSource;
+import org.apache.lucene.queries.function.valuesource.QueryValueSource;
+import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MultiCollector;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
import org.apache.lucene.search.similarities.Similarity;
@@ -64,34 +80,34 @@ public class TestTaxonomyFacetSumValueSo
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
- DocumentBuilder builder = new DocumentBuilder(taxoWriter, new FacetsConfig());
+ FacetsConfig config = new FacetsConfig(taxoWriter);
// Reused across documents, to add the necessary facet
// fields:
Document doc = new Document();
doc.add(new IntField("num", 10, Field.Store.NO));
doc.add(new FacetField("Author", "Bob"));
- writer.addDocument(builder.build(doc));
+ writer.addDocument(config.build(doc));
doc = new Document();
doc.add(new IntField("num", 20, Field.Store.NO));
doc.add(new FacetField("Author", "Lisa"));
- writer.addDocument(builder.build(doc));
+ writer.addDocument(config.build(doc));
doc = new Document();
doc.add(new IntField("num", 30, Field.Store.NO));
doc.add(new FacetField("Author", "Lisa"));
- writer.addDocument(builder.build(doc));
+ writer.addDocument(config.build(doc));
doc = new Document();
doc.add(new IntField("num", 40, Field.Store.NO));
doc.add(new FacetField("Author", "Susan"));
- writer.addDocument(builder.build(doc));
+ writer.addDocument(config.build(doc));
doc = new Document();
doc.add(new IntField("num", 45, Field.Store.NO));
doc.add(new FacetField("Author", "Frank"));
- writer.addDocument(builder.build(doc));
+ writer.addDocument(config.build(doc));
// NRT open
IndexSearcher searcher = newSearcher(writer.getReader());
@@ -131,12 +147,12 @@ public class TestTaxonomyFacetSumValueSo
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
- DocumentBuilder builder = new DocumentBuilder(taxoWriter, new FacetsConfig());
+ FacetsConfig config = new FacetsConfig(taxoWriter);
Document doc = new Document();
doc.add(new IntField("num", 10, Field.Store.NO));
doc.add(new FacetField("a", "foo1"));
- writer.addDocument(builder.build(doc));
+ writer.addDocument(config.build(doc));
if (random().nextBoolean()) {
writer.commit();
@@ -146,7 +162,7 @@ public class TestTaxonomyFacetSumValueSo
doc.add(new IntField("num", 20, Field.Store.NO));
doc.add(new FacetField("a", "foo2"));
doc.add(new FacetField("b", "bar1"));
- writer.addDocument(builder.build(doc));
+ writer.addDocument(config.build(doc));
if (random().nextBoolean()) {
writer.commit();
@@ -157,7 +173,7 @@ public class TestTaxonomyFacetSumValueSo
doc.add(new FacetField("a", "foo3"));
doc.add(new FacetField("b", "bar2"));
doc.add(new FacetField("c", "baz1"));
- writer.addDocument(builder.build(doc));
+ writer.addDocument(config.build(doc));
// NRT open
IndexSearcher searcher = newSearcher(writer.getReader());
@@ -180,10 +196,7 @@ public class TestTaxonomyFacetSumValueSo
assertEquals("b (50.0)\n bar2 (30.0)\n bar1 (20.0)\n", results.get(1).toString());
assertEquals("c (30.0)\n baz1 (30.0)\n", results.get(2).toString());
- searcher.getIndexReader().close();
- taxoReader.close();
- taxoDir.close();
- dir.close();
+ IOUtils.close(searcher.getIndexReader(), taxoReader, dir, taxoDir);
}
public void testWrongIndexFieldName() throws Exception {
@@ -195,16 +208,15 @@ public class TestTaxonomyFacetSumValueSo
// main index:
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
- FacetsConfig config = new FacetsConfig();
+ FacetsConfig config = new FacetsConfig(taxoWriter);
config.setIndexFieldName("a", "$facets2");
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
- DocumentBuilder builder = new DocumentBuilder(taxoWriter, config);
Document doc = new Document();
doc.add(new IntField("num", 10, Field.Store.NO));
doc.add(new FacetField("a", "foo1"));
- writer.addDocument(builder.build(doc));
+ writer.addDocument(config.build(doc));
// NRT open
IndexSearcher searcher = newSearcher(writer.getReader());
@@ -237,10 +249,184 @@ public class TestTaxonomyFacetSumValueSo
// expected
}
- searcher.getIndexReader().close();
- taxoReader.close();
- taxoDir.close();
- dir.close();
+ IOUtils.close(searcher.getIndexReader(), taxoReader, dir, taxoDir);
+ }
+
+ public void testSumScoreAggregator() throws Exception {
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+
+ FacetsConfig config = new FacetsConfig(taxoWriter);
+
+ for(int i = atLeast(30); i > 0; --i) {
+ Document doc = new Document();
+ if (random().nextBoolean()) { // don't match all documents
+ doc.add(new StringField("f", "v", Field.Store.NO));
+ }
+ doc.add(new FacetField("dim", "a"));
+ iw.addDocument(config.build(doc));
+ }
+
+ DirectoryReader r = DirectoryReader.open(iw, true);
+ DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ SimpleFacetsCollector fc = new SimpleFacetsCollector(true);
+ TopScoreDocCollector topDocs = TopScoreDocCollector.create(10, false);
+ ConstantScoreQuery csq = new ConstantScoreQuery(new MatchAllDocsQuery());
+ csq.setBoost(2.0f);
+
+ newSearcher(r).search(csq, MultiCollector.wrap(fc, topDocs));
+
+ Facets facets = new TaxonomyFacetSumValueSource(taxoReader, config, fc, new TaxonomyFacetSumValueSource.ScoreValueSource());
+
+ TopDocs td = topDocs.topDocs();
+ int expected = (int) (td.getMaxScore() * td.totalHits);
+ assertEquals(expected, facets.getSpecificValue("dim", "a").intValue());
+
+ IOUtils.close(iw, taxoWriter, taxoReader, taxoDir, r, indexDir);
+ }
+
+ public void testNoScore() throws Exception {
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ FacetsConfig config = new FacetsConfig(taxoWriter);
+ for (int i = 0; i < 4; i++) {
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("price", (i+1)));
+ doc.add(new FacetField("a", Integer.toString(i % 2)));
+ iw.addDocument(config.build(doc));
+ }
+
+ DirectoryReader r = DirectoryReader.open(iw, true);
+ DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ SimpleFacetsCollector sfc = new SimpleFacetsCollector();
+ newSearcher(r).search(new MatchAllDocsQuery(), sfc);
+ Facets facets = new TaxonomyFacetSumValueSource(taxoReader, config, sfc, new LongFieldSource("price"));
+ assertEquals("a (10.0)\n 1 (6.0)\n 0 (4.0)\n", facets.getTopChildren(10, "a").toString());
+
+ IOUtils.close(taxoWriter, iw, taxoReader, taxoDir, r, indexDir);
+ }
+
+ public void testWithScore() throws Exception {
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+
+ FacetsConfig config = new FacetsConfig(taxoWriter);
+ for (int i = 0; i < 4; i++) {
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("price", (i+1)));
+ doc.add(new FacetField("a", Integer.toString(i % 2)));
+ iw.addDocument(config.build(doc));
+ }
+
+ DirectoryReader r = DirectoryReader.open(iw, true);
+ DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ ValueSource valueSource = new ValueSource() {
+ @Override
+ public FunctionValues getValues(@SuppressWarnings("rawtypes") Map context, AtomicReaderContext readerContext) throws IOException {
+ final Scorer scorer = (Scorer) context.get("scorer");
+ assert scorer != null;
+ return new DoubleDocValues(this) {
+ @Override
+ public double doubleVal(int document) {
+ try {
+ return scorer.score();
+ } catch (IOException exception) {
+ throw new RuntimeException(exception);
+ }
+ }
+ };
+ }
+
+ @Override public boolean equals(Object o) { return o == this; }
+ @Override public int hashCode() { return System.identityHashCode(this); }
+ @Override public String description() { return "score()"; }
+ };
+
+ SimpleFacetsCollector sfc = new SimpleFacetsCollector(true);
+ TopScoreDocCollector tsdc = TopScoreDocCollector.create(10, true);
+ // score documents by their 'price' field - makes asserting the correct counts for the categories easier
+ Query q = new FunctionQuery(new LongFieldSource("price"));
+ newSearcher(r).search(q, MultiCollector.wrap(tsdc, sfc));
+ Facets facets = new TaxonomyFacetSumValueSource(taxoReader, config, sfc, valueSource);
+
+ assertEquals("a (10.0)\n 1 (6.0)\n 0 (4.0)\n", facets.getTopChildren(10, "a").toString());
+
+ IOUtils.close(taxoWriter, iw, taxoReader, taxoDir, r, indexDir);
+ }
+
+ public void testRollupValues() throws Exception {
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ FacetsConfig config = new FacetsConfig(taxoWriter);
+ config.setHierarchical("a", true);
+ //config.setRequireDimCount("a", true);
+
+ for (int i = 0; i < 4; i++) {
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("price", (i+1)));
+ doc.add(new FacetField("a", Integer.toString(i % 2), "1"));
+ iw.addDocument(config.build(doc));
+ }
+
+ DirectoryReader r = DirectoryReader.open(iw, true);
+ DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ ValueSource valueSource = new LongFieldSource("price");
+ SimpleFacetsCollector sfc = new SimpleFacetsCollector();
+ newSearcher(r).search(new MatchAllDocsQuery(), sfc);
+ Facets facets = new TaxonomyFacetSumValueSource(taxoReader, config, sfc, valueSource);
+
+ assertEquals("a (10.0)\n 1 (6.0)\n 0 (4.0)\n", facets.getTopChildren(10, "a").toString());
+
+ IOUtils.close(taxoWriter, iw, taxoReader, taxoDir, r, indexDir);
+ }
+
+ public void testCountAndSumScore() throws Exception {
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ FacetsConfig config = new FacetsConfig(taxoWriter);
+ config.setIndexFieldName("b", "$b");
+
+ for(int i = atLeast(30); i > 0; --i) {
+ Document doc = new Document();
+ doc.add(new StringField("f", "v", Field.Store.NO));
+ doc.add(new FacetField("a", "1"));
+ doc.add(new FacetField("b", "1"));
+ iw.addDocument(config.build(doc));
+ }
+
+ DirectoryReader r = DirectoryReader.open(iw, true);
+ DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ SimpleFacetsCollector sfc = new SimpleFacetsCollector(true);
+ TopScoreDocCollector topDocs = TopScoreDocCollector.create(10, false);
+ newSearcher(r).search(new MatchAllDocsQuery(), MultiCollector.wrap(sfc, topDocs));
+
+ Facets facets1 = getTaxonomyFacetCounts(taxoReader, config, sfc);
+ Facets facets2 = new TaxonomyFacetSumValueSource(new DocValuesOrdinalsReader("$b"), taxoReader, config, sfc, new TaxonomyFacetSumValueSource.ScoreValueSource());
+
+ assertEquals(r.maxDoc(), facets1.getTopChildren(10, "a").value.intValue());
+ double expected = topDocs.topDocs().getMaxScore() * r.numDocs();
+ assertEquals(r.maxDoc(), facets2.getTopChildren(10, "b").value.doubleValue(), 1E-10);
+ IOUtils.close(taxoWriter, iw, taxoReader, taxoDir, r, indexDir);
}
// nocommit in the sparse case test that we are really