You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by sy...@apache.org on 2014/11/14 12:59:23 UTC
[08/26] lucenenet git commit: first commit of facet porting,
failing tests will be fixed in next commits.
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/982eaf60/src/Lucene.Net.Tests/core/Facet/Taxonomy/TestTaxonomyFacetCounts.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/core/Facet/Taxonomy/TestTaxonomyFacetCounts.cs b/src/Lucene.Net.Tests/core/Facet/Taxonomy/TestTaxonomyFacetCounts.cs
new file mode 100644
index 0000000..b4bb012
--- /dev/null
+++ b/src/Lucene.Net.Tests/core/Facet/Taxonomy/TestTaxonomyFacetCounts.cs
@@ -0,0 +1,851 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Lucene.Net.Randomized.Generators;
+using Lucene.Net.Support;
+using NUnit.Framework;
+
+namespace Lucene.Net.Facet.Taxonomy
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using Document = Lucene.Net.Documents.Document;
+ using Field = Lucene.Net.Documents.Field;
+ using StringField = Lucene.Net.Documents.StringField;
+ using DirectoryTaxonomyReader = Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyReader;
+ using DirectoryTaxonomyWriter = Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter;
+ using DirectoryReader = Lucene.Net.Index.DirectoryReader;
+ using IndexWriter = Lucene.Net.Index.IndexWriter;
+ using IndexWriterConfig = Lucene.Net.Index.IndexWriterConfig;
+ using NoMergePolicy = Lucene.Net.Index.NoMergePolicy;
+ using RandomIndexWriter = Lucene.Net.Index.RandomIndexWriter;
+ using Term = Lucene.Net.Index.Term;
+ using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+ using MatchAllDocsQuery = Lucene.Net.Search.MatchAllDocsQuery;
+ using Query = Lucene.Net.Search.Query;
+ using TermQuery = Lucene.Net.Search.TermQuery;
+ using DefaultSimilarity = Lucene.Net.Search.Similarities.DefaultSimilarity;
+ using PerFieldSimilarityWrapper = Lucene.Net.Search.Similarities.PerFieldSimilarityWrapper;
+ using Similarity = Lucene.Net.Search.Similarities.Similarity;
+ using Directory = Lucene.Net.Store.Directory;
+ using IOUtils = Lucene.Net.Util.IOUtils;
+ using TestUtil = Lucene.Net.Util.TestUtil;
+ [TestFixture]
+ public class TestTaxonomyFacetCounts : FacetTestCase
+ {
+
+ [Test]
+ public virtual void TestBasic()
+ {
+ Store.Directory dir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+
+ // Writes facet ords to a separate directory from the
+ // main index:
+ var taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode_e.CREATE);
+
+ FacetsConfig config = new FacetsConfig();
+ config.SetHierarchical("Publish Date", true);
+
+ RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);
+
+ Document doc = new Document();
+ doc.Add(new FacetField("Author", "Bob"));
+ doc.Add(new FacetField("Publish Date", "2010", "10", "15"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ doc = new Document();
+ doc.Add(new FacetField("Author", "Lisa"));
+ doc.Add(new FacetField("Publish Date", "2010", "10", "20"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ doc = new Document();
+ doc.Add(new FacetField("Author", "Lisa"));
+ doc.Add(new FacetField("Publish Date", "2012", "1", "1"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ doc = new Document();
+ doc.Add(new FacetField("Author", "Susan"));
+ doc.Add(new FacetField("Publish Date", "2012", "1", "7"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ doc = new Document();
+ doc.Add(new FacetField("Author", "Frank"));
+ doc.Add(new FacetField("Publish Date", "1999", "5", "5"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ // NRT open
+ IndexSearcher searcher = NewSearcher(writer.Reader);
+
+ // NRT open
+ var taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ // Aggregate the facet counts:
+ FacetsCollector c = new FacetsCollector();
+
+ // MatchAllDocsQuery is for "browsing" (counts facets
+ // for all non-deleted docs in the index); normally
+ // you'd use a "normal" query, and use MultiCollector to
+ // wrap collecting the "normal" hits and also facets:
+ searcher.Search(new MatchAllDocsQuery(), c);
+
+ Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, c);
+
+ // Retrieve & verify results:
+ Assert.AreEqual("dim=Publish Date path=[] value=5 childCount=3\n 2010 (2)\n 2012 (2)\n 1999 (1)\n", facets.GetTopChildren(10, "Publish Date").ToString());
+ Assert.AreEqual("dim=Author path=[] value=5 childCount=4\n Lisa (2)\n Bob (1)\n Susan (1)\n Frank (1)\n", facets.GetTopChildren(10, "Author").ToString());
+
+ // Now user drills down on Publish Date/2010:
+ DrillDownQuery q2 = new DrillDownQuery(config);
+ q2.Add("Publish Date", "2010");
+ c = new FacetsCollector();
+ searcher.Search(q2, c);
+ facets = new FastTaxonomyFacetCounts(taxoReader, config, c);
+ Assert.AreEqual("dim=Author path=[] value=2 childCount=2\n Bob (1)\n Lisa (1)\n", facets.GetTopChildren(10, "Author").ToString());
+
+ Assert.AreEqual(1, facets.GetSpecificValue("Author", "Lisa"));
+
+ Assert.Null(facets.GetTopChildren(10, "Non exitent dim"));
+
+ // Smoke test PrintTaxonomyStats:
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintTaxonomyStats.PrintStats(taxoReader, Console.Out, true);
+ string result = bos.ToString();
+ Assert.True(result.IndexOf("/Author: 4 immediate children; 5 total categories", StringComparison.Ordinal) != -1);
+ Assert.True(result.IndexOf("/Publish Date: 3 immediate children; 12 total categories", StringComparison.Ordinal) != -1);
+ // Make sure at least a few nodes of the tree came out:
+ Assert.True(result.IndexOf(" /1999", StringComparison.Ordinal) != -1);
+ Assert.True(result.IndexOf(" /2012", StringComparison.Ordinal) != -1);
+ Assert.True(result.IndexOf(" /20", StringComparison.Ordinal) != -1);
+
+ IOUtils.Close(writer, taxoWriter, searcher.IndexReader, taxoReader, taxoDir, dir);
+ }
+
+ // LUCENE-5333
+ [Test]
+ public virtual void TestSparseFacets()
+ {
+ Store.Directory dir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+
+ // Writes facet ords to a separate directory from the
+ // main index:
+ var taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode_e.CREATE);
+
+ RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);
+ FacetsConfig config = new FacetsConfig();
+
+ Document doc = new Document();
+ doc.Add(new FacetField("a", "foo1"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ if (Random().NextBoolean())
+ {
+ writer.Commit();
+ }
+
+ doc = new Document();
+ doc.Add(new FacetField("a", "foo2"));
+ doc.Add(new FacetField("b", "bar1"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ if (Random().NextBoolean())
+ {
+ writer.Commit();
+ }
+
+ doc = new Document();
+ doc.Add(new FacetField("a", "foo3"));
+ doc.Add(new FacetField("b", "bar2"));
+ doc.Add(new FacetField("c", "baz1"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ // NRT open
+ IndexSearcher searcher = NewSearcher(writer.Reader);
+
+ // NRT open
+ var taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ FacetsCollector c = new FacetsCollector();
+ searcher.Search(new MatchAllDocsQuery(), c);
+
+ Facets facets = GetTaxonomyFacetCounts(taxoReader, new FacetsConfig(), c);
+
+ // Ask for top 10 labels for any dims that have counts:
+ IList<FacetResult> results = facets.GetAllDims(10);
+
+ Assert.AreEqual(3, results.Count);
+ Assert.AreEqual("dim=a path=[] value=3 childCount=3\n foo1 (1)\n foo2 (1)\n foo3 (1)\n", results[0].ToString());
+ Assert.AreEqual("dim=b path=[] value=2 childCount=2\n bar1 (1)\n bar2 (1)\n", results[1].ToString());
+ Assert.AreEqual("dim=c path=[] value=1 childCount=1\n baz1 (1)\n", results[2].ToString());
+
+ IOUtils.Close(writer, taxoWriter, searcher.IndexReader, taxoReader, taxoDir, dir);
+ }
+
+ [Test]
+ public virtual void TestWrongIndexFieldName()
+ {
+ Store.Directory dir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+
+ // Writes facet ords to a separate directory from the
+ // main index:
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode_e.CREATE);
+
+ FacetsConfig config = new FacetsConfig();
+ config.SetIndexFieldName("a", "$facets2");
+ RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);
+
+ Document doc = new Document();
+ doc.Add(new FacetField("a", "foo1"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ // NRT open
+ IndexSearcher searcher = NewSearcher(writer.Reader);
+
+ // NRT open
+ var taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ FacetsCollector c = new FacetsCollector();
+ searcher.Search(new MatchAllDocsQuery(), c);
+
+ // Uses default $facets field:
+ Facets facets;
+ if (Random().NextBoolean())
+ {
+ facets = new FastTaxonomyFacetCounts(taxoReader, config, c);
+ }
+ else
+ {
+ OrdinalsReader ordsReader = new DocValuesOrdinalsReader();
+ if (Random().NextBoolean())
+ {
+ ordsReader = new CachedOrdinalsReader(ordsReader);
+ }
+ facets = new TaxonomyFacetCounts(ordsReader, taxoReader, config, c);
+ }
+
+ // Ask for top 10 labels for any dims that have counts:
+ IList<FacetResult> results = facets.GetAllDims(10);
+ Assert.True(results.Count == 0);
+
+ try
+ {
+ facets.GetSpecificValue("a");
+ Fail("should have hit exc");
+ }
+ catch (System.ArgumentException)
+ {
+ // expected
+ }
+
+ try
+ {
+ facets.GetTopChildren(10, "a");
+ Fail("should have hit exc");
+ }
+ catch (System.ArgumentException)
+ {
+ // expected
+ }
+
+ IOUtils.Close(writer, taxoWriter, searcher.IndexReader, taxoReader, taxoDir, dir);
+ }
+
+ [Test]
+ public virtual void TestReallyNoNormsForDrillDown()
+ {
+ Store.Directory dir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+ IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ iwc.SetSimilarity(new PerFieldSimilarityWrapperAnonymousInnerClassHelper(this));
+ TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode_e.CREATE);
+ RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, iwc);
+ FacetsConfig config = new FacetsConfig();
+
+ Document doc = new Document();
+ doc.Add(NewTextField("field", "text", Field.Store.NO));
+ doc.Add(new FacetField("a", "path"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+ IOUtils.Close(writer, taxoWriter, dir, taxoDir);
+ }
+
+ private class PerFieldSimilarityWrapperAnonymousInnerClassHelper : PerFieldSimilarityWrapper
+ {
+ private readonly TestTaxonomyFacetCounts outerInstance;
+
+ public PerFieldSimilarityWrapperAnonymousInnerClassHelper(TestTaxonomyFacetCounts outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ sim = new DefaultSimilarity();
+ }
+
+ internal readonly Similarity sim;
+
+ public override Similarity Get(string name)
+ {
+ Assert.AreEqual("field", name);
+ return sim;
+ }
+ }
+
+ [Test]
+ public virtual void TestMultiValuedHierarchy()
+ {
+ Store.Directory dir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode_e.CREATE);
+ FacetsConfig config = new FacetsConfig();
+ config.SetHierarchical("a", true);
+ config.SetMultiValued("a", true);
+ RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);
+
+ Document doc = new Document();
+ doc.Add(NewTextField("field", "text", Field.Store.NO));
+ doc.Add(new FacetField("a", "path", "x"));
+ doc.Add(new FacetField("a", "path", "y"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ // NRT open
+ IndexSearcher searcher = NewSearcher(writer.Reader);
+
+ // NRT open
+ var taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ // Aggregate the facet counts:
+ FacetsCollector c = new FacetsCollector();
+
+ // MatchAllDocsQuery is for "browsing" (counts facets
+ // for all non-deleted docs in the index); normally
+ // you'd use a "normal" query, and use MultiCollector to
+ // wrap collecting the "normal" hits and also facets:
+ searcher.Search(new MatchAllDocsQuery(), c);
+ Facets facets = GetTaxonomyFacetCounts(taxoReader, config, c);
+
+ try
+ {
+ facets.GetSpecificValue("a");
+ Fail("didn't hit expected exception");
+ }
+ catch (System.ArgumentException)
+ {
+ // expected
+ }
+
+ FacetResult result = facets.GetTopChildren(10, "a");
+ Assert.AreEqual(1, result.labelValues.Length);
+ Assert.AreEqual(1, (int)result.labelValues[0].value);
+
+ IOUtils.Close(writer, taxoWriter, searcher.IndexReader, taxoReader, dir, taxoDir);
+ }
+
+ [Test]
+ public virtual void TestLabelWithDelimiter()
+ {
+ Store.Directory dir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);
+ var taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode_e.CREATE);
+
+ FacetsConfig config = new FacetsConfig();
+ config.SetMultiValued("dim", true);
+
+ Document doc = new Document();
+ doc.Add(NewTextField("field", "text", Field.Store.NO));
+ doc.Add(new FacetField("dim", "test\u001Fone"));
+ doc.Add(new FacetField("dim", "test\u001Etwo"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ // NRT open
+ IndexSearcher searcher = NewSearcher(writer.Reader);
+
+ // NRT open
+ var taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ FacetsCollector c = new FacetsCollector();
+ searcher.Search(new MatchAllDocsQuery(), c);
+
+ Facets facets = GetTaxonomyFacetCounts(taxoReader, config, c);
+ Assert.AreEqual(1, facets.GetSpecificValue("dim", "test\u001Fone"));
+ Assert.AreEqual(1, facets.GetSpecificValue("dim", "test\u001Etwo"));
+
+ FacetResult result = facets.GetTopChildren(10, "dim");
+ Assert.AreEqual("dim=dim path=[] value=-1 childCount=2\n test\u001Fone (1)\n test\u001Etwo (1)\n", result.ToString());
+ IOUtils.Close(writer, taxoWriter, searcher.IndexReader, taxoReader, dir, taxoDir);
+ }
+
+ [Test]
+ public virtual void TestRequireDimCount()
+ {
+ Store.Directory dir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode_e.CREATE);
+
+ FacetsConfig config = new FacetsConfig();
+ config.SetRequireDimCount("dim", true);
+
+ config.SetMultiValued("dim2", true);
+ config.SetRequireDimCount("dim2", true);
+
+ config.SetMultiValued("dim3", true);
+ config.SetHierarchical("dim3", true);
+ config.SetRequireDimCount("dim3", true);
+
+ Document doc = new Document();
+ doc.Add(NewTextField("field", "text", Field.Store.NO));
+ doc.Add(new FacetField("dim", "a"));
+ doc.Add(new FacetField("dim2", "a"));
+ doc.Add(new FacetField("dim2", "b"));
+ doc.Add(new FacetField("dim3", "a", "b"));
+ doc.Add(new FacetField("dim3", "a", "c"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ // NRT open
+ IndexSearcher searcher = NewSearcher(writer.Reader);
+
+ // NRT open
+ var taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ FacetsCollector c = new FacetsCollector();
+ searcher.Search(new MatchAllDocsQuery(), c);
+
+ Facets facets = GetTaxonomyFacetCounts(taxoReader, config, c);
+ Assert.AreEqual(1, facets.GetTopChildren(10, "dim").value);
+ Assert.AreEqual(1, facets.GetTopChildren(10, "dim2").value);
+ Assert.AreEqual(1, facets.GetTopChildren(10, "dim3").value);
+ try
+ {
+ Assert.AreEqual(1, facets.GetSpecificValue("dim"));
+ Fail("didn't hit expected exception");
+ }
+ catch (System.ArgumentException)
+ {
+ // expected
+ }
+ Assert.AreEqual(1, facets.GetSpecificValue("dim2"));
+ Assert.AreEqual(1, facets.GetSpecificValue("dim3"));
+ IOUtils.Close(writer, taxoWriter, searcher.IndexReader, taxoReader, dir, taxoDir);
+ }
+
+ // LUCENE-4583: make sure if we require > 32 KB for one
+ // document, we don't hit exc when using Facet42DocValuesFormat
+ [Test]
+ public virtual void TestManyFacetsInOneDocument()
+ {
+ AssumeTrue("default Codec doesn't support huge BinaryDocValues", TestUtil.FieldSupportsHugeBinaryDocValues(FacetsConfig.DEFAULT_INDEX_FIELD_NAME));
+ Store.Directory dir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+ IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, iwc);
+ var taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode_e.CREATE);
+
+ FacetsConfig config = new FacetsConfig();
+ config.SetMultiValued("dim", true);
+
+ int numLabels = TestUtil.NextInt(Random(), 40000, 100000);
+
+ Document doc = new Document();
+ doc.Add(NewTextField("field", "text", Field.Store.NO));
+ for (int i = 0; i < numLabels; i++)
+ {
+ doc.Add(new FacetField("dim", "" + i));
+ }
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ // NRT open
+ IndexSearcher searcher = NewSearcher(writer.Reader);
+
+ // NRT open
+ var taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ // Aggregate the facet counts:
+ FacetsCollector c = new FacetsCollector();
+
+ // MatchAllDocsQuery is for "browsing" (counts facets
+ // for all non-deleted docs in the index); normally
+ // you'd use a "normal" query, and use MultiCollector to
+ // wrap collecting the "normal" hits and also facets:
+ searcher.Search(new MatchAllDocsQuery(), c);
+ Facets facets = GetTaxonomyFacetCounts(taxoReader, config, c);
+
+ FacetResult result = facets.GetTopChildren(int.MaxValue, "dim");
+ Assert.AreEqual(numLabels, result.labelValues.Length);
+ var allLabels = new HashSet<string>();
+ foreach (LabelAndValue labelValue in result.labelValues)
+ {
+ allLabels.Add(labelValue.label);
+ Assert.AreEqual(1, (int)labelValue.value);
+ }
+ Assert.AreEqual(numLabels, allLabels.Count);
+
+ IOUtils.Close(searcher.IndexReader, taxoWriter, writer, taxoReader, dir, taxoDir);
+ }
+
+ // Make sure we catch when app didn't declare field as
+ // hierarchical but it was:
+ [Test]
+ public virtual void TestDetectHierarchicalField()
+ {
+ Store.Directory dir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+ var taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode_e.CREATE);
+ var writer = new RandomIndexWriter(Random(), dir);
+ FacetsConfig config = new FacetsConfig();
+
+ Document doc = new Document();
+ doc.Add(NewTextField("field", "text", Field.Store.NO));
+ doc.Add(new FacetField("a", "path", "other"));
+ try
+ {
+ config.Build(taxoWriter, doc);
+ Fail("did not hit expected exception");
+ }
+ catch (System.ArgumentException)
+ {
+ // expected
+ }
+ IOUtils.Close(writer, taxoWriter, dir, taxoDir);
+ }
+
+ // Make sure we catch when app didn't declare field as
+ // multi-valued but it was:
+ [Test]
+ public virtual void TestDetectMultiValuedField()
+ {
+ Store.Directory dir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+ TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode_e.CREATE);
+ RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);
+ FacetsConfig config = new FacetsConfig();
+
+ Document doc = new Document();
+ doc.Add(NewTextField("field", "text", Field.Store.NO));
+ doc.Add(new FacetField("a", "path"));
+ doc.Add(new FacetField("a", "path2"));
+ try
+ {
+ config.Build(taxoWriter, doc);
+ Fail("did not hit expected exception");
+ }
+ catch (System.ArgumentException)
+ {
+ // expected
+ }
+ IOUtils.Close(writer, taxoWriter, dir, taxoDir);
+ }
+
+ [Test]
+ public virtual void TestSeparateIndexedFields()
+ {
+ Store.Directory indexDir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+
+ var taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ FacetsConfig config = new FacetsConfig();
+ config.SetIndexFieldName("b", "$b");
+
+ for (int i = AtLeast(30); i > 0; --i)
+ {
+ Document doc = new Document();
+ doc.Add(new StringField("f", "v", Field.Store.NO));
+ doc.Add(new FacetField("a", "1"));
+ doc.Add(new FacetField("b", "1"));
+ iw.AddDocument(config.Build(taxoWriter, doc));
+ }
+
+ DirectoryReader r = DirectoryReader.Open(iw, true);
+ var taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ FacetsCollector sfc = new FacetsCollector();
+ NewSearcher(r).Search(new MatchAllDocsQuery(), sfc);
+ Facets facets1 = GetTaxonomyFacetCounts(taxoReader, config, sfc);
+ Facets facets2 = GetTaxonomyFacetCounts(taxoReader, config, sfc, "$b");
+ Assert.AreEqual(r.MaxDoc, (int)facets1.GetTopChildren(10, "a").value);
+ Assert.AreEqual(r.MaxDoc, (int)facets2.GetTopChildren(10, "b").value);
+ IOUtils.Close(taxoWriter, iw, taxoReader, taxoDir, r, indexDir);
+ }
+
+ [Test]
+ public virtual void TestCountRoot()
+ {
+ // LUCENE-4882: FacetsAccumulator threw NPE if a FacetRequest was defined on CP.EMPTY
+ Store.Directory indexDir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+
+ var taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ FacetsConfig config = new FacetsConfig();
+ for (int i = AtLeast(30); i > 0; --i)
+ {
+ Document doc = new Document();
+ doc.Add(new FacetField("a", "1"));
+ doc.Add(new FacetField("b", "1"));
+ iw.AddDocument(config.Build(taxoWriter, doc));
+ }
+
+ DirectoryReader r = DirectoryReader.Open(iw, true);
+ DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ FacetsCollector sfc = new FacetsCollector();
+ NewSearcher(r).Search(new MatchAllDocsQuery(), sfc);
+ Facets facets = GetTaxonomyFacetCounts(taxoReader, config, sfc);
+ foreach (FacetResult result in facets.GetAllDims(10))
+ {
+ Assert.AreEqual(r.NumDocs, (int)result.value);
+ }
+
+ IOUtils.Close(taxoWriter, iw, taxoReader, taxoDir, r, indexDir);
+ }
+
+ [Test]
+ public virtual void TestGetFacetResultsTwice()
+ {
+ // LUCENE-4893: counts were multiplied as many times as getFacetResults was called.
+ Store.Directory indexDir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+
+ var taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ FacetsConfig config = new FacetsConfig();
+
+ Document doc = new Document();
+ doc.Add(new FacetField("a", "1"));
+ doc.Add(new FacetField("b", "1"));
+ iw.AddDocument(config.Build(taxoWriter, doc));
+
+ DirectoryReader r = DirectoryReader.Open(iw, true);
+ var taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ FacetsCollector sfc = new FacetsCollector();
+ NewSearcher(r).Search(new MatchAllDocsQuery(), sfc);
+
+ Facets facets = GetTaxonomyFacetCounts(taxoReader, config, sfc);
+ IList<FacetResult> res1 = facets.GetAllDims(10);
+ IList<FacetResult> res2 = facets.GetAllDims(10);
+ Assert.AreEqual(res1, res2, "calling getFacetResults twice should return the .equals()=true result");
+
+ IOUtils.Close(taxoWriter, iw, taxoReader, taxoDir, r, indexDir);
+ }
+
+ [Test]
+ public virtual void TestChildCount()
+ {
+ // LUCENE-4885: FacetResult.numValidDescendants was not set properly by FacetsAccumulator
+ var indexDir = NewDirectory();
+ var taxoDir = NewDirectory();
+
+ var taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ FacetsConfig config = new FacetsConfig();
+ for (int i = 0; i < 10; i++)
+ {
+ Document doc = new Document();
+ doc.Add(new FacetField("a", Convert.ToString(i)));
+ iw.AddDocument(config.Build(taxoWriter, doc));
+ }
+
+ DirectoryReader r = DirectoryReader.Open(iw, true);
+ DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ FacetsCollector sfc = new FacetsCollector();
+ NewSearcher(r).Search(new MatchAllDocsQuery(), sfc);
+ Facets facets = GetTaxonomyFacetCounts(taxoReader, config, sfc);
+
+ Assert.AreEqual(10, facets.GetTopChildren(2, "a").childCount);
+
+ IOUtils.Close(taxoWriter, iw, taxoReader, taxoDir, r, indexDir);
+ }
+
+ private void indexTwoDocs(TaxonomyWriter taxoWriter, IndexWriter indexWriter, FacetsConfig config, bool withContent)
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ Document doc = new Document();
+ if (withContent)
+ {
+ doc.Add(new StringField("f", "a", Field.Store.NO));
+ }
+ if (config != null)
+ {
+ doc.Add(new FacetField("A", Convert.ToString(i)));
+ indexWriter.AddDocument(config.Build(taxoWriter, doc));
+ }
+ else
+ {
+ indexWriter.AddDocument(doc);
+ }
+ }
+
+ indexWriter.Commit();
+ }
+
+ [Test]
+ public virtual void TestSegmentsWithoutCategoriesOrResults()
+ {
+ // tests the accumulator when there are segments with no results
+ var indexDir = NewDirectory();
+ var taxoDir = NewDirectory();
+
+ IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ //iwc.MergePolicy = NoMergePolicy.INSTANCE; // prevent merges
+ IndexWriter indexWriter = new IndexWriter(indexDir, iwc);
+
+ var taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+ FacetsConfig config = new FacetsConfig();
+ indexTwoDocs(taxoWriter, indexWriter, config, false); // 1st segment, no content, with categories
+ indexTwoDocs(taxoWriter, indexWriter, null, true); // 2nd segment, with content, no categories
+ indexTwoDocs(taxoWriter, indexWriter, config, true); // 3rd segment ok
+ indexTwoDocs(taxoWriter, indexWriter, null, false); // 4th segment, no content, or categories
+ indexTwoDocs(taxoWriter, indexWriter, null, true); // 5th segment, with content, no categories
+ indexTwoDocs(taxoWriter, indexWriter, config, true); // 6th segment, with content, with categories
+ indexTwoDocs(taxoWriter, indexWriter, null, true); // 7th segment, with content, no categories
+ IOUtils.Close(indexWriter, taxoWriter);
+
+ DirectoryReader indexReader = DirectoryReader.Open(indexDir);
+ var taxoReader = new DirectoryTaxonomyReader(taxoDir);
+ IndexSearcher indexSearcher = NewSearcher(indexReader);
+
+ // search for "f:a", only segments 1 and 3 should match results
+ Query q = new TermQuery(new Term("f", "a"));
+ FacetsCollector sfc = new FacetsCollector();
+ indexSearcher.Search(q, sfc);
+ Facets facets = GetTaxonomyFacetCounts(taxoReader, config, sfc);
+ FacetResult result = facets.GetTopChildren(10, "A");
+ Assert.AreEqual(2, result.labelValues.Length, "wrong number of children");
+ foreach (LabelAndValue labelValue in result.labelValues)
+ {
+ Assert.AreEqual(2, (int)labelValue.value, "wrong weight for child " + labelValue.label);
+ }
+
+ IOUtils.Close(indexReader, taxoReader, indexDir, taxoDir);
+ }
+
+ [Test]
+ public virtual void TestRandom()
+ {
+ string[] tokens = GetRandomTokens(10);
+ Store.Directory indexDir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+
+ RandomIndexWriter w = new RandomIndexWriter(Random(), indexDir);
+ var tw = new DirectoryTaxonomyWriter(taxoDir);
+ FacetsConfig config = new FacetsConfig();
+ int numDocs = AtLeast(1000);
+ int numDims = TestUtil.NextInt(Random(), 1, 7);
+ IList<TestDoc> testDocs = GetRandomDocs(tokens, numDocs, numDims);
+ foreach (TestDoc testDoc in testDocs)
+ {
+ Document doc = new Document();
+ doc.Add(NewStringField("content", testDoc.content, Field.Store.NO));
+ for (int j = 0; j < numDims; j++)
+ {
+ if (testDoc.dims[j] != null)
+ {
+ doc.Add(new FacetField("dim" + j, testDoc.dims[j]));
+ }
+ }
+ w.AddDocument(config.Build(tw, doc));
+ }
+
+ // NRT open
+ IndexSearcher searcher = NewSearcher(w.Reader);
+
+ // NRT open
+ var tr = new DirectoryTaxonomyReader(tw);
+
+ int iters = AtLeast(100);
+ for (int iter = 0; iter < iters; iter++)
+ {
+ string searchToken = tokens[Random().Next(tokens.Length)];
+ if (VERBOSE)
+ {
+ Console.WriteLine("\nTEST: iter content=" + searchToken);
+ }
+ FacetsCollector fc = new FacetsCollector();
+ FacetsCollector.Search(searcher, new TermQuery(new Term("content", searchToken)), 10, fc);
+ Facets facets = GetTaxonomyFacetCounts(tr, config, fc);
+
+ // Slow, yet hopefully bug-free, faceting:
+ var expectedCounts = new List<Dictionary<string, int?>>();
+ for (int i = 0; i < numDims; i++)
+ {
+ expectedCounts[i] = new Dictionary<string, int?>();
+ }
+
+ foreach (TestDoc doc in testDocs)
+ {
+ if (doc.content.Equals(searchToken))
+ {
+ for (int j = 0; j < numDims; j++)
+ {
+ if (doc.dims[j] != null)
+ {
+ int? v = expectedCounts[j][doc.dims[j]];
+ if (v == null)
+ {
+ expectedCounts[j][doc.dims[j]] = 1;
+ }
+ else
+ {
+ expectedCounts[j][doc.dims[j]] = (int)v + 1;
+ }
+ }
+ }
+ }
+ }
+
+ IList<FacetResult> expected = new List<FacetResult>();
+ for (int i = 0; i < numDims; i++)
+ {
+ IList<LabelAndValue> labelValues = new List<LabelAndValue>();
+ int totCount = 0;
+ foreach (KeyValuePair<string, int?> ent in expectedCounts[i])
+ {
+ labelValues.Add(new LabelAndValue(ent.Key, ent.Value.Value));
+ totCount += ent.Value.Value;
+ }
+ SortLabelValues(labelValues);
+ if (totCount > 0)
+ {
+ expected.Add(new FacetResult("dim" + i, new string[0], totCount, labelValues.ToArray(), labelValues.Count));
+ }
+ }
+
+ // Sort by highest value, tie break by value:
+ SortFacetResults(expected);
+
+ IList<FacetResult> actual = facets.GetAllDims(10);
+
+ // Messy: fixup ties
+ SortTies(actual);
+
+ Assert.AreEqual(expected, actual);
+ }
+
+ IOUtils.Close(w, tw, searcher.IndexReader, tr, indexDir, taxoDir);
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/982eaf60/src/Lucene.Net.Tests/core/Facet/Taxonomy/TestTaxonomyFacetCounts2.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/core/Facet/Taxonomy/TestTaxonomyFacetCounts2.cs b/src/Lucene.Net.Tests/core/Facet/Taxonomy/TestTaxonomyFacetCounts2.cs
new file mode 100644
index 0000000..f461dde
--- /dev/null
+++ b/src/Lucene.Net.Tests/core/Facet/Taxonomy/TestTaxonomyFacetCounts2.cs
@@ -0,0 +1,403 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Lucene.Net.Randomized.Generators;
+using Lucene.Net.Support;
+using NUnit.Framework;
+
+namespace Lucene.Net.Facet.Taxonomy
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using Document = Lucene.Net.Documents.Document;
+ using Store = Lucene.Net.Documents.Field.Store;
+ using StringField = Lucene.Net.Documents.StringField;
+ using DirectoryTaxonomyReader = Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyReader;
+ using DirectoryTaxonomyWriter = Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter;
+ using DirectoryReader = Lucene.Net.Index.DirectoryReader;
+ using IndexWriter = Lucene.Net.Index.IndexWriter;
+ using IndexWriterConfig = Lucene.Net.Index.IndexWriterConfig;
+ using NoMergePolicy = Lucene.Net.Index.NoMergePolicy;
+ using Term = Lucene.Net.Index.Term;
+ using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+ using MatchAllDocsQuery = Lucene.Net.Search.MatchAllDocsQuery;
+ using TermQuery = Lucene.Net.Search.TermQuery;
+ using Directory = Lucene.Net.Store.Directory;
+ using IOUtils = Lucene.Net.Util.IOUtils;
+ [TestFixture]
+ public class TestTaxonomyFacetCounts2 : FacetTestCase
+ {
+
+ private static readonly Term A = new Term("f", "a");
+ private const string CP_A = "A", CP_B = "B";
+ private const string CP_C = "C", CP_D = "D"; // indexed w/ NO_PARENTS
+ private const int NUM_CHILDREN_CP_A = 5, NUM_CHILDREN_CP_B = 3;
+ private const int NUM_CHILDREN_CP_C = 5, NUM_CHILDREN_CP_D = 5;
+ private static readonly FacetField[] CATEGORIES_A, CATEGORIES_B;
+ private static readonly FacetField[] CATEGORIES_C, CATEGORIES_D;
+ static TestTaxonomyFacetCounts2()
+ {
+ CATEGORIES_A = new FacetField[NUM_CHILDREN_CP_A];
+ for (int i = 0; i < NUM_CHILDREN_CP_A; i++)
+ {
+ CATEGORIES_A[i] = new FacetField(CP_A, Convert.ToString(i));
+ }
+ CATEGORIES_B = new FacetField[NUM_CHILDREN_CP_B];
+ for (int i = 0; i < NUM_CHILDREN_CP_B; i++)
+ {
+ CATEGORIES_B[i] = new FacetField(CP_B, Convert.ToString(i));
+ }
+
+ // NO_PARENTS categories
+ CATEGORIES_C = new FacetField[NUM_CHILDREN_CP_C];
+ for (int i = 0; i < NUM_CHILDREN_CP_C; i++)
+ {
+ CATEGORIES_C[i] = new FacetField(CP_C, Convert.ToString(i));
+ }
+
+ // Multi-level categories
+ CATEGORIES_D = new FacetField[NUM_CHILDREN_CP_D];
+ for (int i = 0; i < NUM_CHILDREN_CP_D; i++)
+ {
+ string val = Convert.ToString(i);
+ CATEGORIES_D[i] = new FacetField(CP_D, val, val + val); // e.g. D/1/11, D/2/22...
+ }
+ }
+
+ private static Net.Store.Directory indexDir, taxoDir;
+ private static IDictionary<string, int?> allExpectedCounts, termExpectedCounts;
+
+ [TestFixtureTearDown]
+ public static void AfterClassCountingFacetsAggregatorTest()
+ {
+ IOUtils.Close(indexDir, taxoDir);
+ }
+
+ private static IList<FacetField> RandomCategories(Random random)
+ {
+ // add random categories from the two dimensions, ensuring that the same
+ // category is not added twice.
+ int numFacetsA = random.Next(3) + 1; // 1-3
+ int numFacetsB = random.Next(2) + 1; // 1-2
+ List<FacetField> categories_a = new List<FacetField>();
+ categories_a.AddRange(Arrays.AsList(CATEGORIES_A));
+ List<FacetField> categories_b = new List<FacetField>();
+ categories_b.AddRange(Arrays.AsList(CATEGORIES_B));
+ categories_a = CollectionsHelper.Shuffle(categories_a).ToList();
+ categories_b = CollectionsHelper.Shuffle(categories_b).ToList();
+
+ List<FacetField> categories = new List<FacetField>();
+ categories.AddRange(categories_a.SubList(0, numFacetsA));
+ categories.AddRange(categories_b.SubList(0, numFacetsB));
+
+ // add the NO_PARENT categories
+ categories.Add(CATEGORIES_C[Random().Next(NUM_CHILDREN_CP_C)]);
+ categories.Add(CATEGORIES_D[Random().Next(NUM_CHILDREN_CP_D)]);
+
+ return categories;
+ }
+
+ private static void AddField(Document doc)
+ {
+ doc.Add(new StringField(A.Field(), A.Text(), Store.NO));
+ }
+
+ private static void AddFacets(Document doc, FacetsConfig config, bool updateTermExpectedCounts)
+ {
+ IList<FacetField> docCategories = RandomCategories(Random());
+ foreach (FacetField ff in docCategories)
+ {
+ doc.Add(ff);
+ string cp = ff.dim + "/" + ff.path[0];
+ allExpectedCounts[cp] = allExpectedCounts[cp] + 1;
+ if (updateTermExpectedCounts)
+ {
+ termExpectedCounts[cp] = termExpectedCounts[cp] + 1;
+ }
+ }
+ // add 1 to each NO_PARENTS dimension
+ allExpectedCounts[CP_B] = allExpectedCounts[CP_B] + 1;
+ allExpectedCounts[CP_C] = allExpectedCounts[CP_C] + 1;
+ allExpectedCounts[CP_D] = allExpectedCounts[CP_D] + 1;
+ if (updateTermExpectedCounts)
+ {
+ termExpectedCounts[CP_B] = termExpectedCounts[CP_B] + 1;
+ termExpectedCounts[CP_C] = termExpectedCounts[CP_C] + 1;
+ termExpectedCounts[CP_D] = termExpectedCounts[CP_D] + 1;
+ }
+ }
+
+ private static FacetsConfig Config
+ {
+ get
+ {
+ FacetsConfig config = new FacetsConfig();
+ config.SetMultiValued("A", true);
+ config.SetMultiValued("B", true);
+ config.SetRequireDimCount("B", true);
+ config.SetHierarchical("D", true);
+ return config;
+ }
+ }
+
+ private static void IndexDocsNoFacets(IndexWriter indexWriter)
+ {
+ int numDocs = AtLeast(2);
+ for (int i = 0; i < numDocs; i++)
+ {
+ Document doc = new Document();
+ AddField(doc);
+ indexWriter.AddDocument(doc);
+ }
+ indexWriter.Commit(); // flush a segment
+ }
+
+ private static void IndexDocsWithFacetsNoTerms(IndexWriter indexWriter, TaxonomyWriter taxoWriter, IDictionary<string, int?> expectedCounts)
+ {
+ Random random = Random();
+ int numDocs = AtLeast(random, 2);
+ FacetsConfig config = Config;
+ for (int i = 0; i < numDocs; i++)
+ {
+ Document doc = new Document();
+ AddFacets(doc, config, false);
+ indexWriter.AddDocument(config.Build(taxoWriter, doc));
+ }
+ indexWriter.Commit(); // flush a segment
+ }
+
+ private static void IndexDocsWithFacetsAndTerms(IndexWriter indexWriter, TaxonomyWriter taxoWriter, IDictionary<string, int?> expectedCounts)
+ {
+ Random random = Random();
+ int numDocs = AtLeast(random, 2);
+ FacetsConfig config = Config;
+ for (int i = 0; i < numDocs; i++)
+ {
+ Document doc = new Document();
+ AddFacets(doc, config, true);
+ AddField(doc);
+ indexWriter.AddDocument(config.Build(taxoWriter, doc));
+ }
+ indexWriter.Commit(); // flush a segment
+ }
+
+ private static void IndexDocsWithFacetsAndSomeTerms(IndexWriter indexWriter, TaxonomyWriter taxoWriter, IDictionary<string, int?> expectedCounts)
+ {
+ Random random = Random();
+ int numDocs = AtLeast(random, 2);
+ FacetsConfig config = Config;
+ for (int i = 0; i < numDocs; i++)
+ {
+ Document doc = new Document();
+ bool hasContent = random.NextBoolean();
+ if (hasContent)
+ {
+ AddField(doc);
+ }
+ AddFacets(doc, config, hasContent);
+ indexWriter.AddDocument(config.Build(taxoWriter, doc));
+ }
+ indexWriter.Commit(); // flush a segment
+ }
+
+ // initialize expectedCounts w/ 0 for all categories
+ private static IDictionary<string, int?> newCounts()
+ {
+ IDictionary<string, int?> counts = new Dictionary<string, int?>();
+ counts[CP_A] = 0;
+ counts[CP_B] = 0;
+ counts[CP_C] = 0;
+ counts[CP_D] = 0;
+ foreach (FacetField ff in CATEGORIES_A)
+ {
+ counts[ff.dim + "/" + ff.path[0]] = 0;
+ }
+ foreach (FacetField ff in CATEGORIES_B)
+ {
+ counts[ff.dim + "/" + ff.path[0]] = 0;
+ }
+ foreach (FacetField ff in CATEGORIES_C)
+ {
+ counts[ff.dim + "/" + ff.path[0]] = 0;
+ }
+ foreach (FacetField ff in CATEGORIES_D)
+ {
+ counts[ff.dim + "/" + ff.path[0]] = 0;
+ }
+ return counts;
+ }
+
+ [TestFixtureSetUp]
+ public static void BeforeClassCountingFacetsAggregatorTest()
+ {
+ indexDir = NewDirectory();
+ taxoDir = NewDirectory();
+
+ // create an index which has:
+ // 1. Segment with no categories, but matching results
+ // 2. Segment w/ categories, but no results
+ // 3. Segment w/ categories and results
+ // 4. Segment w/ categories, but only some results
+
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ //conf.MergePolicy = NoMergePolicy.INSTANCE; // prevent merges, so we can control the index segments
+ IndexWriter indexWriter = new IndexWriter(indexDir, conf);
+ TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+
+ allExpectedCounts = newCounts();
+ termExpectedCounts = newCounts();
+
+ // segment w/ no categories
+ IndexDocsNoFacets(indexWriter);
+
+ // segment w/ categories, no content
+ IndexDocsWithFacetsNoTerms(indexWriter, taxoWriter, allExpectedCounts);
+
+ // segment w/ categories and content
+ IndexDocsWithFacetsAndTerms(indexWriter, taxoWriter, allExpectedCounts);
+
+ // segment w/ categories and some content
+ IndexDocsWithFacetsAndSomeTerms(indexWriter, taxoWriter, allExpectedCounts);
+
+ IOUtils.Close(indexWriter, taxoWriter);
+ }
+
+ [Test]
+ public virtual void TestDifferentNumResults()
+ {
+ // test the collector w/ FacetRequests and different numResults
+ DirectoryReader indexReader = DirectoryReader.Open(indexDir);
+ var taxoReader = new DirectoryTaxonomyReader(taxoDir);
+ IndexSearcher searcher = NewSearcher(indexReader);
+
+ FacetsCollector sfc = new FacetsCollector();
+ TermQuery q = new TermQuery(A);
+ searcher.Search(q, sfc);
+ Facets facets = GetTaxonomyFacetCounts(taxoReader, Config, sfc);
+ FacetResult result = facets.GetTopChildren(NUM_CHILDREN_CP_A, CP_A);
+ Assert.AreEqual(-1, (int)result.value);
+ foreach (LabelAndValue labelValue in result.labelValues)
+ {
+ Assert.AreEqual(termExpectedCounts[CP_A + "/" + labelValue.label], labelValue.value);
+ }
+ result = facets.GetTopChildren(NUM_CHILDREN_CP_B, CP_B);
+ Assert.AreEqual(termExpectedCounts[CP_B], result.value);
+ foreach (LabelAndValue labelValue in result.labelValues)
+ {
+ Assert.AreEqual(termExpectedCounts[CP_B + "/" + labelValue.label], labelValue.value);
+ }
+
+ IOUtils.Close(indexReader, taxoReader);
+ }
+
+ [Test]
+ public virtual void TestAllCounts()
+ {
+ DirectoryReader indexReader = DirectoryReader.Open(indexDir);
+ var taxoReader = new DirectoryTaxonomyReader(taxoDir);
+ IndexSearcher searcher = NewSearcher(indexReader);
+
+ FacetsCollector sfc = new FacetsCollector();
+ searcher.Search(new MatchAllDocsQuery(), sfc);
+
+ Facets facets = GetTaxonomyFacetCounts(taxoReader, Config, sfc);
+
+ FacetResult result = facets.GetTopChildren(NUM_CHILDREN_CP_A, CP_A);
+ Assert.AreEqual(-1, (int)result.value);
+ int prevValue = int.MaxValue;
+ foreach (LabelAndValue labelValue in result.labelValues)
+ {
+ Assert.AreEqual(allExpectedCounts[CP_A + "/" + labelValue.label], labelValue.value);
+ Assert.True((int)labelValue.value <= prevValue, "wrong sort order of sub results: labelValue.value=" + labelValue.value + " prevValue=" + prevValue);
+ prevValue = (int)labelValue.value;
+ }
+
+ result = facets.GetTopChildren(NUM_CHILDREN_CP_B, CP_B);
+ Assert.AreEqual(allExpectedCounts[CP_B], result.value);
+ prevValue = int.MaxValue;
+ foreach (LabelAndValue labelValue in result.labelValues)
+ {
+ Assert.AreEqual(allExpectedCounts[CP_B + "/" + labelValue.label], labelValue.value);
+ Assert.True((int)labelValue.value <= prevValue, "wrong sort order of sub results: labelValue.value=" + labelValue.value + " prevValue=" + prevValue);
+ prevValue = (int)labelValue.value;
+ }
+
+ IOUtils.Close(indexReader, taxoReader);
+ }
+
+ [Test]
+ public virtual void TestBigNumResults()
+ {
+ DirectoryReader indexReader = DirectoryReader.Open(indexDir);
+ var taxoReader = new DirectoryTaxonomyReader(taxoDir);
+ IndexSearcher searcher = NewSearcher(indexReader);
+
+ FacetsCollector sfc = new FacetsCollector();
+ searcher.Search(new MatchAllDocsQuery(), sfc);
+
+ Facets facets = GetTaxonomyFacetCounts(taxoReader, Config, sfc);
+
+ FacetResult result = facets.GetTopChildren(int.MaxValue, CP_A);
+ Assert.AreEqual(-1, (int)result.value);
+ foreach (LabelAndValue labelValue in result.labelValues)
+ {
+ Assert.AreEqual(allExpectedCounts[CP_A + "/" + labelValue.label], labelValue.value);
+ }
+ result = facets.GetTopChildren(int.MaxValue, CP_B);
+ Assert.AreEqual(allExpectedCounts[CP_B], result.value);
+ foreach (LabelAndValue labelValue in result.labelValues)
+ {
+ Assert.AreEqual(allExpectedCounts[CP_B + "/" + labelValue.label], labelValue.value);
+ }
+
+ IOUtils.Close(indexReader, taxoReader);
+ }
+
+ [Test]
+ public virtual void TestNoParents()
+ {
+ DirectoryReader indexReader = DirectoryReader.Open(indexDir);
+ var taxoReader = new DirectoryTaxonomyReader(taxoDir);
+ IndexSearcher searcher = NewSearcher(indexReader);
+
+ var sfc = new FacetsCollector();
+ searcher.Search(new MatchAllDocsQuery(), sfc);
+
+ Facets facets = GetTaxonomyFacetCounts(taxoReader, Config, sfc);
+
+ FacetResult result = facets.GetTopChildren(NUM_CHILDREN_CP_C, CP_C);
+ Assert.AreEqual(allExpectedCounts[CP_C], result.value);
+ foreach (LabelAndValue labelValue in result.labelValues)
+ {
+ Assert.AreEqual(allExpectedCounts[CP_C + "/" + labelValue.label], labelValue.value);
+ }
+ result = facets.GetTopChildren(NUM_CHILDREN_CP_D, CP_D);
+ Assert.AreEqual(allExpectedCounts[CP_C], result.value);
+ foreach (LabelAndValue labelValue in result.labelValues)
+ {
+ Assert.AreEqual(allExpectedCounts[CP_D + "/" + labelValue.label], labelValue.value);
+ }
+
+ IOUtils.Close(indexReader, taxoReader);
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/982eaf60/src/Lucene.Net.Tests/core/Facet/Taxonomy/TestTaxonomyFacetSumValueSource.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/core/Facet/Taxonomy/TestTaxonomyFacetSumValueSource.cs b/src/Lucene.Net.Tests/core/Facet/Taxonomy/TestTaxonomyFacetSumValueSource.cs
new file mode 100644
index 0000000..f90b9f6
--- /dev/null
+++ b/src/Lucene.Net.Tests/core/Facet/Taxonomy/TestTaxonomyFacetSumValueSource.cs
@@ -0,0 +1,610 @@
+using System;
+using System.Diagnostics;
+using System.Collections;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Lucene.Net.Randomized.Generators;
+
+namespace Lucene.Net.Facet.Taxonomy
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using Document = Lucene.Net.Documents.Document;
+ using Field = Lucene.Net.Documents.Field;
+ using FloatDocValuesField = Lucene.Net.Documents.FloatDocValuesField;
+ using IntField = Lucene.Net.Documents.IntField;
+ using NumericDocValuesField = Lucene.Net.Documents.NumericDocValuesField;
+ using StringField = Lucene.Net.Documents.StringField;
+ using DirectoryTaxonomyReader = Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyReader;
+ using DirectoryTaxonomyWriter = Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter;
+ using AtomicReaderContext = Lucene.Net.Index.AtomicReaderContext;
+ using DirectoryReader = Lucene.Net.Index.DirectoryReader;
+ using IndexWriter = Lucene.Net.Index.IndexWriter;
+ using IndexWriterConfig = Lucene.Net.Index.IndexWriterConfig;
+ using RandomIndexWriter = Lucene.Net.Index.RandomIndexWriter;
+ using Term = Lucene.Net.Index.Term;
+ using FunctionQuery = Lucene.Net.Queries.Function.FunctionQuery;
+ using FunctionValues = Lucene.Net.Queries.Function.FunctionValues;
+ using ValueSource = Lucene.Net.Queries.Function.ValueSource;
+ using DoubleDocValues = Lucene.Net.Queries.Function.DocValues.DoubleDocValues;
+ using FloatFieldSource = Lucene.Net.Queries.Function.ValueSources.FloatFieldSource;
+ using IntFieldSource = Lucene.Net.Queries.Function.ValueSources.IntFieldSource;
+ using LongFieldSource = Lucene.Net.Queries.Function.ValueSources.LongFieldSource;
+ using ConstantScoreQuery = Lucene.Net.Search.ConstantScoreQuery;
+ using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+ using MatchAllDocsQuery = Lucene.Net.Search.MatchAllDocsQuery;
+ using Query = Lucene.Net.Search.Query;
+ using Scorer = Lucene.Net.Search.Scorer;
+ using TermQuery = Lucene.Net.Search.TermQuery;
+ using TopDocs = Lucene.Net.Search.TopDocs;
+ using Directory = Lucene.Net.Store.Directory;
+ using IOUtils = Lucene.Net.Util.IOUtils;
+ using TestUtil = Lucene.Net.Util.TestUtil;
+ using NUnit.Framework;
+ [TestFixture]
+ public class TestTaxonomyFacetSumValueSource : FacetTestCase
+ {
+
+ [Test]
+ public virtual void TestBasic()
+ {
+
+ Store.Directory dir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+
+ // Writes facet ords to a separate directory from the
+ // main index:
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode_e.CREATE);
+
+ RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);
+ FacetsConfig config = new FacetsConfig();
+
+ // Reused across documents, to add the necessary facet
+ // fields:
+ Document doc = new Document();
+ doc.Add(new IntField("num", 10, Field.Store.NO));
+ doc.Add(new FacetField("Author", "Bob"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ doc = new Document();
+ doc.Add(new IntField("num", 20, Field.Store.NO));
+ doc.Add(new FacetField("Author", "Lisa"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ doc = new Document();
+ doc.Add(new IntField("num", 30, Field.Store.NO));
+ doc.Add(new FacetField("Author", "Lisa"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ doc = new Document();
+ doc.Add(new IntField("num", 40, Field.Store.NO));
+ doc.Add(new FacetField("Author", "Susan"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ doc = new Document();
+ doc.Add(new IntField("num", 45, Field.Store.NO));
+ doc.Add(new FacetField("Author", "Frank"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ // NRT open
+ IndexSearcher searcher = NewSearcher(writer.Reader);
+ writer.Dispose();
+
+ // NRT open
+ var taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+ taxoWriter.Dispose();
+
+ // Aggregate the facet counts:
+ FacetsCollector c = new FacetsCollector();
+
+ // MatchAllDocsQuery is for "browsing" (counts facets
+ // for all non-deleted docs in the index); normally
+ // you'd use a "normal" query and one of the
+ // Facets.search utility methods:
+ searcher.Search(new MatchAllDocsQuery(), c);
+
+ TaxonomyFacetSumValueSource facets = new TaxonomyFacetSumValueSource(taxoReader, new FacetsConfig(), c, new IntFieldSource("num"));
+
+ // Retrieve & verify results:
+ Assert.AreEqual("dim=Author path=[] value=145.0 childCount=4\n Lisa (50.0)\n Frank (45.0)\n Susan (40.0)\n Bob (10.0)\n", facets.GetTopChildren(10, "Author").ToString());
+
+ taxoReader.Dispose();
+ searcher.IndexReader.Dispose();
+ dir.Dispose();
+ taxoDir.Dispose();
+ }
+
+ // LUCENE-5333
+ [Test]
+ public virtual void TestSparseFacets()
+ {
+ Store.Directory dir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+
+ // Writes facet ords to a separate directory from the
+ // main index:
+ var taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode_e.CREATE);
+
+ RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);
+ FacetsConfig config = new FacetsConfig();
+
+ Document doc = new Document();
+ doc.Add(new IntField("num", 10, Field.Store.NO));
+ doc.Add(new FacetField("a", "foo1"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ if (Random().NextBoolean())
+ {
+ writer.Commit();
+ }
+
+ doc = new Document();
+ doc.Add(new IntField("num", 20, Field.Store.NO));
+ doc.Add(new FacetField("a", "foo2"));
+ doc.Add(new FacetField("b", "bar1"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ if (Random().NextBoolean())
+ {
+ writer.Commit();
+ }
+
+ doc = new Document();
+ doc.Add(new IntField("num", 30, Field.Store.NO));
+ doc.Add(new FacetField("a", "foo3"));
+ doc.Add(new FacetField("b", "bar2"));
+ doc.Add(new FacetField("c", "baz1"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ // NRT open
+ IndexSearcher searcher = NewSearcher(writer.Reader);
+ writer.Dispose();
+
+ // NRT open
+ var taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+ taxoWriter.Dispose();
+
+ FacetsCollector c = new FacetsCollector();
+ searcher.Search(new MatchAllDocsQuery(), c);
+
+ TaxonomyFacetSumValueSource facets = new TaxonomyFacetSumValueSource(taxoReader, new FacetsConfig(), c, new IntFieldSource("num"));
+
+ // Ask for top 10 labels for any dims that have counts:
+ IList<FacetResult> results = facets.GetAllDims(10);
+
+ Assert.AreEqual(3, results.Count);
+ Assert.AreEqual("dim=a path=[] value=60.0 childCount=3\n foo3 (30.0)\n foo2 (20.0)\n foo1 (10.0)\n", results[0].ToString());
+ Assert.AreEqual("dim=b path=[] value=50.0 childCount=2\n bar2 (30.0)\n bar1 (20.0)\n", results[1].ToString());
+ Assert.AreEqual("dim=c path=[] value=30.0 childCount=1\n baz1 (30.0)\n", results[2].ToString());
+
+ IOUtils.Close(searcher.IndexReader, taxoReader, dir, taxoDir);
+ }
+
+ [Test]
+ public virtual void TestWrongIndexFieldName()
+ {
+
+ Store.Directory dir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+
+ // Writes facet ords to a separate directory from the
+ // main index:
+ var taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode_e.CREATE);
+
+ FacetsConfig config = new FacetsConfig();
+ config.SetIndexFieldName("a", "$facets2");
+
+ RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);
+
+ Document doc = new Document();
+ doc.Add(new IntField("num", 10, Field.Store.NO));
+ doc.Add(new FacetField("a", "foo1"));
+ writer.AddDocument(config.Build(taxoWriter, doc));
+
+ // NRT open
+ IndexSearcher searcher = NewSearcher(writer.Reader);
+ writer.Dispose();
+
+ // NRT open
+ var taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+ taxoWriter.Dispose();
+
+ FacetsCollector c = new FacetsCollector();
+ searcher.Search(new MatchAllDocsQuery(), c);
+
+ TaxonomyFacetSumValueSource facets = new TaxonomyFacetSumValueSource(taxoReader, config, c, new IntFieldSource("num"));
+
+ // Ask for top 10 labels for any dims that have counts:
+ IList<FacetResult> results = facets.GetAllDims(10);
+ Assert.True(results.Count == 0);
+
+ try
+ {
+ facets.GetSpecificValue("a");
+ Fail("should have hit exc");
+ }
+ catch (System.ArgumentException)
+ {
+ // expected
+ }
+
+ try
+ {
+ facets.GetTopChildren(10, "a");
+ Fail("should have hit exc");
+ }
+ catch (System.ArgumentException)
+ {
+ // expected
+ }
+
+ IOUtils.Close(searcher.IndexReader, taxoReader, dir, taxoDir);
+ }
+
+ [Test]
+ public virtual void TestSumScoreAggregator()
+ {
+ Store.Directory indexDir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+
+ FacetsConfig config = new FacetsConfig();
+
+ for (int i = AtLeast(30); i > 0; --i)
+ {
+ Document doc = new Document();
+ if (Random().NextBoolean()) // don't match all documents
+ {
+ doc.Add(new StringField("f", "v", Field.Store.NO));
+ }
+ doc.Add(new FacetField("dim", "a"));
+ iw.AddDocument(config.Build(taxoWriter, doc));
+ }
+
+ DirectoryReader r = DirectoryReader.Open(iw, true);
+ DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ FacetsCollector fc = new FacetsCollector(true);
+ ConstantScoreQuery csq = new ConstantScoreQuery(new MatchAllDocsQuery());
+ csq.Boost = 2.0f;
+
+ TopDocs td = FacetsCollector.Search(NewSearcher(r), csq, 10, fc);
+
+ Facets facets = new TaxonomyFacetSumValueSource(taxoReader, config, fc, new TaxonomyFacetSumValueSource.ScoreValueSource());
+
+ int expected = (int)(td.MaxScore * td.TotalHits);
+ Assert.AreEqual(expected, (int)facets.GetSpecificValue("dim", "a"));
+
+ IOUtils.Close(iw, taxoWriter, taxoReader, taxoDir, r, indexDir);
+ }
+
+ [Test]
+ public virtual void TestNoScore()
+ {
+ Store.Directory indexDir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ FacetsConfig config = new FacetsConfig();
+ for (int i = 0; i < 4; i++)
+ {
+ Document doc = new Document();
+ doc.Add(new NumericDocValuesField("price", (i + 1)));
+ doc.Add(new FacetField("a", Convert.ToString(i % 2)));
+ iw.AddDocument(config.Build(taxoWriter, doc));
+ }
+
+ DirectoryReader r = DirectoryReader.Open(iw, true);
+ DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ FacetsCollector sfc = new FacetsCollector();
+ NewSearcher(r).Search(new MatchAllDocsQuery(), sfc);
+ Facets facets = new TaxonomyFacetSumValueSource(taxoReader, config, sfc, new LongFieldSource("price"));
+ Assert.AreEqual("dim=a path=[] value=10.0 childCount=2\n 1 (6.0)\n 0 (4.0)\n", facets.GetTopChildren(10, "a").ToString());
+
+ IOUtils.Close(taxoWriter, iw, taxoReader, taxoDir, r, indexDir);
+ }
+
+ [Test]
+ public virtual void TestWithScore()
+ {
+ Store.Directory indexDir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+
+ FacetsConfig config = new FacetsConfig();
+ for (int i = 0; i < 4; i++)
+ {
+ Document doc = new Document();
+ doc.Add(new NumericDocValuesField("price", (i + 1)));
+ doc.Add(new FacetField("a", Convert.ToString(i % 2)));
+ iw.AddDocument(config.Build(taxoWriter, doc));
+ }
+
+ DirectoryReader r = DirectoryReader.Open(iw, true);
+ DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ ValueSource valueSource = new ValueSourceAnonymousInnerClassHelper(this);
+
+ FacetsCollector fc = new FacetsCollector(true);
+ // score documents by their 'price' field - makes asserting the correct counts for the categories easier
+ Query q = new FunctionQuery(new LongFieldSource("price"));
+ FacetsCollector.Search(NewSearcher(r), q, 10, fc);
+ Facets facets = new TaxonomyFacetSumValueSource(taxoReader, config, fc, valueSource);
+
+ Assert.AreEqual("dim=a path=[] value=10.0 childCount=2\n 1 (6.0)\n 0 (4.0)\n", facets.GetTopChildren(10, "a").ToString());
+
+ IOUtils.Close(taxoWriter, iw, taxoReader, taxoDir, r, indexDir);
+ }
+
+ private class ValueSourceAnonymousInnerClassHelper : ValueSource
+ {
+ private readonly TestTaxonomyFacetSumValueSource outerInstance;
+
+ public ValueSourceAnonymousInnerClassHelper(TestTaxonomyFacetSumValueSource outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ public override FunctionValues GetValues(IDictionary context, AtomicReaderContext readerContext)
+ {
+ Scorer scorer = (Scorer)context["scorer"];
+ Debug.Assert(scorer != null);
+ return new DoubleDocValuesAnonymousInnerClassHelper(this, scorer);
+ }
+
+ private class DoubleDocValuesAnonymousInnerClassHelper : DoubleDocValues
+ {
+ private readonly ValueSourceAnonymousInnerClassHelper outerInstance;
+
+ private Scorer scorer;
+
+ public DoubleDocValuesAnonymousInnerClassHelper(ValueSourceAnonymousInnerClassHelper outerInstance, Scorer scorer)
+ : base(null) //todo: value source
+ {
+ this.outerInstance = outerInstance;
+ this.scorer = scorer;
+ }
+
+
+
+ public override double DoubleVal(int document)
+ {
+ try
+ {
+ return scorer.Score();
+ }
+ catch (IOException exception)
+ {
+ throw new Exception(exception.Message, exception);
+ }
+ }
+ }
+
+ public override bool Equals(object o)
+ {
+ return o == this;
+ }
+ public override int GetHashCode()
+ {
+ throw new NotImplementedException();
+ }
+
+ public override string Description
+ {
+ get
+ {
+ return "score()";
+ }
+
+ }
+ }
+
+ [Test]
+ public virtual void TestRollupValues()
+ {
+ Store.Directory indexDir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+
+ var taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ FacetsConfig config = new FacetsConfig();
+ config.SetHierarchical("a", true);
+ //config.setRequireDimCount("a", true);
+
+ for (int i = 0; i < 4; i++)
+ {
+ Document doc = new Document();
+ doc.Add(new NumericDocValuesField("price", (i + 1)));
+ doc.Add(new FacetField("a", Convert.ToString(i % 2), "1"));
+ iw.AddDocument(config.Build(taxoWriter, doc));
+ }
+
+ DirectoryReader r = DirectoryReader.Open(iw, true);
+ DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ ValueSource valueSource = new LongFieldSource("price");
+ FacetsCollector sfc = new FacetsCollector();
+ NewSearcher(r).Search(new MatchAllDocsQuery(), sfc);
+ Facets facets = new TaxonomyFacetSumValueSource(taxoReader, config, sfc, valueSource);
+
+ Assert.AreEqual("dim=a path=[] value=10.0 childCount=2\n 1 (6.0)\n 0 (4.0)\n", facets.GetTopChildren(10, "a").ToString());
+
+ IOUtils.Close(taxoWriter, iw, taxoReader, taxoDir, r, indexDir);
+ }
+
+ [Test]
+ public virtual void TestCountAndSumScore()
+ {
+ Store.Directory indexDir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+
+ var taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ FacetsConfig config = new FacetsConfig();
+ config.SetIndexFieldName("b", "$b");
+
+ for (int i = AtLeast(30); i > 0; --i)
+ {
+ Document doc = new Document();
+ doc.Add(new StringField("f", "v", Field.Store.NO));
+ doc.Add(new FacetField("a", "1"));
+ doc.Add(new FacetField("b", "1"));
+ iw.AddDocument(config.Build(taxoWriter, doc));
+ }
+
+ DirectoryReader r = DirectoryReader.Open(iw, true);
+ var taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ FacetsCollector fc = new FacetsCollector(true);
+ FacetsCollector.Search(NewSearcher(r), new MatchAllDocsQuery(), 10, fc);
+
+ Facets facets1 = GetTaxonomyFacetCounts(taxoReader, config, fc);
+ Facets facets2 = new TaxonomyFacetSumValueSource(new DocValuesOrdinalsReader("$b"), taxoReader, config, fc, new TaxonomyFacetSumValueSource.ScoreValueSource());
+
+ Assert.AreEqual(r.MaxDoc, (int)facets1.GetTopChildren(10, "a").value);
+ Assert.AreEqual(r.MaxDoc, (double)facets2.GetTopChildren(10, "b").value, 1E-10);
+ IOUtils.Close(taxoWriter, iw, taxoReader, taxoDir, r, indexDir);
+ }
+
+ [Test]
+ public virtual void TestRandom()
+ {
+ string[] tokens = GetRandomTokens(10);
+ Store.Directory indexDir = NewDirectory();
+ Store.Directory taxoDir = NewDirectory();
+
+ RandomIndexWriter w = new RandomIndexWriter(Random(), indexDir);
+ var tw = new DirectoryTaxonomyWriter(taxoDir);
+ FacetsConfig config = new FacetsConfig();
+ int numDocs = AtLeast(1000);
+ int numDims = TestUtil.NextInt(Random(), 1, 7);
+ IList<TestDoc> testDocs = GetRandomDocs(tokens, numDocs, numDims);
+ foreach (TestDoc testDoc in testDocs)
+ {
+ Document doc = new Document();
+ doc.Add(NewStringField("content", testDoc.content, Field.Store.NO));
+ testDoc.value = Random().NextFloat();
+ doc.Add(new FloatDocValuesField("value", testDoc.value));
+ for (int j = 0; j < numDims; j++)
+ {
+ if (testDoc.dims[j] != null)
+ {
+ doc.Add(new FacetField("dim" + j, testDoc.dims[j]));
+ }
+ }
+ w.AddDocument(config.Build(tw, doc));
+ }
+
+ // NRT open
+ IndexSearcher searcher = NewSearcher(w.Reader);
+
+ // NRT open
+ var tr = new DirectoryTaxonomyReader(tw);
+
+ ValueSource values = new FloatFieldSource("value");
+
+ int iters = AtLeast(100);
+ for (int iter = 0; iter < iters; iter++)
+ {
+ string searchToken = tokens[Random().Next(tokens.Length)];
+ if (VERBOSE)
+ {
+ Console.WriteLine("\nTEST: iter content=" + searchToken);
+ }
+ FacetsCollector fc = new FacetsCollector();
+ FacetsCollector.Search(searcher, new TermQuery(new Term("content", searchToken)), 10, fc);
+ Facets facets = new TaxonomyFacetSumValueSource(tr, config, fc, values);
+
+ // Slow, yet hopefully bug-free, faceting:
+ var expectedValues = new List<Dictionary<string, float?>>();
+ for (int i = 0; i < numDims; i++)
+ {
+ expectedValues[i] = new Dictionary<string, float?>();
+ }
+
+ foreach (TestDoc doc in testDocs)
+ {
+ if (doc.content.Equals(searchToken))
+ {
+ for (int j = 0; j < numDims; j++)
+ {
+ if (doc.dims[j] != null)
+ {
+ float? v = expectedValues[j][doc.dims[j]];
+ if (v == null)
+ {
+ expectedValues[j][doc.dims[j]] = doc.value;
+ }
+ else
+ {
+ expectedValues[j][doc.dims[j]] = (float)v + doc.value;
+ }
+ }
+ }
+ }
+ }
+
+ IList<FacetResult> expected = new List<FacetResult>();
+ for (int i = 0; i < numDims; i++)
+ {
+ IList<LabelAndValue> labelValues = new List<LabelAndValue>();
+ float totValue = 0;
+ foreach (KeyValuePair<string, float?> ent in expectedValues[i])
+ {
+ labelValues.Add(new LabelAndValue(ent.Key, ent.Value.Value));
+ totValue += ent.Value.Value;
+ }
+ SortLabelValues(labelValues);
+ if (totValue > 0)
+ {
+ expected.Add(new FacetResult("dim" + i, new string[0], totValue, labelValues.ToArray(), labelValues.Count));
+ }
+ }
+
+ // Sort by highest value, tie break by value:
+ SortFacetResults(expected);
+
+ IList<FacetResult> actual = facets.GetAllDims(10);
+
+ // Messy: fixup ties
+ SortTies(actual);
+
+ if (VERBOSE)
+ {
+ Console.WriteLine("expected=\n" + expected.ToString());
+ Console.WriteLine("actual=\n" + actual.ToString());
+ }
+
+ AssertFloatValuesEquals(expected, actual);
+ }
+
+ IOUtils.Close(w, tw, searcher.IndexReader, tr, indexDir, taxoDir);
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/982eaf60/src/Lucene.Net.Tests/core/Facet/Taxonomy/WriterCache/TestCharBlockArray.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/core/Facet/Taxonomy/WriterCache/TestCharBlockArray.cs b/src/Lucene.Net.Tests/core/Facet/Taxonomy/WriterCache/TestCharBlockArray.cs
new file mode 100644
index 0000000..f4be988
--- /dev/null
+++ b/src/Lucene.Net.Tests/core/Facet/Taxonomy/WriterCache/TestCharBlockArray.cs
@@ -0,0 +1,110 @@
+using System.Text;
+using Lucene.Net.Support;
+using NUnit.Framework;
+
+namespace Lucene.Net.Facet.Taxonomy.WriterCache
+{
+
+
+ using TestUtil = Lucene.Net.Util.TestUtil;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ [TestFixture]
+ public class TestCharBlockArray : FacetTestCase
+ {
+
+ /* not finished yet because of missing charset decoder */
+
+ /*
+ public virtual void testArray()
+ {
+ CharBlockArray array = new CharBlockArray();
+ StringBuilder builder = new StringBuilder();
+
+ const int n = 100 * 1000;
+
+ byte[] buffer = new byte[50];
+
+ for (int i = 0; i < n; i++)
+ {
+ Random().NextBytes(buffer);
+ int size = 1 + Random().Next(50);
+ // This test is turning random bytes into a string,
+ // this is asking for trouble.
+
+ CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder().onUnmappableCharacter(CodingErrorAction.REPLACE).onMalformedInput(CodingErrorAction.REPLACE);
+ string s = decoder.Decode(ByteBuffer.Wrap(buffer, 0, size)).ToString();
+ array.append(s);
+ builder.Append(s);
+ }
+
+ for (int i = 0; i < n; i++)
+ {
+ Random().NextBytes(buffer);
+ int size = 1 + Random().Next(50);
+ // This test is turning random bytes into a string,
+ // this is asking for trouble.
+ CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder().onUnmappableCharacter(CodingErrorAction.REPLACE).onMalformedInput(CodingErrorAction.REPLACE);
+ string s = decoder.decode(ByteBuffer.Wrap(buffer, 0, size)).ToString();
+ array.append((CharSequence)s);
+ builder.Append(s);
+ }
+
+ for (int i = 0; i < n; i++)
+ {
+ Random().NextBytes(buffer);
+ int size = 1 + Random().Next(50);
+ // This test is turning random bytes into a string,
+ // this is asking for trouble.
+ CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder().onUnmappableCharacter(CodingErrorAction.REPLACE).onMalformedInput(CodingErrorAction.REPLACE);
+ string s = decoder.decode(ByteBuffer.Wrap(buffer, 0, size)).ToString();
+ for (int j = 0; j < s.Length; j++)
+ {
+ array.append(s[j]);
+ }
+ builder.Append(s);
+ }
+
+ AssertEqualsInternal("GrowingCharArray<->StringBuilder mismatch.", builder, array);
+
+ File tempDir = CreateTempDir("growingchararray");
+ File f = new File(tempDir, "GrowingCharArrayTest.tmp");
+ BufferedOutputStream @out = new BufferedOutputStream(new FileOutputStream(f));
+ array.flush(@out);
+ @out.flush();
+ @out.Close();
+
+ BufferedInputStream @in = new BufferedInputStream(new FileInputStream(f));
+ array = CharBlockArray.open(@in);
+ AssertEqualsInternal("GrowingCharArray<->StringBuilder mismatch after flush/load.", builder, array);
+ @in.Close();
+ f.delete();
+ }
+
+ private static void AssertEqualsInternal(string msg, StringBuilder expected, CharBlockArray actual)
+ {
+ Assert.AreEqual(expected.Length, actual.Length, msg);
+ for (int i = 0; i < expected.Length; i++)
+ {
+ Assert.AreEqual(expected[i], actual.CharAt(i), msg);
+ }
+ }
+ */
+ }
+
+}
\ No newline at end of file