You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/01/30 15:00:43 UTC
svn commit: r1440419 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/facet/ lucene/facet/src/java/org/apache/lucene/facet/index/
lucene/facet/src/java/org/apache/lucene/facet/index/params/
lucene/facet/src/java/org/apache/lucene/facet/search/ lu...
Author: shaie
Date: Wed Jan 30 14:00:42 2013
New Revision: 1440419
URL: http://svn.apache.org/viewvc?rev=1440419&view=rev
Log:
LUCENE-4715: Add OrdinalPolicy.ALL_BUT_DIMENSION
Added:
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/params/PerDimensionOrdinalPolicy.java
- copied unchanged from r1440416, lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/index/params/PerDimensionOrdinalPolicy.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/CountingListBuilder.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/CountingFacetsCollector.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsCollector.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestMultiCLExample.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/index/TestFacetsPayloadMigrationReader.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsCollectorTest.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestSameRequestAccumulation.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Wed Jan 30 14:00:42 2013
@@ -46,6 +46,11 @@ Optimizations
* LUCENE-4690: Performance improvements and non-hashing versions
of NumericUtils.*ToPrefixCoded() (yonik)
+* LUCENE-4715: CategoryListParams.getOrdinalPolicy now allows to return a
+ different OrdinalPolicy per dimension, to better tune how you index
+ facets. Also added OrdinalPolicy.ALL_BUT_DIMENSION.
+ (Shai Erera, Michael McCandless)
+
New Features
* LUCENE-4686: New specialized DGapVInt8IntEncoder for facets (now the
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/CountingListBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/CountingListBuilder.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/CountingListBuilder.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/CountingListBuilder.java Wed Jan 30 14:00:42 2013
@@ -3,6 +3,7 @@ package org.apache.lucene.facet.index;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
@@ -115,12 +116,12 @@ public class CountingListBuilder impleme
private final OrdinalsEncoder ordinalsEncoder;
private final TaxonomyWriter taxoWriter;
- private final OrdinalPolicy ordinalPolicy;
+ private final CategoryListParams clp;
public CountingListBuilder(CategoryListParams categoryListParams, FacetIndexingParams indexingParams,
TaxonomyWriter taxoWriter) {
this.taxoWriter = taxoWriter;
- this.ordinalPolicy = categoryListParams.getOrdinalPolicy();
+ this.clp = categoryListParams;
if (indexingParams.getPartitionSize() == Integer.MAX_VALUE) {
ordinalsEncoder = new NoPartitionsOrdinalsEncoder(categoryListParams);
} else {
@@ -141,16 +142,23 @@ public class CountingListBuilder impleme
*/
@Override
public Map<String,BytesRef> build(IntsRef ordinals, Iterable<CategoryPath> categories) throws IOException {
- int upto = ordinals.length; // since we add ordinals to IntsRef, iterate upto original length
-
- if (ordinalPolicy == OrdinalPolicy.ALL_PARENTS) { // add all parents too
- for (int i = 0; i < upto; i++) {
- int ordinal = ordinals.ints[i];
+ int upto = ordinals.length; // since we may add ordinals to IntsRef, iterate upto original length
+
+ Iterator<CategoryPath> iter = categories.iterator();
+ for (int i = 0; i < upto; i++) {
+ int ordinal = ordinals.ints[i];
+ CategoryPath cp = iter.next();
+ OrdinalPolicy op = clp.getOrdinalPolicy(cp.components[0]);
+ if (op != OrdinalPolicy.NO_PARENTS) {
+ // need to add parents too
int parent = taxoWriter.getParent(ordinal);
while (parent > 0) {
ordinals.ints[ordinals.length++] = parent;
parent = taxoWriter.getParent(parent);
}
+ if (op == OrdinalPolicy.ALL_BUT_DIMENSION) { // discard the last added parent, which is the dimension
+ ordinals.length--;
+ }
}
}
return ordinalsEncoder.encode(ordinals);
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java Wed Jan 30 14:00:42 2013
@@ -4,6 +4,7 @@ import java.io.IOException;
import org.apache.lucene.facet.search.CategoryListIterator;
import org.apache.lucene.facet.search.DocValuesCategoryListIterator;
+import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.util.PartitionsUtils;
import org.apache.lucene.util.encoding.DGapVInt8IntEncoder;
import org.apache.lucene.util.encoding.IntDecoder;
@@ -35,25 +36,61 @@ import org.apache.lucene.util.encoding.U
*/
public class CategoryListParams {
- /** OrdinalPolicy defines which ordinals are encoded for every document. */
+ /**
+ * Defines which category ordinals are encoded for every document. This also
+ * affects how category ordinals are aggregated, check the different policies
+ * for more details.
+ */
public static enum OrdinalPolicy {
/**
- * Encodes only the ordinal of leaf nodes. That is, the category A/B/C will
- * not encode the ordinals of A and A/B.
+ * Encodes only the ordinals of leaf nodes. That is, for the category A/B/C,
+ * the ordinals of A and A/B will not be encoded. This policy is efficient
+ * for hierarchical dimensions, as it reduces the number of ordinals that
+ * are visited per document. During faceted search, this policy behaves
+ * exactly like {@link #ALL_PARENTS}, and the counts of all path components
+ * will be computed as well.
*
* <p>
* <b>NOTE:</b> this {@link OrdinalPolicy} requires a special collector or
- * accumulator, which will fix the parents' counts, unless you are not
- * interested in the parents counts.
+ * accumulator, which will fix the parents' counts.
+ *
+ * <p>
+ * <b>NOTE:</b> since only leaf nodes are encoded for the document, you
+ * should use this policy when the same document doesn't share two
+ * categories that have a mutual parent, or otherwise the counts will be
+ * wrong (the mutual parent will be over-counted). For example, if a
+ * document has the categories A/B/C and A/B/D, then with this policy the
+ * counts of "A" and "B" will be 2, which is wrong. If you intend to index
+ * hierarchical dimensions, with more than one category per document, you
+ * should use either {@link #ALL_PARENTS} or {@link #ALL_BUT_DIMENSION}.
*/
NO_PARENTS,
/**
* Encodes the ordinals of all path components. That is, the category A/B/C
- * will encode the ordinals of A and A/B as well. This is the default
- * {@link OrdinalPolicy}.
+ * will encode the ordinals of A and A/B as well. If you don't require the
+ * dimension's count during search, consider using
+ * {@link #ALL_BUT_DIMENSION}.
+ */
+ ALL_PARENTS,
+
+ /**
+ * Encodes the ordinals of all path components except the dimension. The
+ * dimension of a category is defined to be the first components in
+ * {@link CategoryPath#components}. For the category A/B/C, the ordinal of
+ * A/B will be encoded as well, however not the ordinal of A.
+ *
+ * <p>
+ * <b>NOTE:</b> when facets are aggregated, this policy behaves exactly like
+ * {@link #ALL_PARENTS}, except that the dimension is never counted. I.e. if
+ * you ask to count the facet "A", then while in {@link #ALL_PARENTS} you
+ * will get counts for "A" <u>and its children</u>, with this policy you
+ * will get counts for <u>only its children</u>. This policy is the default
+ * one, and makes sense for using with flat dimensions, whenever your
+ * application does not require the dimension's count. Otherwise, use
+ * {@link #ALL_PARENTS}.
*/
- ALL_PARENTS
+ ALL_BUT_DIMENSION
}
/** The default field used to store the facets information. */
@@ -63,7 +100,7 @@ public class CategoryListParams {
* The default {@link OrdinalPolicy} that's used when encoding a document's
* category ordinals.
*/
- public static final OrdinalPolicy DEFAULT_ORDINAL_POLICY = OrdinalPolicy.ALL_PARENTS;
+ public static final OrdinalPolicy DEFAULT_ORDINAL_POLICY = OrdinalPolicy.ALL_BUT_DIMENSION;
public final String field;
@@ -115,19 +152,15 @@ public class CategoryListParams {
return false;
}
CategoryListParams other = (CategoryListParams) o;
- if (this.hashCode != other.hashCode) {
+ if (hashCode != other.hashCode) {
return false;
}
-
- // The above hashcodes might equal each other in the case of a collision,
- // so at this point only directly term equality testing will settle
- // the equality test.
return field.equals(other.field);
}
@Override
public int hashCode() {
- return this.hashCode;
+ return hashCode;
}
/** Create the {@link CategoryListIterator} for the specified partition. */
@@ -137,14 +170,18 @@ public class CategoryListParams {
return new DocValuesCategoryListIterator(docValuesField, createEncoder().createMatchingDecoder());
}
- /** Returns the {@link OrdinalPolicy} to use for this {@link CategoryListParams}. */
- public OrdinalPolicy getOrdinalPolicy() {
+ /**
+ * Returns the {@link OrdinalPolicy} to use for the given dimension. This
+ * {@link CategoryListParams} always returns {@link #DEFAULT_ORDINAL_POLICY}
+ * for all dimensions.
+ */
+ public OrdinalPolicy getOrdinalPolicy(String dimension) {
return DEFAULT_ORDINAL_POLICY;
}
@Override
public String toString() {
- return "field=" + field + " encoder=" + createEncoder() + " ordinalPolicy=" + getOrdinalPolicy();
+ return "field=" + field + " encoder=" + createEncoder() + " ordinalPolicy=" + getOrdinalPolicy(null);
}
-}
\ No newline at end of file
+}
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/CountingFacetsCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/CountingFacetsCollector.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/CountingFacetsCollector.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/CountingFacetsCollector.java Wed Jan 30 14:00:42 2013
@@ -85,7 +85,7 @@ import org.apache.lucene.util.encoding.D
public class CountingFacetsCollector extends FacetsCollector {
private final FacetSearchParams fsp;
- private final OrdinalPolicy ordinalPolicy;
+ private final CategoryListParams clp;
private final TaxonomyReader taxoReader;
private final BytesRef buf = new BytesRef(32);
private final FacetArrays facetArrays;
@@ -107,8 +107,7 @@ public class CountingFacetsCollector ext
assert assertParams(fsp) == null : assertParams(fsp);
this.fsp = fsp;
- CategoryListParams clp = fsp.indexingParams.getCategoryListParams(fsp.facetRequests.get(0).categoryPath);
- this.ordinalPolicy = clp.getOrdinalPolicy();
+ this.clp = fsp.indexingParams.getCategoryListParams(fsp.facetRequests.get(0).categoryPath);
this.facetsField = clp.field;
this.taxoReader = taxoReader;
this.facetArrays = facetArrays;
@@ -217,21 +216,21 @@ public class CountingFacetsCollector ext
}
}
- private void countParents(int[] parents) {
- // counts[0] is the count of ROOT, which we don't care about and counts[1]
- // can only update counts[0], so we don't bother to visit it too. also,
- // since parents always have lower ordinals than their children, we traverse
- // the array backwards. this also allows us to update just the immediate
- // parent's count (actually, otherwise it would be a mistake).
- for (int i = counts.length - 1; i > 1; i--) {
- int count = counts[i];
- if (count > 0) {
- int parent = parents[i];
- if (parent != 0) {
- counts[parent] += count;
- }
- }
+ /**
+ * Computes the counts of ordinals under the given ordinal's tree, by
+ * recursively going down to leaf nodes and rollin up their counts (called
+ * only with categories are indexing with OrdinalPolicy.NO_PARENTS).
+ */
+ private int rollupCounts(int ordinal, int[] children, int[] siblings) {
+ int count = 0;
+ while (ordinal != TaxonomyReader.INVALID_ORDINAL) {
+ int childCount = counts[ordinal];
+ childCount += rollupCounts(children[ordinal], children, siblings);
+ counts[ordinal] = childCount;
+ count += childCount;
+ ordinal = siblings[ordinal];
}
+ return count;
}
@Override
@@ -242,11 +241,6 @@ public class CountingFacetsCollector ext
ParallelTaxonomyArrays arrays = taxoReader.getParallelTaxonomyArrays();
- if (ordinalPolicy == OrdinalPolicy.NO_PARENTS) {
- // need to count parents
- countParents(arrays.parents());
- }
-
// compute top-K
final int[] children = arrays.children();
final int[] siblings = arrays.siblings();
@@ -256,6 +250,12 @@ public class CountingFacetsCollector ext
if (rootOrd == TaxonomyReader.INVALID_ORDINAL) { // category does not exist
continue;
}
+ OrdinalPolicy ordinalPolicy = clp.getOrdinalPolicy(fr.categoryPath.components[0]);
+ if (ordinalPolicy == OrdinalPolicy.NO_PARENTS) {
+ // need to count parents
+ counts[rootOrd] += rollupCounts(children[rootOrd], children, siblings);
+ }
+
FacetResultNode root = new FacetResultNode();
root.ordinal = rootOrd;
root.label = fr.categoryPath;
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java Wed Jan 30 14:00:42 2013
@@ -43,13 +43,21 @@ public abstract class FacetsCollector ex
* Returns the most optimized {@link FacetsCollector} for the given search
* parameters. The returned {@link FacetsCollector} is guaranteed to satisfy
* the requested parameters.
+ *
+ * @throws IllegalArgumentException
+ * if there is no built-in collector that can satisfy the search
+ * parameters.
*/
public static FacetsCollector create(FacetSearchParams fsp, IndexReader indexReader, TaxonomyReader taxoReader) {
if (CountingFacetsCollector.assertParams(fsp) == null) {
return new CountingFacetsCollector(fsp, taxoReader);
}
- return new StandardFacetsCollector(fsp, indexReader, taxoReader);
+ if (StandardFacetsCollector.assertParams(fsp) == null) {
+ return new StandardFacetsCollector(fsp, indexReader, taxoReader);
+ }
+
+ throw new IllegalArgumentException("None of the built-in FacetsCollectors can handle the given search params");
}
/**
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsCollector.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsCollector.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsCollector.java Wed Jan 30 14:00:42 2013
@@ -49,6 +49,17 @@ public class StandardFacetsCollector ext
private List<FacetResult> results;
private Object resultsGuard;
+ static String assertParams(FacetSearchParams fsp) {
+ // make sure none of the categories in the given FacetRequests was indexed with NO_PARENTS
+ for (FacetRequest fr : fsp.facetRequests) {
+ CategoryListParams clp = fsp.indexingParams.getCategoryListParams(fr.categoryPath);
+ if (clp.getOrdinalPolicy(fr.categoryPath.components[0]) == OrdinalPolicy.NO_PARENTS) {
+ return "this collector does not support aggregating categories that were indexed with OrdinalPolicy.NO_PARENTS";
+ }
+ }
+ return null;
+ }
+
/**
* Create a collector for accumulating facets while collecting documents
* during search.
@@ -62,6 +73,7 @@ public class StandardFacetsCollector ext
* taxonomy containing the facets.
*/
public StandardFacetsCollector(FacetSearchParams facetSearchParams, IndexReader indexReader, TaxonomyReader taxonomyReader) {
+ assert assertParams(facetSearchParams) == null : assertParams(facetSearchParams);
facetsAccumulator = initFacetsAccumulator(facetSearchParams, indexReader, taxonomyReader);
scoreDocIdCollector = initScoredDocCollector(facetSearchParams, indexReader, taxonomyReader);
resultsGuard = new Object();
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java Wed Jan 30 14:00:42 2013
@@ -74,6 +74,7 @@ public class ScoredDocIdsUtils {
/** Clear all deleted documents from a given open-bit-set according to a given reader */
private static void clearDeleted(final IndexReader reader, final FixedBitSet set) throws IOException {
+ // TODO use BitsFilteredDocIdSet?
// If there are no deleted docs
if (!reader.hasDeletions()) {
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java Wed Jan 30 14:00:42 2013
@@ -6,6 +6,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -17,6 +18,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.facet.index.FacetFields;
import org.apache.lucene.facet.index.params.CategoryListParams;
+import org.apache.lucene.facet.index.params.CategoryListParams.OrdinalPolicy;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.params.FacetRequest;
import org.apache.lucene.facet.search.params.FacetSearchParams;
@@ -44,6 +46,7 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.collections.IntToObjectMap;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -73,7 +76,8 @@ public abstract class FacetTestBase exte
SearchTaxoDirPair() {}
}
- private static HashMap<Integer, SearchTaxoDirPair> dirsPerPartitionSize;
+ private static IntToObjectMap<SearchTaxoDirPair> dirsPerPartitionSize;
+ private static IntToObjectMap<FacetIndexingParams> fipPerPartitionSize;
private static File TEST_DIR;
/** Documents text field. */
@@ -91,12 +95,15 @@ public abstract class FacetTestBase exte
@BeforeClass
public static void beforeClassFacetTestBase() {
TEST_DIR = _TestUtil.getTempDir("facets");
- dirsPerPartitionSize = new HashMap<Integer, FacetTestBase.SearchTaxoDirPair>();
+ dirsPerPartitionSize = new IntToObjectMap<FacetTestBase.SearchTaxoDirPair>();
+ fipPerPartitionSize = new IntToObjectMap<FacetIndexingParams>();
}
@AfterClass
public static void afterClassFacetTestBase() throws Exception {
- for (SearchTaxoDirPair pair : dirsPerPartitionSize.values()) {
+ Iterator<SearchTaxoDirPair> iter = dirsPerPartitionSize.iterator();
+ while (iter.hasNext()) {
+ SearchTaxoDirPair pair = iter.next();
IOUtils.close(pair.searchDir, pair.taxoDir);
}
}
@@ -128,20 +135,16 @@ public abstract class FacetTestBase exte
return DEFAULT_CONTENT[doc];
}
- /** Prepare index (in RAM) with single partition */
- protected final void initIndex() throws Exception {
- initIndex(Integer.MAX_VALUE);
- }
-
- /** Prepare index (in RAM) with some documents and some facets */
- protected final void initIndex(int partitionSize) throws Exception {
- initIndex(partitionSize, false);
+ /** Prepare index (in RAM) with some documents and some facets. */
+ protected final void initIndex(FacetIndexingParams fip) throws Exception {
+ initIndex(false, fip);
}
- /** Prepare index (in RAM/Disk) with some documents and some facets */
- protected final void initIndex(int partitionSize, boolean forceDisk) throws Exception {
+ /** Prepare index (in RAM/Disk) with some documents and some facets. */
+ protected final void initIndex(boolean forceDisk, FacetIndexingParams fip) throws Exception {
+ int partitionSize = fip.getPartitionSize();
if (VERBOSE) {
- System.out.println("Partition Size: " + partitionSize+" forceDisk: "+forceDisk);
+ System.out.println("Partition Size: " + partitionSize + " forceDisk: "+forceDisk);
}
SearchTaxoDirPair pair = dirsPerPartitionSize.get(Integer.valueOf(partitionSize));
@@ -158,7 +161,7 @@ public abstract class FacetTestBase exte
RandomIndexWriter iw = new RandomIndexWriter(random(), pair.searchDir, getIndexWriterConfig(getAnalyzer()));
TaxonomyWriter taxo = new DirectoryTaxonomyWriter(pair.taxoDir, OpenMode.CREATE);
- populateIndex(iw, taxo, getFacetIndexingParams(partitionSize));
+ populateIndex(iw, taxo, fip);
// commit changes (taxonomy prior to search index for consistency)
taxo.commit();
@@ -182,14 +185,40 @@ public abstract class FacetTestBase exte
/** Returns a {@link FacetIndexingParams} per the given partition size. */
protected FacetIndexingParams getFacetIndexingParams(final int partSize) {
- // several of our encoders don't support the value 0,
- // which is one of the values encoded when dealing w/ partitions.
- return new FacetIndexingParams() {
- @Override
- public int getPartitionSize() {
- return partSize;
- }
- };
+ return getFacetIndexingParams(partSize, false);
+ }
+
+ /**
+ * Returns a {@link FacetIndexingParams} per the given partition size. If
+ * requested, then {@link OrdinalPolicy} will be set to
+ * {@link OrdinalPolicy#ALL_PARENTS}, otherwise it will randomize.
+ */
+ protected FacetIndexingParams getFacetIndexingParams(final int partSize, final boolean forceAllParents) {
+ FacetIndexingParams fip = fipPerPartitionSize.get(partSize);
+ if (fip == null) {
+ // randomize OrdinalPolicy. Since not all Collectors / Accumulators
+ // support NO_PARENTS, don't include it.
+ // TODO: once all code paths support NO_PARENTS, randomize it too.
+ CategoryListParams randomOP = new CategoryListParams() {
+ final OrdinalPolicy op = random().nextBoolean() ? OrdinalPolicy.ALL_BUT_DIMENSION : OrdinalPolicy.ALL_PARENTS;
+ @Override
+ public OrdinalPolicy getOrdinalPolicy(String dimension) {
+ return forceAllParents ? OrdinalPolicy.ALL_PARENTS : op;
+ }
+ };
+
+ // several of our encoders don't support the value 0,
+ // which is one of the values encoded when dealing w/ partitions,
+ // therefore don't randomize the encoder.
+ fip = new FacetIndexingParams(randomOP) {
+ @Override
+ public int getPartitionSize() {
+ return partSize;
+ }
+ };
+ fipPerPartitionSize.put(partSize, fip);
+ }
+ return fip;
}
/**
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestMultiCLExample.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestMultiCLExample.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestMultiCLExample.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestMultiCLExample.java Wed Jan 30 14:00:42 2013
@@ -43,7 +43,6 @@ public class TestMultiCLExample extends
assertNotNull("Result should not be null", result);
FacetResultNode node = result.getFacetResultNode();
assertEquals("Invalid label", "5", node.label.toString());
- assertEquals("Invalid value", 2.0, node.value, 0.0);
assertEquals("Invalid # of subresults", 3, node.subResults.size());
Iterator<? extends FacetResultNode> subResults = node.subResults.iterator();
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/index/TestFacetsPayloadMigrationReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/index/TestFacetsPayloadMigrationReader.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/index/TestFacetsPayloadMigrationReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/index/TestFacetsPayloadMigrationReader.java Wed Jan 30 14:00:42 2013
@@ -25,6 +25,8 @@ import org.apache.lucene.facet.FacetTest
import org.apache.lucene.facet.index.params.CategoryListParams;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.index.params.PerDimensionIndexingParams;
+import org.apache.lucene.facet.index.params.CategoryListParams.OrdinalPolicy;
+import org.apache.lucene.facet.index.params.PerDimensionOrdinalPolicy;
import org.apache.lucene.facet.search.CategoryListIterator;
import org.apache.lucene.facet.search.DrillDown;
import org.apache.lucene.facet.search.FacetsCollector;
@@ -369,9 +371,23 @@ public class TestFacetsPayloadMigrationR
// set custom CLP fields for two dimensions and use the default ($facets) for the other two
HashMap<CategoryPath,CategoryListParams> params = new HashMap<CategoryPath,CategoryListParams>();
- params.put(new CategoryPath(DIMENSIONS[0]), new CategoryListParams(DIMENSIONS[0]));
- params.put(new CategoryPath(DIMENSIONS[1]), new CategoryListParams(DIMENSIONS[1]));
- FacetIndexingParams fip = new PerDimensionIndexingParams(params) {
+ params.put(new CategoryPath(DIMENSIONS[0]), new CategoryListParams(DIMENSIONS[0]) {
+ @Override
+ public OrdinalPolicy getOrdinalPolicy(String dimension) {
+ return OrdinalPolicy.ALL_PARENTS;
+ }
+ });
+ params.put(new CategoryPath(DIMENSIONS[1]), new CategoryListParams(DIMENSIONS[1]) {
+ @Override
+ public OrdinalPolicy getOrdinalPolicy(String dimension) {
+ return OrdinalPolicy.ALL_PARENTS;
+ }
+ });
+
+ HashMap<String,OrdinalPolicy> policies = new HashMap<String,CategoryListParams.OrdinalPolicy>();
+ policies.put(DIMENSIONS[2], OrdinalPolicy.ALL_PARENTS);
+ policies.put(DIMENSIONS[3], OrdinalPolicy.ALL_PARENTS);
+ FacetIndexingParams fip = new PerDimensionIndexingParams(params, new PerDimensionOrdinalPolicy(policies)) {
@Override
public int getPartitionSize() {
return partitionSize;
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java Wed Jan 30 14:00:42 2013
@@ -51,10 +51,9 @@ public abstract class BaseTestTopK exten
private int nextInt;
@Override
- protected void populateIndex(RandomIndexWriter iw, TaxonomyWriter taxo,
- FacetIndexingParams iParams) throws IOException {
+ protected void populateIndex(RandomIndexWriter iw, TaxonomyWriter taxo, FacetIndexingParams fip) throws IOException {
currDoc = -1;
- super.populateIndex(iw, taxo, iParams);
+ super.populateIndex(iw, taxo, fip);
}
/** prepare the next random int */
@@ -94,17 +93,13 @@ public abstract class BaseTestTopK exten
return Arrays.asList(cp);
}
- protected FacetSearchParams searchParamsWithRequests(int numResults) {
- return searchParamsWithRequests(numResults, Integer.MAX_VALUE);
- }
-
- protected FacetSearchParams searchParamsWithRequests(int numResults, int partitionSize) {
+ protected FacetSearchParams searchParamsWithRequests(int numResults, FacetIndexingParams fip) {
List<FacetRequest> facetRequests = new ArrayList<FacetRequest>();
facetRequests.add(new CountFacetRequest(new CategoryPath("a"), numResults));
facetRequests.add(new CountFacetRequest(new CategoryPath("a", "1"), numResults));
facetRequests.add(new CountFacetRequest(new CategoryPath("a", "1", "10"), numResults));
facetRequests.add(new CountFacetRequest(new CategoryPath("a", "2", "26", "267"), numResults));
- return getFacetSearchParams(facetRequests, getFacetIndexingParams(partitionSize));
+ return getFacetSearchParams(facetRequests, fip);
}
@Override
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsCollectorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsCollectorTest.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsCollectorTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsCollectorTest.java Wed Jan 30 14:00:42 2013
@@ -16,8 +16,9 @@ import org.apache.lucene.document.String
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.index.FacetFields;
import org.apache.lucene.facet.index.params.CategoryListParams;
+import org.apache.lucene.facet.index.params.CategoryListParams.OrdinalPolicy;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
-import org.apache.lucene.facet.index.params.PerDimensionIndexingParams;
+import org.apache.lucene.facet.index.params.PerDimensionOrdinalPolicy;
import org.apache.lucene.facet.search.params.CountFacetRequest;
import org.apache.lucene.facet.search.params.FacetRequest;
import org.apache.lucene.facet.search.params.FacetRequest.SortBy;
@@ -146,13 +147,11 @@ public class CountingFacetsCollectorTest
termExpectedCounts.put(cp, termExpectedCounts.get(cp) + 1);
}
}
- // add 1 to each dimension
- allExpectedCounts.put(CP_A, allExpectedCounts.get(CP_A) + 1);
+ // add 1 to each NO_PARENTS dimension
allExpectedCounts.put(CP_B, allExpectedCounts.get(CP_B) + 1);
allExpectedCounts.put(CP_C, allExpectedCounts.get(CP_C) + 1);
allExpectedCounts.put(CP_D, allExpectedCounts.get(CP_D) + 1);
if (updateTermExpectedCounts) {
- termExpectedCounts.put(CP_A, termExpectedCounts.get(CP_A) + 1);
termExpectedCounts.put(CP_B, termExpectedCounts.get(CP_B) + 1);
termExpectedCounts.put(CP_C, termExpectedCounts.get(CP_C) + 1);
termExpectedCounts.put(CP_D, termExpectedCounts.get(CP_D) + 1);
@@ -252,19 +251,13 @@ public class CountingFacetsCollectorTest
conf.setMergePolicy(NoMergePolicy.COMPOUND_FILES); // prevent merges, so we can control the index segments
IndexWriter indexWriter = new IndexWriter(indexDir, conf);
TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
- CategoryListParams allParents = new CategoryListParams();
- CategoryListParams noParents = new CategoryListParams("no_parents") {
- @Override
- public OrdinalPolicy getOrdinalPolicy() {
- return OrdinalPolicy.NO_PARENTS;
- }
- };
- Map<CategoryPath,CategoryListParams> params = new HashMap<CategoryPath,CategoryListParams>();
- params.put(CP_A, allParents);
- params.put(CP_B, allParents);
- params.put(CP_C, noParents);
- params.put(CP_D, noParents);
- fip = new PerDimensionIndexingParams(params);
+
+ Map<String,OrdinalPolicy> policies = new HashMap<String,CategoryListParams.OrdinalPolicy>();
+ policies.put(CP_B.components[0], OrdinalPolicy.ALL_PARENTS);
+ policies.put(CP_C.components[0], OrdinalPolicy.NO_PARENTS);
+ policies.put(CP_D.components[0], OrdinalPolicy.NO_PARENTS);
+ CategoryListParams clp = new PerDimensionOrdinalPolicy(policies);
+ fip = new FacetIndexingParams(clp);
allExpectedCounts = newCounts();
termExpectedCounts = newCounts();
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java Wed Jan 30 14:00:42 2013
@@ -104,9 +104,9 @@ public class TestDemoFacets extends Face
// Retrieve & verify results:
List<FacetResult> results = c.getFacetResults();
assertEquals(2, results.size());
- assertEquals("Publish Date (5)\n 2012 (2)\n 2010 (2)\n 1999 (1)\n",
+ assertEquals("Publish Date (0)\n 2012 (2)\n 2010 (2)\n 1999 (1)\n",
FacetTestUtils.toSimpleString(results.get(0)));
- assertEquals("Author (5)\n Lisa (2)\n Frank (1)\n Susan (1)\n Bob (1)\n",
+ assertEquals("Author (0)\n Lisa (2)\n Frank (1)\n Susan (1)\n Bob (1)\n",
FacetTestUtils.toSimpleString(results.get(1)));
@@ -117,7 +117,7 @@ public class TestDemoFacets extends Face
searcher.search(q2, c);
results = c.getFacetResults();
assertEquals(1, results.size());
- assertEquals("Author (2)\n Lisa (1)\n Bob (1)\n",
+ assertEquals("Author (0)\n Lisa (1)\n Bob (1)\n",
FacetTestUtils.toSimpleString(results.get(0)));
// Smoke test PrintTaxonomyStats:
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java Wed Jan 30 14:00:42 2013
@@ -14,6 +14,7 @@ import org.junit.Before;
import org.junit.Test;
import org.apache.lucene.facet.FacetTestBase;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.FacetsAccumulator;
import org.apache.lucene.facet.search.ScoredDocIDs;
import org.apache.lucene.facet.search.ScoredDocIdCollector;
@@ -48,11 +49,14 @@ import org.apache.lucene.facet.taxonomy.
*/
public class TestFacetsAccumulatorWithComplement extends FacetTestBase {
+ private FacetIndexingParams fip;
+
@Override
@Before
public void setUp() throws Exception {
super.setUp();
- initIndex();
+ fip = getFacetIndexingParams(Integer.MAX_VALUE);
+ initIndex(fip);
}
@Override
@@ -125,7 +129,7 @@ public class TestFacetsAccumulatorWithCo
/** compute facets with certain facet requests and docs */
private List<FacetResult> findFacets(ScoredDocIDs sDocids, boolean withComplement) throws IOException {
- FacetSearchParams fsp = new FacetSearchParams(getFacetIndexingParams(Integer.MAX_VALUE), new CountFacetRequest(new CategoryPath("root","a"), 10));
+ FacetSearchParams fsp = new FacetSearchParams(fip, new CountFacetRequest(new CategoryPath("root","a"), 10));
FacetsAccumulator fAccumulator = new StandardFacetsAccumulator(fsp, indexReader, taxoReader);
fAccumulator.setComplementThreshold(
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java Wed Jan 30 14:00:42 2013
@@ -274,7 +274,6 @@ public class TestMultipleCategoryLists e
Iterable<? extends FacetResultNode> subResults = resNode.subResults;
Iterator<? extends FacetResultNode> subIter = subResults.iterator();
- checkResult(resNode, "Band", 5.0);
checkResult(subIter.next(), "Band/Rock & Pop", 4.0);
checkResult(subIter.next(), "Band/Punk", 1.0);
@@ -283,7 +282,6 @@ public class TestMultipleCategoryLists e
subResults = resNode.subResults;
subIter = subResults.iterator();
- checkResult(resNode, "Band", 5.0);
checkResult(subIter.next(), "Band/Rock & Pop", 4.0);
checkResult(subIter.next(), "Band/Rock & Pop/Dave Matthews Band", 1.0);
checkResult(subIter.next(), "Band/Rock & Pop/REM", 1.0);
@@ -297,7 +295,6 @@ public class TestMultipleCategoryLists e
subResults = resNode.subResults;
subIter = subResults.iterator();
- checkResult(resNode, "Author", 3.0);
checkResult(subIter.next(), "Author/Kurt Vonnegut", 1.0);
checkResult(subIter.next(), "Author/Stephen King", 1.0);
checkResult(subIter.next(), "Author/Mark Twain", 1.0);
@@ -307,7 +304,6 @@ public class TestMultipleCategoryLists e
subResults = resNode.subResults;
subIter = subResults.iterator();
- checkResult(resNode, "Band/Rock & Pop", 4.0);
checkResult(subIter.next(), "Band/Rock & Pop/Dave Matthews Band", 1.0);
checkResult(subIter.next(), "Band/Rock & Pop/REM", 1.0);
checkResult(subIter.next(), "Band/Rock & Pop/U2", 1.0);
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestSameRequestAccumulation.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestSameRequestAccumulation.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestSameRequestAccumulation.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestSameRequestAccumulation.java Wed Jan 30 14:00:42 2013
@@ -3,6 +3,7 @@ package org.apache.lucene.facet.search;
import java.util.List;
import org.apache.lucene.facet.FacetTestBase;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.FacetsCollector;
import org.apache.lucene.facet.search.params.CountFacetRequest;
import org.apache.lucene.facet.search.params.FacetSearchParams;
@@ -31,18 +32,21 @@ import org.junit.Before;
public class TestSameRequestAccumulation extends FacetTestBase {
+ private FacetIndexingParams fip;
+
@Override
@Before
public void setUp() throws Exception {
super.setUp();
- initIndex();
+ fip = getFacetIndexingParams(Integer.MAX_VALUE);
+ initIndex(fip);
}
// Following LUCENE-4461 - ensure requesting the (exact) same request more
// than once does not alter the results
public void testTwoSameRequests() throws Exception {
final CountFacetRequest facetRequest = new CountFacetRequest(new CategoryPath("root"), 10);
- FacetSearchParams fsp = new FacetSearchParams(facetRequest);
+ FacetSearchParams fsp = new FacetSearchParams(fip, facetRequest);
FacetsCollector fc = FacetsCollector.create(fsp, indexReader, taxoReader);
searcher.search(new MatchAllDocsQuery(), fc);
@@ -50,7 +54,7 @@ public class TestSameRequestAccumulation
final String expected = fc.getFacetResults().get(0).toString();
// now add the same facet request with duplicates (same instance and same one)
- fsp = new FacetSearchParams(facetRequest, facetRequest, new CountFacetRequest(new CategoryPath("root"), 10));
+ fsp = new FacetSearchParams(fip, facetRequest, facetRequest, new CountFacetRequest(new CategoryPath("root"), 10));
// make sure the search params holds 3 requests now
assertEquals(3, fsp.facetRequests.size());
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java Wed Jan 30 14:00:42 2013
@@ -5,6 +5,7 @@ import java.util.Arrays;
import java.util.List;
import org.apache.lucene.facet.FacetTestBase;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.params.CountFacetRequest;
import org.apache.lucene.facet.search.params.FacetSearchParams;
import org.apache.lucene.facet.search.params.ScoreFacetRequest;
@@ -37,11 +38,14 @@ import org.junit.Test;
/** Test ScoredDocIdCollector. */
public class TestScoredDocIdCollector extends FacetTestBase {
+ private FacetIndexingParams fip;
+
@Override
@Before
public void setUp() throws Exception {
super.setUp();
- initIndex();
+ fip = getFacetIndexingParams(Integer.MAX_VALUE);
+ initIndex(fip);
}
@Override
@@ -73,8 +77,8 @@ public class TestScoredDocIdCollector ex
// verify by facet values
CategoryPath cp = new CategoryPath("root","a");
- FacetSearchParams countFSP = new FacetSearchParams(getFacetIndexingParams(Integer.MAX_VALUE), new CountFacetRequest(cp, 10));
- FacetSearchParams scoreFSP = new FacetSearchParams(getFacetIndexingParams(Integer.MAX_VALUE), new ScoreFacetRequest(cp, 10));
+ FacetSearchParams countFSP = new FacetSearchParams(fip, new CountFacetRequest(cp, 10));
+ FacetSearchParams scoreFSP = new FacetSearchParams(fip, new ScoreFacetRequest(cp, 10));
List<FacetResult> countRes = findFacets(scoredDocIDs, countFSP);
List<FacetResult> scoreRes = findFacets(scoredDocIDs, scoreFSP);
@@ -101,10 +105,8 @@ public class TestScoredDocIdCollector ex
}
// compute facets with certain facet requests and docs
- private List<FacetResult> findFacets(ScoredDocIDs sDocids,
- FacetSearchParams facetSearchParams) throws IOException {
- FacetsAccumulator fAccumulator = new StandardFacetsAccumulator(
- facetSearchParams, indexReader, taxoReader);
+ private List<FacetResult> findFacets(ScoredDocIDs sDocids, FacetSearchParams facetSearchParams) throws IOException {
+ FacetsAccumulator fAccumulator = new StandardFacetsAccumulator(facetSearchParams, indexReader, taxoReader);
List<FacetResult> res = fAccumulator.accumulate(sDocids);
// Results are ready, printing them...
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java Wed Jan 30 14:00:42 2013
@@ -113,7 +113,6 @@ public class TestStandardFacetsAccumulat
List<FacetResult> results = fc.getFacetResults();
assertEquals("received too many facet results", 1, results.size());
FacetResultNode frn = results.get(0).getFacetResultNode();
- assertEquals("wrong weight for \"A\"", 4, (int) frn.value);
assertEquals("wrong number of children", 2, frn.subResults.size());
for (FacetResultNode node : frn.subResults) {
assertEquals("wrong weight for child " + node.label, 2, (int) node.value);
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java Wed Jan 30 14:00:42 2013
@@ -181,7 +181,6 @@ public class TestTopKInEachNodeResultHan
boolean hasDoctor = "Doctor".equals(fr.getFacetRequest().categoryPath.components[0]);
assertEquals(9, fr.getNumValidDescendants());
FacetResultNode parentRes = fr.getFacetResultNode();
- assertEquals(16.0, parentRes.value, Double.MIN_VALUE);
assertEquals(2, parentRes.subResults.size());
// two nodes sorted by descending values: a/b with 8 and a/c with 6
// a/b has two children a/b/2 with value 3, and a/b/1 with value 2.
@@ -217,7 +216,6 @@ public class TestTopKInEachNodeResultHan
hasDoctor |= "Doctor".equals(fr.getFacetRequest().categoryPath.components[0]);
assertEquals(9, fr.getNumValidDescendants());
parentRes = fr.getFacetResultNode();
- assertEquals(16.0, parentRes.value, Double.MIN_VALUE);
assertEquals(2, parentRes.subResults.size());
// two nodes sorted by descending values: a/b with 8 and a/c with 6
// a/b has two children a/b/2 with value 3, and a/b/1 with value 2.
@@ -234,7 +232,6 @@ public class TestTopKInEachNodeResultHan
hasDoctor |= "Doctor".equals(fr.getFacetRequest().categoryPath.components[0]);
assertEquals(4, fr.getNumValidDescendants(), 4);
parentRes = fr.getFacetResultNode();
- assertEquals(16.0, parentRes.value, Double.MIN_VALUE);
assertEquals(2, parentRes.subResults.size());
// two nodes sorted by descending values:
// a/b with value 8 and a/c with value 6
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java Wed Jan 30 14:00:42 2013
@@ -4,6 +4,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.lucene.facet.index.params.CategoryListParams.OrdinalPolicy;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.params.CountFacetRequest;
import org.apache.lucene.facet.search.params.FacetRequest;
import org.apache.lucene.facet.search.params.FacetRequest.ResultMode;
@@ -73,7 +75,9 @@ public class TestTopKResultsHandler exte
@Test
public void testSimple() throws Exception {
for (int partitionSize : partitionSizes) {
- initIndex(partitionSize);
+ FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
+ OrdinalPolicy op = fip.getCategoryListParams(null).getOrdinalPolicy(null);
+ initIndex(fip);
List<FacetRequest> facetRequests = new ArrayList<FacetRequest>();
facetRequests.add(new CountFacetRequest(new CategoryPath("a"), 100));
@@ -87,8 +91,8 @@ public class TestTopKResultsHandler exte
facetRequests.add(new CountFacetRequest(new CategoryPath("a", "c"), 100));
// do different facet counts and compare to control
- FacetSearchParams sParams = getFacetSearchParams(facetRequests, getFacetIndexingParams(partitionSize));
-
+ FacetSearchParams sParams = getFacetSearchParams(facetRequests, fip);
+
FacetsCollector fc = new StandardFacetsCollector(sParams, indexReader, taxoReader) {
@Override
protected FacetsAccumulator initFacetsAccumulator(FacetSearchParams facetSearchParams, IndexReader indexReader, TaxonomyReader taxonomyReader) {
@@ -100,17 +104,21 @@ public class TestTopKResultsHandler exte
searcher.search(new MatchAllDocsQuery(), fc);
List<FacetResult> facetResults = fc.getFacetResults();
-
+
FacetResult fr = facetResults.get(0);
FacetResultNode parentRes = fr.getFacetResultNode();
- assertEquals(13.0, parentRes.value, Double.MIN_VALUE);
+ if (op == OrdinalPolicy.ALL_PARENTS) {
+ assertEquals(13.0, parentRes.value, Double.MIN_VALUE);
+ }
FacetResultNode[] frn = resultNodesAsArray(parentRes);
assertEquals(7.0, frn[0].value, Double.MIN_VALUE);
assertEquals(6.0, frn[1].value, Double.MIN_VALUE);
fr = facetResults.get(1);
parentRes = fr.getFacetResultNode();
- assertEquals(13.0, parentRes.value, Double.MIN_VALUE);
+ if (op == OrdinalPolicy.ALL_PARENTS) {
+ assertEquals(13.0, parentRes.value, Double.MIN_VALUE);
+ }
frn = resultNodesAsArray(parentRes);
assertEquals(7.0, frn[0].value, Double.MIN_VALUE);
assertEquals(6.0, frn[1].value, Double.MIN_VALUE);
@@ -121,7 +129,9 @@ public class TestTopKResultsHandler exte
fr = facetResults.get(2);
parentRes = fr.getFacetResultNode();
- assertEquals(7.0, parentRes.value, Double.MIN_VALUE);
+ if (op == OrdinalPolicy.ALL_PARENTS) {
+ assertEquals(7.0, parentRes.value, Double.MIN_VALUE);
+ }
frn = resultNodesAsArray(parentRes);
assertEquals(2.0, frn[0].value, Double.MIN_VALUE);
assertEquals(2.0, frn[1].value, Double.MIN_VALUE);
@@ -130,13 +140,17 @@ public class TestTopKResultsHandler exte
fr = facetResults.get(3);
parentRes = fr.getFacetResultNode();
- assertEquals(2.0, parentRes.value, Double.MIN_VALUE);
+ if (op == OrdinalPolicy.ALL_PARENTS) {
+ assertEquals(2.0, parentRes.value, Double.MIN_VALUE);
+ }
frn = resultNodesAsArray(parentRes);
assertEquals(0, frn.length);
fr = facetResults.get(4);
parentRes = fr.getFacetResultNode();
- assertEquals(6.0, parentRes.value, Double.MIN_VALUE);
+ if (op == OrdinalPolicy.ALL_PARENTS) {
+ assertEquals(6.0, parentRes.value, Double.MIN_VALUE);
+ }
frn = resultNodesAsArray(parentRes);
assertEquals(1.0, frn[0].value, Double.MIN_VALUE);
closeAll();
@@ -149,12 +163,12 @@ public class TestTopKResultsHandler exte
@Test
public void testGetMaxIntFacets() throws Exception {
for (int partitionSize : partitionSizes) {
- initIndex(partitionSize);
+ FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
+ initIndex(fip);
// do different facet counts and compare to control
CategoryPath path = new CategoryPath("a", "b");
- FacetSearchParams sParams = getFacetSearchParams(getFacetIndexingParams(partitionSize),
- new CountFacetRequest(path, Integer.MAX_VALUE));
+ FacetSearchParams sParams = getFacetSearchParams(fip, new CountFacetRequest(path, Integer.MAX_VALUE));
FacetsCollector fc = new StandardFacetsCollector(sParams, indexReader, taxoReader) {
@Override
@@ -174,7 +188,7 @@ public class TestTopKResultsHandler exte
// As a control base results, ask for top-1000 results
FacetSearchParams sParams2 = getFacetSearchParams(
- getFacetIndexingParams(partitionSize), new CountFacetRequest(path, Integer.MAX_VALUE));
+ fip, new CountFacetRequest(path, Integer.MAX_VALUE));
FacetsCollector fc2 = new StandardFacetsCollector(sParams2, indexReader, taxoReader) {
@Override
@@ -207,12 +221,11 @@ public class TestTopKResultsHandler exte
@Test
public void testSimpleSearchForNonexistentFacet() throws Exception {
for (int partitionSize : partitionSizes) {
- initIndex(partitionSize);
+ FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
+ initIndex(fip);
CategoryPath path = new CategoryPath("Miau Hattulla");
- FacetSearchParams sParams = getFacetSearchParams(
- getFacetIndexingParams(partitionSize),
- new CountFacetRequest(path, 10));
+ FacetSearchParams sParams = getFacetSearchParams(fip, new CountFacetRequest(path, 10));
FacetsCollector fc = FacetsCollector.create(sParams, indexReader, taxoReader);
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java Wed Jan 30 14:00:42 2013
@@ -4,6 +4,7 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.List;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.params.FacetSearchParams;
import org.apache.lucene.facet.search.results.FacetResult;
import org.apache.lucene.facet.search.results.FacetResultNode;
@@ -32,10 +33,10 @@ import org.junit.Test;
public class TestTopKResultsHandlerRandom extends BaseTestTopK {
- private List<FacetResult> countFacets(int partitionSize, int numResults, final boolean doComplement)
+ private List<FacetResult> countFacets(FacetIndexingParams fip, int numResults, final boolean doComplement)
throws IOException {
Query q = new MatchAllDocsQuery();
- FacetSearchParams facetSearchParams = searchParamsWithRequests(numResults, partitionSize);
+ FacetSearchParams facetSearchParams = searchParamsWithRequests(numResults, fip);
FacetsCollector fc = new StandardFacetsCollector(facetSearchParams, indexReader, taxoReader) {
@Override
protected FacetsAccumulator initFacetsAccumulator(
@@ -59,7 +60,8 @@ public class TestTopKResultsHandlerRando
@Test
public void testTopCountsOrder() throws Exception {
for (int partitionSize : partitionSizes) {
- initIndex(partitionSize);
+ FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
+ initIndex(fip);
/*
* Try out faceted search in it's most basic form (no sampling nor complement
@@ -67,7 +69,7 @@ public class TestTopKResultsHandlerRando
* being indexed, and later on an "over-all" faceted search is performed. The
* results are checked against the DF of each facet by itself
*/
- List<FacetResult> facetResults = countFacets(partitionSize, 100000, false);
+ List<FacetResult> facetResults = countFacets(fip, 100000, false);
assertCountsAndCardinality(facetCountsTruth(), facetResults);
/*
@@ -77,10 +79,10 @@ public class TestTopKResultsHandlerRando
* place in here. The results are checked against the a regular (a.k.a
* no-complement, no-sampling) faceted search with the same parameters.
*/
- facetResults = countFacets(partitionSize, 100000, true);
+ facetResults = countFacets(fip, 100000, true);
assertCountsAndCardinality(facetCountsTruth(), facetResults);
- List<FacetResult> allFacetResults = countFacets(partitionSize, 100000, false);
+ List<FacetResult> allFacetResults = countFacets(fip, 100000, false);
HashMap<String,Integer> all = new HashMap<String,Integer>();
int maxNumNodes = 0;
@@ -108,7 +110,7 @@ public class TestTopKResultsHandlerRando
if (VERBOSE) {
System.out.println("------- verify for "+n+" top results");
}
- List<FacetResult> someResults = countFacets(partitionSize, n, false);
+ List<FacetResult> someResults = countFacets(fip, n, false);
k = 0;
for (FacetResult fr : someResults) {
FacetResultNode topResNode = fr.getFacetResultNode();
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java Wed Jan 30 14:00:42 2013
@@ -8,7 +8,9 @@ import org.apache.lucene.facet.FacetTest
import org.apache.lucene.facet.FacetTestUtils;
import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyReaderPair;
import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyWriterPair;
+import org.apache.lucene.facet.index.params.CategoryListParams;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
+import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util._TestUtil;
@@ -59,6 +61,16 @@ public class TestTotalFacetCounts extend
public int getPartitionSize() {
return partitionSize;
}
+
+ @Override
+ public CategoryListParams getCategoryListParams(CategoryPath category) {
+ return new CategoryListParams() {
+ @Override
+ public OrdinalPolicy getOrdinalPolicy(String dimension) {
+ return OrdinalPolicy.ALL_PARENTS;
+ }
+ };
+ }
};
// The counts that the TotalFacetCountsArray should have after adding
// the below facets to the index.
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java Wed Jan 30 14:00:42 2013
@@ -87,7 +87,7 @@ public class TestTotalFacetCountsCache e
/** Utility method to add a document and facets to an index/taxonomy. */
static void addFacets(FacetIndexingParams iParams, IndexWriter iw,
- TaxonomyWriter tw, String... strings) throws IOException {
+ TaxonomyWriter tw, String... strings) throws IOException {
Document doc = new Document();
FacetFields facetFields = new FacetFields(tw, iParams);
facetFields.addFields(doc, Collections.singletonList(new CategoryPath(strings)));
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java?rev=1440419&r1=1440418&r2=1440419&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java Wed Jan 30 14:00:42 2013
@@ -3,6 +3,7 @@ package org.apache.lucene.facet.search.s
import java.util.List;
import java.util.Random;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.BaseTestTopK;
import org.apache.lucene.facet.search.FacetsAccumulator;
import org.apache.lucene.facet.search.FacetsCollector;
@@ -46,8 +47,8 @@ public abstract class BaseSampleTestTopK
protected static final int RETRIES = 10;
@Override
- protected FacetSearchParams searchParamsWithRequests(int numResults, int partitionSize) {
- FacetSearchParams res = super.searchParamsWithRequests(numResults, partitionSize);
+ protected FacetSearchParams searchParamsWithRequests(int numResults, FacetIndexingParams fip) {
+ FacetSearchParams res = super.searchParamsWithRequests(numResults, fip);
for (FacetRequest req : res.facetRequests) {
// randomize the way we aggregate results
if (random().nextBoolean()) {
@@ -71,20 +72,23 @@ public abstract class BaseSampleTestTopK
boolean useRandomSampler = random().nextBoolean();
for (int partitionSize : partitionSizes) {
try {
- initIndex(partitionSize);
+ // complements return counts for all ordinals, so force ALL_PARENTS indexing
+ // so that it's easier to compare
+ FacetIndexingParams fip = getFacetIndexingParams(partitionSize, true);
+ initIndex(fip);
// Get all of the documents and run the query, then do different
// facet counts and compare to control
Query q = new TermQuery(new Term(CONTENT_FIELD, BETA)); // 90% of the docs
ScoredDocIdCollector docCollector = ScoredDocIdCollector.create(indexReader.maxDoc(), false);
- FacetSearchParams expectedSearchParams = searchParamsWithRequests(K, partitionSize);
+ FacetSearchParams expectedSearchParams = searchParamsWithRequests(K, fip);
FacetsCollector fc = FacetsCollector.create(expectedSearchParams, indexReader, taxoReader);
searcher.search(q, MultiCollector.wrap(docCollector, fc));
List<FacetResult> expectedResults = fc.getFacetResults();
- FacetSearchParams samplingSearchParams = searchParamsWithRequests(K, partitionSize);
+ FacetSearchParams samplingSearchParams = searchParamsWithRequests(K, fip);
// try several times in case of failure, because the test has a chance to fail
// if the top K facets are not sufficiently common with the sample set