You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/01/16 11:09:45 UTC
svn commit: r1433878 [1/2] - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/facet/
lucene/facet/src/examples/org/apache/lucene/facet/example/merge/
lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/
lucene/facet/src/java/org/apach...
Author: shaie
Date: Wed Jan 16 10:09:43 2013
New Revision: 1433878
URL: http://svn.apache.org/viewvc?rev=1433878&view=rev
Log:
LUCENE-4602: migrate facets to DocValues
Added:
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsIterator.java
- copied unchanged from r1433869, lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsIterator.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/associations/FloatAssociationsIterator.java
- copied unchanged from r1433869, lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/associations/FloatAssociationsIterator.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/associations/IntAssociationsIterator.java
- copied unchanged from r1433869, lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/associations/IntAssociationsIterator.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/FacetsPayloadMigrationReader.java
- copied unchanged from r1433869, lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/index/FacetsPayloadMigrationReader.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/DocValuesCategoryListIterator.java
- copied unchanged from r1433869, lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/search/DocValuesCategoryListIterator.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/index/TestFacetsPayloadMigrationReader.java
- copied, changed from r1433869, lucene/dev/trunk/lucene/facet/src/test/org/apache/lucene/facet/index/TestFacetsPayloadMigrationReader.java
Removed:
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsPayloadIterator.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/associations/FloatAssociationsPayloadIterator.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/associations/IntAssociationsPayloadIterator.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadCategoryListIteraor.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/cache/
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestCategoryListCache.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java
lucene/dev/branches/branch_4x/lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLIndexer.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsFacetFields.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/CountingListBuilder.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/FacetFields.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/OrdinalMappingAtomicReader.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/DrillDown.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TopKFacetResultsHandler.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TotalFacetCounts.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TotalFacetCountsCache.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/associations/AssociationFloatSumAggregator.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/associations/AssociationIntSumAggregator.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/FacetRequest.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/FacetSearchParams.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/associations/AssociationFloatSumFacetRequest.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/associations/AssociationIntSumFacetRequest.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/results/FacetResult.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/Sampler.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/util/PartitionsUtils.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestAdaptiveExample.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestAssociationExample.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestMultiCLExample.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/index/OrdinalMappingReaderTest.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/index/params/FacetIndexingParamsTest.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/index/params/PerDimensionIndexingParamsTest.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/CategoryListIteratorTest.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsCollector.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/associations/AssociationsFacetRequestTest.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/params/FacetRequestTest.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/params/FacetSearchParamsTest.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/params/MultiCategoryListIteratorTest.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/OversampleWithDepthTest.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestConcurrentFacetedIndexing.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Wed Jan 16 10:09:43 2013
@@ -6,7 +6,13 @@ http://s.apache.org/luceneversions
======================= Lucene 4.2.0 =======================
-(No changes yet)
+Changes in backwards compatibility policy
+
+* LUCENE-4602: FacetFields now stores facet ordinals in a DocValues field,
+ rather than a payload. This forces rebuilding existing indexes, or do a
+ one time migration using FacetsPayloadMigratingReader. Since DocValues
+ support in-memory caching, CategoryListCache was removed too.
+ (Shai Erera, Michael McCandless)
======================= Lucene 4.1.0 =======================
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java Wed Jan 16 10:09:43 2013
@@ -90,8 +90,9 @@ public class TaxonomyMergeUtils {
DirectoryReader reader = DirectoryReader.open(srcIndexDir, -1);
List<AtomicReaderContext> leaves = reader.leaves();
- AtomicReader wrappedLeaves[] = new AtomicReader[leaves.size()];
- for (int i = 0; i < leaves.size(); i++) {
+ int numReaders = leaves.size();
+ AtomicReader wrappedLeaves[] = new AtomicReader[numReaders];
+ for (int i = 0; i < numReaders; i++) {
wrappedLeaves[i] = new OrdinalMappingAtomicReader(leaves.get(i).reader(), ordinalMap, params);
}
try {
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLIndexer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLIndexer.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLIndexer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLIndexer.java Wed Jan 16 10:09:43 2013
@@ -20,7 +20,6 @@ import org.apache.lucene.facet.taxonomy.
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
@@ -80,12 +79,12 @@ public class MultiCLIndexer {
// Initialize PerDimensionIndexingParams
static {
Map<CategoryPath, CategoryListParams> paramsMap = new HashMap<CategoryPath,CategoryListParams>();
- paramsMap.put(new CategoryPath("0"), new CategoryListParams(new Term("$Digits", "Zero")));
- paramsMap.put(new CategoryPath("1"), new CategoryListParams(new Term("$Digits", "One")));
- paramsMap.put(new CategoryPath("2"), new CategoryListParams(new Term("$Digits", "Two")));
- paramsMap.put(new CategoryPath("3"), new CategoryListParams(new Term("$Digits", "Three")));
- paramsMap.put(new CategoryPath("4"), new CategoryListParams(new Term("$Digits", "Four")));
- paramsMap.put(new CategoryPath("5"), new CategoryListParams(new Term("$Digits", "Five")));
+ paramsMap.put(new CategoryPath("0"), new CategoryListParams("$Digits$Zero"));
+ paramsMap.put(new CategoryPath("1"), new CategoryListParams("$Digits$One"));
+ paramsMap.put(new CategoryPath("2"), new CategoryListParams("$Digits$Two"));
+ paramsMap.put(new CategoryPath("3"), new CategoryListParams("$Digits$Three"));
+ paramsMap.put(new CategoryPath("4"), new CategoryListParams("$Digits$Four"));
+ paramsMap.put(new CategoryPath("5"), new CategoryListParams("$Digits$Five"));
MULTI_IPARAMS = new PerDimensionIndexingParams(paramsMap);
}
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsFacetFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsFacetFields.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsFacetFields.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsFacetFields.java Wed Jan 16 10:09:43 2013
@@ -114,7 +114,7 @@ public class AssociationsFacetFields ext
}
@Override
- protected FieldType fieldType() {
+ protected FieldType drillDownFieldType() {
return DRILL_DOWN_TYPE;
}
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/CountingListBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/CountingListBuilder.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/CountingListBuilder.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/CountingListBuilder.java Wed Jan 16 10:09:43 2013
@@ -56,10 +56,9 @@ public class CountingListBuilder impleme
private static final class NoPartitionsOrdinalsEncoder extends OrdinalsEncoder {
private final IntEncoder encoder;
- private final String name;
+ private final String name = "";
NoPartitionsOrdinalsEncoder(CategoryListParams categoryListParams) {
- name = categoryListParams.getTerm().text();
encoder = categoryListParams.createEncoder();
}
@@ -91,7 +90,7 @@ public class CountingListBuilder impleme
final HashMap<String,IntsRef> partitionOrdinals = new HashMap<String,IntsRef>();
for (int i = 0; i < ordinals.length; i++) {
int ordinal = ordinals.ints[i];
- final String name = PartitionsUtils.partitionNameByOrdinal(indexingParams, categoryListParams, ordinal);
+ final String name = PartitionsUtils.partitionNameByOrdinal(indexingParams, ordinal);
IntsRef partitionOrds = partitionOrdinals.get(name);
if (partitionOrds == null) {
partitionOrds = new IntsRef(32);
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/FacetFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/FacetFields.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/FacetFields.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/FacetFields.java Wed Jan 16 10:09:43 2013
@@ -4,17 +4,14 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StraightBytesDocValuesField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.facet.index.params.CategoryListParams;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
@@ -51,32 +48,6 @@ import org.apache.lucene.util.IntsRef;
*/
public class FacetFields {
- // a TokenStream for writing the counting list payload
- private static final class CountingListStream extends TokenStream {
- private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class);
- private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
- private Iterator<Entry<String,BytesRef>> categoriesData;
-
- CountingListStream() {}
-
- @Override
- public boolean incrementToken() throws IOException {
- if (!categoriesData.hasNext()) {
- return false;
- }
-
- Entry<String,BytesRef> entry = categoriesData.next();
- termAtt.setEmpty().append(entry.getKey());
- payloadAtt.setPayload(entry.getValue());
- return true;
- }
-
- void setCategoriesData(Map<String,BytesRef> categoriesData) {
- this.categoriesData = categoriesData.entrySet().iterator();
- }
-
- }
-
// The counting list is written in a payload, but we don't store it
// nor need norms.
private static final FieldType COUNTING_LIST_PAYLOAD_TYPE = new FieldType();
@@ -94,9 +65,7 @@ public class FacetFields {
// Therefore we set its IndexOptions to DOCS_ONLY.
private static final FieldType DRILL_DOWN_TYPE = new FieldType(TextField.TYPE_NOT_STORED);
static {
- // TODO: once we cutover to DocValues, we can set it to DOCS_ONLY for this
- // FacetFields (not associations)
- DRILL_DOWN_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ DRILL_DOWN_TYPE.setIndexOptions(IndexOptions.DOCS_ONLY);
DRILL_DOWN_TYPE.freeze();
}
@@ -175,10 +144,20 @@ public class FacetFields {
* Returns the {@link FieldType} with which the drill-down terms should be
* indexed. The default is {@link IndexOptions#DOCS_ONLY}.
*/
- protected FieldType fieldType() {
+ protected FieldType drillDownFieldType() {
return DRILL_DOWN_TYPE;
}
+ /**
+ * Add the counting list data to the document under the given field. Note that
+ * the field is determined by the {@link CategoryListParams}.
+ */
+ protected void addCountingListData(Document doc, Map<String,BytesRef> categoriesData, String field) {
+ for (Entry<String,BytesRef> entry : categoriesData.entrySet()) {
+ doc.add(new StraightBytesDocValuesField(field + entry.getKey(), entry.getValue()));
+ }
+ }
+
/** Adds the needed facet fields to the document. */
public void addFields(Document doc, Iterable<CategoryPath> categories) throws IOException {
if (categories == null) {
@@ -198,7 +177,7 @@ public class FacetFields {
IntsRef ordinals = new IntsRef(32); // should be enough for most common applications
for (Entry<CategoryListParams, Iterable<CategoryPath>> e : categoryLists.entrySet()) {
final CategoryListParams clp = e.getKey();
- final String field = clp.getTerm().field();
+ final String field = clp.field;
// build category list data
ordinals.length = 0; // reset
@@ -214,13 +193,11 @@ public class FacetFields {
Map<String,BytesRef> categoriesData = getCategoryListData(clp, ordinals, e.getValue());
// add the counting list data
- CountingListStream ts = new CountingListStream();
- ts.setCategoriesData(categoriesData);
- doc.add(new Field(field, ts, COUNTING_LIST_PAYLOAD_TYPE));
+ addCountingListData(doc, categoriesData, field);
// add the drill-down field
DrillDownStream drillDownStream = getDrillDownStream(e.getValue());
- Field drillDown = new Field(field, drillDownStream, fieldType());
+ Field drillDown = new Field(field, drillDownStream, drillDownFieldType());
doc.add(drillDown);
}
}
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/OrdinalMappingAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/OrdinalMappingAtomicReader.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/OrdinalMappingAtomicReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/OrdinalMappingAtomicReader.java Wed Jan 16 10:09:43 2013
@@ -25,13 +25,10 @@ import org.apache.lucene.facet.index.par
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter.OrdinalMap;
import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.DocsAndPositionsEnum;
-import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValues.Source;
+import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.index.FilterAtomicReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.encoding.IntDecoder;
@@ -41,8 +38,8 @@ import org.apache.lucene.util.encoding.I
* A {@link FilterAtomicReader} for updating facets ordinal references,
* based on an ordinal map. You should use this code in conjunction with merging
* taxonomies - after you merge taxonomies, you receive an {@link OrdinalMap}
- * which maps the 'old' payloads to the 'new' ones. You can use that map to
- * re-map the payloads which contain the facets information (ordinals) either
+ * which maps the 'old' ordinals to the 'new' ones. You can use that map to
+ * re-map the doc values which contain the facets information (ordinals) either
* before or while merging the indexes.
* <p>
* For re-mapping the ordinals during index merge, do the following:
@@ -70,9 +67,8 @@ import org.apache.lucene.util.encoding.I
public class OrdinalMappingAtomicReader extends FilterAtomicReader {
private final int[] ordinalMap;
- // a little obtuse: but we dont need to create Term objects this way
- private final Map<String,Map<BytesRef,CategoryListParams>> termMap =
- new HashMap<String,Map<BytesRef,CategoryListParams>>(1);
+
+ private final Map<String,CategoryListParams> dvFieldMap = new HashMap<String,CategoryListParams>();
/**
* Wraps an AtomicReader, mapping ordinals according to the ordinalMap.
@@ -91,125 +87,85 @@ public class OrdinalMappingAtomicReader
super(in);
this.ordinalMap = ordinalMap;
for (CategoryListParams params: indexingParams.getAllCategoryListParams()) {
- Term term = params.getTerm();
- Map<BytesRef,CategoryListParams> fieldMap = termMap.get(term.field());
- if (fieldMap == null) {
- fieldMap = new HashMap<BytesRef,CategoryListParams>(1);
- termMap.put(term.field(), fieldMap);
- }
- fieldMap.put(term.bytes(), params);
+ dvFieldMap.put(params.field, params);
}
}
@Override
- public Fields getTermVectors(int docID) throws IOException {
- Fields fields = super.getTermVectors(docID);
- if (fields == null) {
- return null;
- } else {
- return new OrdinalMappingFields(fields);
+ public DocValues docValues(String field) throws IOException {
+ DocValues inner = super.docValues(field);
+ if (inner == null) {
+ return inner;
}
- }
-
- @Override
- public Fields fields() throws IOException {
- Fields fields = super.fields();
- if (fields == null) {
- return null;
+
+ CategoryListParams clp = dvFieldMap.get(field);
+ if (clp == null) {
+ return inner;
} else {
- return new OrdinalMappingFields(fields);
+ return new OrdinalMappingDocValues(inner, clp);
}
}
- private class OrdinalMappingFields extends FilterFields {
+ private class OrdinalMappingDocValues extends DocValues {
- public OrdinalMappingFields(Fields in) {
- super(in);
+ private final CategoryListParams clp;
+ private final DocValues delegate;
+
+ public OrdinalMappingDocValues(DocValues delegate, CategoryListParams clp) {
+ this.delegate = delegate;
+ this.clp = clp;
}
@Override
- public Terms terms(String field) throws IOException {
- Terms terms = super.terms(field);
- if (terms == null) {
- return terms;
- }
- Map<BytesRef,CategoryListParams> termsMap = termMap.get(field);
- if (termsMap == null) {
- return terms;
- } else {
- return new OrdinalMappingTerms(terms, termsMap);
- }
- }
- }
-
- private class OrdinalMappingTerms extends FilterTerms {
- private final Map<BytesRef,CategoryListParams> termsMap;
-
- public OrdinalMappingTerms(Terms in, Map<BytesRef,CategoryListParams> termsMap) {
- super(in);
- this.termsMap = termsMap;
+ protected Source loadSource() throws IOException {
+ return new OrdinalMappingSource(getType(), clp, delegate.getSource());
}
@Override
- public TermsEnum iterator(TermsEnum reuse) throws IOException {
- // TODO: should we reuse the inner termsenum?
- return new OrdinalMappingTermsEnum(super.iterator(reuse), termsMap);
- }
- }
-
- private class OrdinalMappingTermsEnum extends FilterTermsEnum {
- private final Map<BytesRef,CategoryListParams> termsMap;
-
- public OrdinalMappingTermsEnum(TermsEnum in, Map<BytesRef,CategoryListParams> termsMap) {
- super(in);
- this.termsMap = termsMap;
+ protected Source loadDirectSource() throws IOException {
+ return new OrdinalMappingSource(getType(), clp, delegate.getDirectSource());
}
@Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
- // TODO: we could reuse our D&P enum if we need
- DocsAndPositionsEnum inner = super.docsAndPositions(liveDocs, reuse, flags);
- if (inner == null) {
- return inner;
- }
-
- CategoryListParams params = termsMap.get(term());
- if (params == null) {
- return inner;
- }
-
- return new OrdinalMappingDocsAndPositionsEnum(inner, params);
+ public Type getType() {
+ return Type.BYTES_VAR_STRAIGHT;
}
+
}
- private class OrdinalMappingDocsAndPositionsEnum extends FilterDocsAndPositionsEnum {
+ private class OrdinalMappingSource extends Source {
+
private final IntEncoder encoder;
private final IntDecoder decoder;
private final IntsRef ordinals = new IntsRef(32);
- private final BytesRef payloadOut = new BytesRef();
-
- public OrdinalMappingDocsAndPositionsEnum(DocsAndPositionsEnum in, CategoryListParams params) {
- super(in);
- encoder = params.createEncoder();
+ private final Source delegate;
+
+ protected OrdinalMappingSource(Type type, CategoryListParams clp, Source delegate) {
+ super(type);
+ this.delegate = delegate;
+ encoder = clp.createEncoder();
decoder = encoder.createMatchingDecoder();
}
-
+
+ @SuppressWarnings("synthetic-access")
@Override
- public BytesRef getPayload() throws IOException {
- BytesRef payload = super.getPayload();
- if (payload == null) {
- return payload;
+ public BytesRef getBytes(int docID, BytesRef ref) {
+ ref = delegate.getBytes(docID, ref);
+ if (ref == null || ref.length == 0) {
+ return ref;
} else {
- decoder.decode(payload, ordinals);
+ decoder.decode(ref, ordinals);
// map the ordinals
for (int i = 0; i < ordinals.length; i++) {
ordinals.ints[i] = ordinalMap[ordinals.ints[i]];
}
- encoder.encode(ordinals, payloadOut);
- return payloadOut;
+ encoder.encode(ordinals, ref);
+ return ref;
}
}
+
}
+
}
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java Wed Jan 16 10:09:43 2013
@@ -4,9 +4,8 @@ import java.io.IOException;
import java.io.Serializable;
import org.apache.lucene.facet.search.CategoryListIterator;
-import org.apache.lucene.facet.search.PayloadCategoryListIteraor;
+import org.apache.lucene.facet.search.DocValuesCategoryListIterator;
import org.apache.lucene.facet.util.PartitionsUtils;
-import org.apache.lucene.index.Term;
import org.apache.lucene.util.encoding.DGapIntEncoder;
import org.apache.lucene.util.encoding.IntDecoder;
import org.apache.lucene.util.encoding.IntEncoder;
@@ -38,39 +37,26 @@ import org.apache.lucene.util.encoding.V
*/
public class CategoryListParams implements Serializable {
- /** The default term used to store the facets information. */
- public static final Term DEFAULT_TERM = new Term("$facets", "$fulltree$");
+ /** The default field used to store the facets information. */
+ public static final String DEFAULT_FIELD = "$facets";
- private final Term term;
+ public final String field;
private final int hashCode;
- /**
- * Constructs a default category list parameters object, using
- * {@link #DEFAULT_TERM}.
- */
+ /** Constructs a default category list parameters object, using {@link #DEFAULT_FIELD}. */
public CategoryListParams() {
- this(DEFAULT_TERM);
+ this(DEFAULT_FIELD);
}
- /**
- * Constructs a category list parameters object, using the given {@link Term}.
- * @param term who's payload hold the category-list.
- */
- public CategoryListParams(Term term) {
- this.term = term;
+ /** Constructs a category list parameters object, using the given field. */
+ public CategoryListParams(String field) {
+ this.field = field;
// Pre-compute the hashCode because these objects are immutable. Saves
// some time on the comparisons later.
- this.hashCode = term.hashCode();
+ this.hashCode = field.hashCode();
}
- /**
- * A {@link Term} who's payload holds the category-list.
- */
- public final Term getTerm() {
- return term;
- }
-
/**
* Allows to override how categories are encoded and decoded. A matching
* {@link IntDecoder} is provided by the {@link IntEncoder}.
@@ -110,7 +96,7 @@ public class CategoryListParams implemen
// The above hashcodes might equal each other in the case of a collision,
// so at this point only directly term equality testing will settle
// the equality test.
- return this.term.equals(other.term);
+ return field.equals(other.field);
}
@Override
@@ -120,9 +106,9 @@ public class CategoryListParams implemen
/** Create the {@link CategoryListIterator} for the specified partition. */
public CategoryListIterator createCategoryListIterator(int partition) throws IOException {
- String categoryListTermStr = PartitionsUtils.partitionName(this, partition);
- Term payloadTerm = new Term(term.field(), categoryListTermStr);
- return new PayloadCategoryListIteraor(payloadTerm, createEncoder().createMatchingDecoder());
+ String categoryListTermStr = PartitionsUtils.partitionName(partition);
+ String docValuesField = field + categoryListTermStr;
+ return new DocValuesCategoryListIterator(docValuesField, createEncoder().createMatchingDecoder());
}
}
\ No newline at end of file
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/DrillDown.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/DrillDown.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/DrillDown.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/DrillDown.java Wed Jan 16 10:09:43 2013
@@ -55,7 +55,7 @@ public final class DrillDown {
CategoryListParams clp = iParams.getCategoryListParams(path);
char[] buffer = new char[path.fullPathLength()];
iParams.drillDownTermText(path, buffer);
- return new Term(clp.getTerm().field(), String.valueOf(buffer));
+ return new Term(clp.field, String.valueOf(buffer));
}
/**
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java Wed Jan 16 10:09:43 2013
@@ -10,6 +10,7 @@ import java.util.Map.Entry;
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.aggregator.Aggregator;
import org.apache.lucene.facet.search.params.FacetRequest;
import org.apache.lucene.facet.search.params.FacetSearchParams;
@@ -110,9 +111,8 @@ public class StandardFacetsAccumulator e
if (isUsingComplements) {
try {
- totalFacetCounts = TotalFacetCountsCache.getSingleton()
- .getTotalCounts(indexReader, taxonomyReader,
- searchParams.getFacetIndexingParams(), searchParams.getCategoryListCache());
+ totalFacetCounts = TotalFacetCountsCache.getSingleton().getTotalCounts(indexReader, taxonomyReader,
+ searchParams.getFacetIndexingParams());
if (totalFacetCounts != null) {
docids = ScoredDocIdsUtils.getComplementSet(docids, indexReader);
} else {
@@ -242,20 +242,29 @@ public class StandardFacetsAccumulator e
int maxDoc = -1;
while (iterator.next()) {
int docID = iterator.getDocID();
- while (docID >= maxDoc) { // find the segment which contains this document
- if (!contexts.hasNext()) {
- throw new RuntimeException("ScoredDocIDs contains documents outside this reader's segments !?");
- }
- current = contexts.next();
- maxDoc = current.docBase + current.reader().maxDoc();
- if (docID < maxDoc) { // segment has docs, check if it has categories
- boolean validSegment = categoryListIter.setNextReader(current);
- validSegment &= aggregator.setNextReader(current);
- if (!validSegment) { // if categoryList or aggregtor say it's an invalid segment, skip all docs
- while (docID < maxDoc && iterator.next()) {
- docID = iterator.getDocID();
+ if (docID >= maxDoc) {
+ boolean iteratorDone = false;
+ do { // find the segment which contains this document
+ if (!contexts.hasNext()) {
+ throw new RuntimeException("ScoredDocIDs contains documents outside this reader's segments !?");
+ }
+ current = contexts.next();
+ maxDoc = current.docBase + current.reader().maxDoc();
+ if (docID < maxDoc) { // segment has docs, check if it has categories
+ boolean validSegment = categoryListIter.setNextReader(current);
+ validSegment &= aggregator.setNextReader(current);
+ if (!validSegment) { // if categoryList or aggregtor say it's an invalid segment, skip all docs
+ while (docID < maxDoc && iterator.next()) {
+ docID = iterator.getDocID();
+ }
+ if (docID < maxDoc) {
+ iteratorDone = true;
+ }
}
}
+ } while (docID >= maxDoc);
+ if (iteratorDone) { // iterator finished, terminate the loop
+ break;
}
}
docID -= current.docBase;
@@ -312,19 +321,17 @@ public class StandardFacetsAccumulator e
HashMap<CategoryListIterator, Aggregator> categoryLists = new HashMap<CategoryListIterator, Aggregator>();
+ FacetIndexingParams indexingParams = searchParams.getFacetIndexingParams();
for (FacetRequest facetRequest : searchParams.getFacetRequests()) {
- Aggregator categoryAggregator = facetRequest.createAggregator(
- isUsingComplements, facetArrays, taxonomyReader);
+ Aggregator categoryAggregator = facetRequest.createAggregator(isUsingComplements, facetArrays, taxonomyReader);
- CategoryListIterator cli = facetRequest.createCategoryListIterator(taxonomyReader, searchParams, partition);
+ CategoryListIterator cli = indexingParams.getCategoryListParams(facetRequest.categoryPath).createCategoryListIterator(partition);
// get the aggregator
Aggregator old = categoryLists.put(cli, categoryAggregator);
if (old != null && !old.equals(categoryAggregator)) {
- // TODO (Facet): create a more meaningful RE class, and throw it.
- throw new RuntimeException(
- "Overriding existing category list with different aggregator. THAT'S A NO NO!");
+ throw new RuntimeException("Overriding existing category list with different aggregator");
}
// if the aggregator is the same we're covered
}
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TopKFacetResultsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TopKFacetResultsHandler.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TopKFacetResultsHandler.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TopKFacetResultsHandler.java Wed Jan 16 10:09:43 2013
@@ -55,7 +55,7 @@ public class TopKFacetResultsHandler ext
public IntermediateFacetResult fetchPartitionResult(FacetArrays facetArrays, int offset)
throws IOException {
TopKFacetResult res = null;
- int ordinal = taxonomyReader.getOrdinal(facetRequest.getCategoryPath());
+ int ordinal = taxonomyReader.getOrdinal(facetRequest.categoryPath);
if (ordinal != TaxonomyReader.INVALID_ORDINAL) {
double value = 0;
if (isSelfPartition(ordinal, facetArrays, offset)) {
@@ -79,7 +79,7 @@ public class TopKFacetResultsHandler ext
@Override
public IntermediateFacetResult mergeResults(IntermediateFacetResult... tmpResults) throws IOException {
- int ordinal = taxonomyReader.getOrdinal(facetRequest.getCategoryPath());
+ int ordinal = taxonomyReader.getOrdinal(facetRequest.categoryPath);
MutableFacetResultNode resNode = new MutableFacetResultNode(ordinal, 0);
int totalFacets = 0;
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java Wed Jan 16 10:09:43 2013
@@ -34,35 +34,37 @@ import org.apache.lucene.util.collection
*/
/**
- * Generates {@link FacetResult} from the count arrays aggregated for a particular
- * {@link FacetRequest}.
- * The generated {@link FacetResult} is a subtree of the taxonomy tree.
- * Its root node, {@link FacetResult#getFacetResultNode()},
- * is the facet specified by {@link FacetRequest#getCategoryPath()},
- * and the enumerated children, {@link FacetResultNode#getSubResults()}, of each node in that
- * {@link FacetResult} are the top K ( = {@link FacetRequest#getNumResults()}) among its children
- * in the taxonomy.
- * Top in the sense {@link FacetRequest#getSortBy()},
- * which can be by the values aggregated in the count arrays, or by ordinal numbers;
- * also specified is the sort order, {@link FacetRequest#getSortOrder()},
- * ascending or descending, of these values or ordinals before their top K are selected.
- * The depth (number of levels excluding the root) of the
- * {@link FacetResult} tree is specified by {@link FacetRequest#getDepth()}.
+ * Generates {@link FacetResult} from the count arrays aggregated for a
+ * particular {@link FacetRequest}. The generated {@link FacetResult} is a
+ * subtree of the taxonomy tree. Its root node,
+ * {@link FacetResult#getFacetResultNode()}, is the facet specified by
+ * {@link FacetRequest#categoryPath}, and the enumerated children,
+ * {@link FacetResultNode#getSubResults()}, of each node in that
+ * {@link FacetResult} are the top K ( = {@link FacetRequest#getNumResults()})
+ * among its children in the taxonomy. Top in the sense
+ * {@link FacetRequest#getSortBy()}, which can be by the values aggregated in
+ * the count arrays, or by ordinal numbers; also specified is the sort order,
+ * {@link FacetRequest#getSortOrder()}, ascending or descending, of these values
+ * or ordinals before their top K are selected. The depth (number of levels
+ * excluding the root) of the {@link FacetResult} tree is specified by
+ * {@link FacetRequest#getDepth()}.
* <p>
- * Because the number of selected children of each node is restricted,
- * and not the overall number of nodes in the {@link FacetResult}, facets not selected
+ * Because the number of selected children of each node is restricted, and not
+ * the overall number of nodes in the {@link FacetResult}, facets not selected
* into {@link FacetResult} might have better values, or ordinals, (typically,
* higher counts), than facets that are selected into the {@link FacetResult}.
* <p>
- * The generated {@link FacetResult} also provides with
- * {@link FacetResult#getNumValidDescendants()}, which returns the total number of facets
- * that are descendants of the root node, no deeper than {@link FacetRequest#getDepth()}, and
- * which have valid value. The rootnode itself is not counted here.
- * Valid value is determined by the {@link FacetResultsHandler}.
- * {@link TopKInEachNodeHandler} defines valid as != 0.
+ * The generated {@link FacetResult} also provides with
+ * {@link FacetResult#getNumValidDescendants()}, which returns the total number
+ * of facets that are descendants of the root node, no deeper than
+ * {@link FacetRequest#getDepth()}, and which have valid value. The rootnode
+ * itself is not counted here. Valid value is determined by the
+ * {@link FacetResultsHandler}. {@link TopKInEachNodeHandler} defines valid as
+ * != 0.
* <p>
- * <b>NOTE:</b> this code relies on the assumption that {@link TaxonomyReader#INVALID_ORDINAL} == -1, a smaller
- * value than any valid ordinal.
+ * <b>NOTE:</b> this code relies on the assumption that
+ * {@link TaxonomyReader#INVALID_ORDINAL} == -1, a smaller value than any valid
+ * ordinal.
*
* @lucene.experimental
*/
@@ -109,7 +111,7 @@ public class TopKInEachNodeHandler exten
// get the root of the result tree to be returned, and the depth of that result tree
// (depth means number of node levels excluding the root).
- int rootNode = this.taxonomyReader.getOrdinal(this.facetRequest.getCategoryPath());
+ int rootNode = this.taxonomyReader.getOrdinal(facetRequest.categoryPath);
if (rootNode == TaxonomyReader.INVALID_ORDINAL) {
return null;
}
@@ -767,7 +769,7 @@ public class TopKInEachNodeHandler exten
@Override
public FacetResult renderFacetResult(IntermediateFacetResult tmpResult) throws IOException {
IntermediateFacetResultWithHash tmp = (IntermediateFacetResultWithHash) tmpResult;
- int ordinal = this.taxonomyReader.getOrdinal(this.facetRequest.getCategoryPath());
+ int ordinal = this.taxonomyReader.getOrdinal(this.facetRequest.categoryPath);
if ((tmp == null) || (ordinal == TaxonomyReader.INVALID_ORDINAL)) {
return null;
}
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TotalFacetCounts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TotalFacetCounts.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TotalFacetCounts.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TotalFacetCounts.java Wed Jan 16 10:09:43 2013
@@ -17,8 +17,6 @@ import org.apache.lucene.facet.index.par
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.aggregator.Aggregator;
import org.apache.lucene.facet.search.aggregator.CountingAggregator;
-import org.apache.lucene.facet.search.cache.CategoryListCache;
-import org.apache.lucene.facet.search.cache.CategoryListData;
import org.apache.lucene.facet.search.params.CountFacetRequest;
import org.apache.lucene.facet.search.params.FacetRequest;
import org.apache.lucene.facet.search.params.FacetSearchParams;
@@ -155,9 +153,8 @@ public class TotalFacetCounts {
private static final List<FacetRequest> DUMMY_REQ = Arrays.asList(
new FacetRequest[] { new CountFacetRequest(CategoryPath.EMPTY, 1) });
- static TotalFacetCounts compute(final IndexReader indexReader,
- final TaxonomyReader taxonomy, final FacetIndexingParams facetIndexingParams,
- final CategoryListCache clCache) throws IOException {
+ static TotalFacetCounts compute(final IndexReader indexReader, final TaxonomyReader taxonomy,
+ final FacetIndexingParams facetIndexingParams) throws IOException {
int partitionSize = PartitionsUtils.partitionSize(facetIndexingParams, taxonomy);
final int[][] counts = new int[(int) Math.ceil(taxonomy.getSize() /(float) partitionSize)][partitionSize];
FacetSearchParams newSearchParams = new FacetSearchParams(DUMMY_REQ, facetIndexingParams);
@@ -170,8 +167,7 @@ public class TotalFacetCounts {
Aggregator aggregator = new CountingAggregator(counts[partition]);
HashMap<CategoryListIterator, Aggregator> map = new HashMap<CategoryListIterator, Aggregator>();
for (CategoryListParams clp: facetIndexingParams.getAllCategoryListParams()) {
- final CategoryListIterator cli = clIteraor(clCache, clp, partition);
- map.put(cli, aggregator);
+ map.put(clp.createCategoryListIterator(partition), aggregator);
}
return map;
}
@@ -181,14 +177,4 @@ public class TotalFacetCounts {
return new TotalFacetCounts(taxonomy, facetIndexingParams, counts, CreationType.Computed);
}
- static CategoryListIterator clIteraor(CategoryListCache clCache, CategoryListParams clp, int partition)
- throws IOException {
- if (clCache != null) {
- CategoryListData cld = clCache.get(clp);
- if (cld != null) {
- return cld.iterator(partition);
- }
- }
- return clp.createCategoryListIterator(partition);
- }
}
\ No newline at end of file
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TotalFacetCountsCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TotalFacetCountsCache.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TotalFacetCountsCache.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/TotalFacetCountsCache.java Wed Jan 16 10:09:43 2013
@@ -7,12 +7,10 @@ import java.util.LinkedHashMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
-import org.apache.lucene.index.IndexReader;
-
import org.apache.lucene.facet.index.params.CategoryListParams;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
-import org.apache.lucene.facet.search.cache.CategoryListCache;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
+import org.apache.lucene.index.IndexReader;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -80,16 +78,20 @@ public final class TotalFacetCountsCache
}
/**
- * Get the total facet counts for a reader/taxonomy pair and facet indexing parameters.
- * If not in cache, computed here and added to the cache for later use.
- * @param indexReader the documents index
- * @param taxonomy the taxonomy index
- * @param facetIndexingParams facet indexing parameters
- * @param clCache category list cache for faster computation, can be null
+ * Get the total facet counts for a reader/taxonomy pair and facet indexing
+ * parameters. If not in cache, computed here and added to the cache for later
+ * use.
+ *
+ * @param indexReader
+ * the documents index
+ * @param taxonomy
+ * the taxonomy index
+ * @param facetIndexingParams
+ * facet indexing parameters
* @return the total facet counts.
*/
public TotalFacetCounts getTotalCounts(IndexReader indexReader, TaxonomyReader taxonomy,
- FacetIndexingParams facetIndexingParams, CategoryListCache clCache) throws IOException {
+ FacetIndexingParams facetIndexingParams) throws IOException {
// create the key
TFCKey key = new TFCKey(indexReader, taxonomy, facetIndexingParams);
// it is important that this call is not synchronized, so that available TFC
@@ -99,7 +101,7 @@ public final class TotalFacetCountsCache
markRecentlyUsed(key);
return tfc;
}
- return computeAndCache(key, clCache);
+ return computeAndCache(key);
}
/**
@@ -149,10 +151,10 @@ public final class TotalFacetCountsCache
* matter this method is synchronized, which is not too bad, because there is
* lots of work done in the computations.
*/
- private synchronized TotalFacetCounts computeAndCache(TFCKey key, CategoryListCache clCache) throws IOException {
+ private synchronized TotalFacetCounts computeAndCache(TFCKey key) throws IOException {
TotalFacetCounts tfc = cache.get(key);
if (tfc == null) {
- tfc = TotalFacetCounts.compute(key.indexReader, key.taxonomy, key.facetIndexingParams, clCache);
+ tfc = TotalFacetCounts.compute(key.indexReader, key.taxonomy, key.facetIndexingParams);
lruKeys.add(key);
cache.put(key,tfc);
trimCache();
@@ -161,16 +163,22 @@ public final class TotalFacetCountsCache
}
/**
- * Load {@link TotalFacetCounts} matching input parameters from the provided outputFile
- * and add them into the cache for the provided indexReader, taxonomy, and facetIndexingParams.
- * If a {@link TotalFacetCounts} for these parameters already exists in the cache, it will be
- * replaced by the loaded one.
- * @param inputFile file from which to read the data
- * @param indexReader the documents index
- * @param taxonomy the taxonomy index
- * @param facetIndexingParams the facet indexing parameters
- * @throws IOException on error
- * @see #store(File, IndexReader, TaxonomyReader, FacetIndexingParams, CategoryListCache)
+ * Load {@link TotalFacetCounts} matching input parameters from the provided
+ * outputFile and add them into the cache for the provided indexReader,
+ * taxonomy, and facetIndexingParams. If a {@link TotalFacetCounts} for these
+ * parameters already exists in the cache, it will be replaced by the loaded
+ * one.
+ *
+ * @param inputFile
+ * file from which to read the data
+ * @param indexReader
+ * the documents index
+ * @param taxonomy
+ * the taxonomy index
+ * @param facetIndexingParams
+ * the facet indexing parameters
+ * @throws IOException
+ * on error
*/
public synchronized void load(File inputFile, IndexReader indexReader, TaxonomyReader taxonomy,
FacetIndexingParams facetIndexingParams) throws IOException {
@@ -185,21 +193,27 @@ public final class TotalFacetCountsCache
}
/**
- * Store the {@link TotalFacetCounts} matching input parameters into the provided outputFile,
- * making them available for a later call to {@link #load(File, IndexReader, TaxonomyReader, FacetIndexingParams)}.
- * If these {@link TotalFacetCounts} are available in the cache, they are used. But if they are
- * not in the cache, this call will first compute them (which will also add them to the cache).
- * @param outputFile file to store in.
- * @param indexReader the documents index
- * @param taxonomy the taxonomy index
- * @param facetIndexingParams the facet indexing parameters
- * @param clCache category list cache for faster computation, can be null
- * @throws IOException on error
+ * Store the {@link TotalFacetCounts} matching input parameters into the
+ * provided outputFile, making them available for a later call to
+ * {@link #load(File, IndexReader, TaxonomyReader, FacetIndexingParams)}. If
+ * these {@link TotalFacetCounts} are available in the cache, they are used.
+ * But if they are not in the cache, this call will first compute them (which
+ * will also add them to the cache).
+ *
+ * @param outputFile
+ * file to store in.
+ * @param indexReader
+ * the documents index
+ * @param taxonomy
+ * the taxonomy index
+ * @param facetIndexingParams
+ * the facet indexing parameters
+ * @throws IOException
+ * on error
* @see #load(File, IndexReader, TaxonomyReader, FacetIndexingParams)
- * @see #getTotalCounts(IndexReader, TaxonomyReader, FacetIndexingParams, CategoryListCache)
*/
public void store(File outputFile, IndexReader indexReader, TaxonomyReader taxonomy,
- FacetIndexingParams facetIndexingParams, CategoryListCache clCache) throws IOException {
+ FacetIndexingParams facetIndexingParams) throws IOException {
File parentFile = outputFile.getParentFile();
if (
( outputFile.exists() && (!outputFile.isFile() || !outputFile.canWrite())) ||
@@ -207,7 +221,7 @@ public final class TotalFacetCountsCache
) {
throw new IllegalArgumentException("Exepecting a writable file: "+outputFile);
}
- TotalFacetCounts tfc = getTotalCounts(indexReader, taxonomy, facetIndexingParams, clCache);
+ TotalFacetCounts tfc = getTotalCounts(indexReader, taxonomy, facetIndexingParams);
TotalFacetCounts.storeToFile(outputFile, tfc);
}
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/associations/AssociationFloatSumAggregator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/associations/AssociationFloatSumAggregator.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/associations/AssociationFloatSumAggregator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/associations/AssociationFloatSumAggregator.java Wed Jan 16 10:09:43 2013
@@ -3,7 +3,7 @@ package org.apache.lucene.facet.search.a
import java.io.IOException;
import org.apache.lucene.facet.associations.CategoryFloatAssociation;
-import org.apache.lucene.facet.associations.FloatAssociationsPayloadIterator;
+import org.apache.lucene.facet.associations.FloatAssociationsIterator;
import org.apache.lucene.facet.index.params.CategoryListParams;
import org.apache.lucene.facet.search.aggregator.Aggregator;
import org.apache.lucene.index.AtomicReaderContext;
@@ -37,15 +37,15 @@ public class AssociationFloatSumAggregat
protected final String field;
protected final float[] sumArray;
- protected final FloatAssociationsPayloadIterator associations;
+ protected final FloatAssociationsIterator associations;
public AssociationFloatSumAggregator(float[] sumArray) throws IOException {
- this(CategoryListParams.DEFAULT_TERM.field(), sumArray);
+ this(CategoryListParams.DEFAULT_FIELD, sumArray);
}
public AssociationFloatSumAggregator(String field, float[] sumArray) throws IOException {
this.field = field;
- associations = new FloatAssociationsPayloadIterator(field, new CategoryFloatAssociation());
+ associations = new FloatAssociationsIterator(field, new CategoryFloatAssociation());
this.sumArray = sumArray;
}
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/associations/AssociationIntSumAggregator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/associations/AssociationIntSumAggregator.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/associations/AssociationIntSumAggregator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/associations/AssociationIntSumAggregator.java Wed Jan 16 10:09:43 2013
@@ -3,7 +3,7 @@ package org.apache.lucene.facet.search.a
import java.io.IOException;
import org.apache.lucene.facet.associations.CategoryIntAssociation;
-import org.apache.lucene.facet.associations.IntAssociationsPayloadIterator;
+import org.apache.lucene.facet.associations.IntAssociationsIterator;
import org.apache.lucene.facet.index.params.CategoryListParams;
import org.apache.lucene.facet.search.aggregator.Aggregator;
import org.apache.lucene.index.AtomicReaderContext;
@@ -37,15 +37,15 @@ public class AssociationIntSumAggregator
protected final String field;
protected final int[] sumArray;
- protected final IntAssociationsPayloadIterator associations;
+ protected final IntAssociationsIterator associations;
public AssociationIntSumAggregator(int[] sumArray) throws IOException {
- this(CategoryListParams.DEFAULT_TERM.field(), sumArray);
+ this(CategoryListParams.DEFAULT_FIELD, sumArray);
}
public AssociationIntSumAggregator(String field, int[] sumArray) throws IOException {
this.field = field;
- associations = new IntAssociationsPayloadIterator(field, new CategoryIntAssociation());
+ associations = new IntAssociationsIterator(field, new CategoryIntAssociation());
this.sumArray = sumArray;
}
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/FacetRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/FacetRequest.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/FacetRequest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/FacetRequest.java Wed Jan 16 10:09:43 2013
@@ -2,15 +2,11 @@ package org.apache.lucene.facet.search.p
import java.io.IOException;
-import org.apache.lucene.facet.index.params.CategoryListParams;
-import org.apache.lucene.facet.search.CategoryListIterator;
import org.apache.lucene.facet.search.FacetArrays;
import org.apache.lucene.facet.search.FacetResultsHandler;
import org.apache.lucene.facet.search.TopKFacetResultsHandler;
import org.apache.lucene.facet.search.TopKInEachNodeHandler;
import org.apache.lucene.facet.search.aggregator.Aggregator;
-import org.apache.lucene.facet.search.cache.CategoryListCache;
-import org.apache.lucene.facet.search.cache.CategoryListData;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
@@ -64,7 +60,7 @@ public abstract class FacetRequest imple
*/
public static final ResultMode DEFAULT_RESULT_MODE = ResultMode.PER_NODE_IN_TREE;
- private final CategoryPath categoryPath;
+ public final CategoryPath categoryPath;
private final int numResults;
private int numLabel;
private int depth;
@@ -134,17 +130,6 @@ public abstract class FacetRequest imple
}
/**
- * The root category of this facet request. The categories that are returned
- * as a result of this request will all be descendants of this root.
- * <p>
- * <b>NOTE:</b> you should not modify the returned {@link CategoryPath}, or
- * otherwise some methonds may not work properly, e.g. {@link #hashCode()}.
- */
- public final CategoryPath getCategoryPath() {
- return categoryPath;
- }
-
- /**
* How deeply to look under the given category. If the depth is 0,
* only the category itself is counted. If the depth is 1, its immediate
* children are also counted, and so on. If the depth is Integer.MAX_VALUE,
@@ -160,24 +145,22 @@ public abstract class FacetRequest imple
* will have their category paths calculated, and the rest will only be
* available as ordinals (category numbers) and will have null paths.
* <P>
- * If Integer.MAX_VALUE is specified, all
- * results are labled.
+ * If Integer.MAX_VALUE is specified, all results are labled.
* <P>
- * The purpose of this parameter is to avoid having to run the whole
- * faceted search again when the user asks for more values for the facet;
- * The application can ask (getNumResults()) for more values than it needs
- * to show, but keep getNumLabel() only the number it wants to immediately
- * show. The slow-down caused by finding more values is negligible, because
- * the slowest part - finding the categories' paths, is avoided.
+ * The purpose of this parameter is to avoid having to run the whole faceted
+ * search again when the user asks for more values for the facet; The
+ * application can ask (getNumResults()) for more values than it needs to
+ * show, but keep getNumLabel() only the number it wants to immediately show.
+ * The slow-down caused by finding more values is negligible, because the
+ * slowest part - finding the categories' paths, is avoided.
* <p>
- * Depending on the {@link #getResultMode() LimitsMode},
- * this limit is applied globally or per results node.
- * In the global mode, if this limit is 3,
- * only 3 top results would be labeled.
- * In the per-node mode, if this limit is 3,
- * 3 top children of {@link #getCategoryPath() the target category} would be labeled,
- * as well as 3 top children of each of them, and so forth, until the depth defined
- * by {@link #getDepth()}.
+ * Depending on the {@link #getResultMode() LimitsMode}, this limit is applied
+ * globally or per results node. In the global mode, if this limit is 3, only
+ * 3 top results would be labeled. In the per-node mode, if this limit is 3, 3
+ * top children of {@link #categoryPath the target category} would be labeled,
+ * as well as 3 top children of each of them, and so forth, until the depth
+ * defined by {@link #getDepth()}.
+ *
* @see #getResultMode()
*/
public final int getNumLabel() {
@@ -185,20 +168,18 @@ public abstract class FacetRequest imple
}
/**
- * The number of sub-categories to return (at most).
- * If the sub-categories are returned.
+ * The number of sub-categories to return (at most). If the sub-categories are
+ * returned.
* <p>
- * If Integer.MAX_VALUE is specified, all
- * sub-categories are returned.
+ * If Integer.MAX_VALUE is specified, all sub-categories are returned.
* <p>
- * Depending on the {@link #getResultMode() LimitsMode},
- * this limit is applied globally or per results node.
- * In the global mode, if this limit is 3,
- * only 3 top results would be computed.
- * In the per-node mode, if this limit is 3,
- * 3 top children of {@link #getCategoryPath() the target category} would be returned,
- * as well as 3 top children of each of them, and so forth, until the depth defined
- * by {@link #getDepth()}.
+ * Depending on the {@link #getResultMode() LimitsMode}, this limit is applied
+ * globally or per results node. In the global mode, if this limit is 3, only
+ * 3 top results would be computed. In the per-node mode, if this limit is 3,
+ * 3 top children of {@link #categoryPath the target category} would be
+ * returned, as well as 3 top children of each of them, and so forth, until
+ * the depth defined by {@link #getDepth()}.
+ *
* @see #getResultMode()
*/
public final int getNumResults() {
@@ -320,24 +301,6 @@ public abstract class FacetRequest imple
throws IOException;
/**
- * Create the category list iterator for the specified partition. If a non
- * null cache is provided which contains the required data, use it for the
- * iteration.
- */
- public CategoryListIterator createCategoryListIterator(TaxonomyReader taxo, FacetSearchParams sParams, int partition)
- throws IOException {
- CategoryListCache clCache = sParams.getCategoryListCache();
- CategoryListParams clParams = sParams.getFacetIndexingParams().getCategoryListParams(categoryPath);
- if (clCache != null) {
- CategoryListData clData = clCache.get(clParams);
- if (clData != null) {
- return clData.iterator(partition);
- }
- }
- return clParams.createCategoryListIterator(partition);
- }
-
- /**
* Return the value of a category used for facets computations for this
* request. For a count request this would be the count for that facet, i.e.
* an integer number. but for other requests this can be the result of a more
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/FacetSearchParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/FacetSearchParams.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/FacetSearchParams.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/FacetSearchParams.java Wed Jan 16 10:09:43 2013
@@ -4,7 +4,6 @@ import java.util.Arrays;
import java.util.List;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
-import org.apache.lucene.facet.search.cache.CategoryListCache;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -72,14 +71,6 @@ public class FacetSearchParams {
}
/**
- * Returns the {@link CategoryListCache}. By default returns {@code null}, you
- * should override if you want to use a cache.
- */
- public CategoryListCache getCategoryListCache() {
- return null;
- }
-
- /**
* Returns the {@link FacetIndexingParams} that were passed to the
* constructor.
*/
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/associations/AssociationFloatSumFacetRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/associations/AssociationFloatSumFacetRequest.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/associations/AssociationFloatSumFacetRequest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/associations/AssociationFloatSumFacetRequest.java Wed Jan 16 10:09:43 2013
@@ -28,7 +28,10 @@ import org.apache.lucene.facet.taxonomy.
/**
* A {@link FacetRequest} for weighting facets according to their float
- * association by summing the association values.
+ * association by summing the association values. Note that this class caches
+ * the associations data in-memory by default. You can override
+ * {@link #createAggregator(boolean, FacetArrays, TaxonomyReader)} to return an
+ * {@link AssociationFloatSumAggregator} which does otherwise.
*
* @lucene.experimental
*/
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/associations/AssociationIntSumFacetRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/associations/AssociationIntSumFacetRequest.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/associations/AssociationIntSumFacetRequest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/params/associations/AssociationIntSumFacetRequest.java Wed Jan 16 10:09:43 2013
@@ -4,6 +4,7 @@ import java.io.IOException;
import org.apache.lucene.facet.search.FacetArrays;
import org.apache.lucene.facet.search.aggregator.Aggregator;
+import org.apache.lucene.facet.search.aggregator.associations.AssociationFloatSumAggregator;
import org.apache.lucene.facet.search.aggregator.associations.AssociationIntSumAggregator;
import org.apache.lucene.facet.search.params.FacetRequest;
import org.apache.lucene.facet.taxonomy.CategoryPath;
@@ -28,7 +29,10 @@ import org.apache.lucene.facet.taxonomy.
/**
* A {@link FacetRequest} for weighting facets according to their integer
- * association by summing the association values.
+ * association by summing the association values. Note that this class caches
+ * the associations data in-memory by default. You can override
+ * {@link #createAggregator(boolean, FacetArrays, TaxonomyReader)} to return an
+ * {@link AssociationFloatSumAggregator} which does otherwise.
*
* @lucene.experimental
*/
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/results/FacetResult.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/results/FacetResult.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/results/FacetResult.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/results/FacetResult.java Wed Jan 16 10:09:43 2013
@@ -39,7 +39,7 @@ public class FacetResult {
/**
* Facet result node matching the root of the {@link #getFacetRequest() facet request}.
* @see #getFacetRequest()
- * @see FacetRequest#getCategoryPath()
+ * @see FacetRequest#categoryPath
*/
public final FacetResultNode getFacetResultNode() {
return this.rootNode;
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/Sampler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/Sampler.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/Sampler.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/sampling/Sampler.java Wed Jan 16 10:09:43 2013
@@ -4,9 +4,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.index.IndexReader;
-
-import org.apache.lucene.facet.search.CategoryListIterator;
import org.apache.lucene.facet.search.FacetArrays;
import org.apache.lucene.facet.search.ScoredDocIDs;
import org.apache.lucene.facet.search.aggregator.Aggregator;
@@ -16,6 +13,7 @@ import org.apache.lucene.facet.search.re
import org.apache.lucene.facet.search.results.FacetResultNode;
import org.apache.lucene.facet.search.results.MutableFacetResultNode;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
+import org.apache.lucene.index.IndexReader;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -205,7 +203,7 @@ public abstract class Sampler {
private static class OverSampledFacetRequest extends FacetRequest {
final FacetRequest orig;
public OverSampledFacetRequest(FacetRequest orig, int num) {
- super(orig.getCategoryPath(), num);
+ super(orig.categoryPath, num);
this.orig = orig;
setDepth(orig.getDepth());
setNumLabel(orig.getNumLabel());
@@ -215,12 +213,6 @@ public abstract class Sampler {
}
@Override
- public CategoryListIterator createCategoryListIterator(TaxonomyReader taxo, FacetSearchParams sParams,
- int partition) throws IOException {
- return orig.createCategoryListIterator(taxo, sParams, partition);
- }
-
- @Override
public Aggregator createAggregator(boolean useComplements, FacetArrays arrays, TaxonomyReader taxonomy)
throws IOException {
return orig.createAggregator(useComplements, arrays, taxonomy);
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/util/PartitionsUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/util/PartitionsUtils.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/util/PartitionsUtils.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/util/PartitionsUtils.java Wed Jan 16 10:09:43 2013
@@ -1,6 +1,5 @@
package org.apache.lucene.facet.util;
-import org.apache.lucene.facet.index.params.CategoryListParams;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
@@ -28,15 +27,9 @@ import org.apache.lucene.facet.taxonomy.
*/
public final class PartitionsUtils {
- /**
- * Get the offset for a given partition. That is, what is the minimum number an
- * ordinal could be for a particular partition.
- */
- public final static int partitionOffset(FacetIndexingParams iParams,
- int partitionNumber, final TaxonomyReader taxonomyReader) {
- return partitionNumber * partitionSize(iParams, taxonomyReader);
- }
-
+ /** The prefix that is added to the name of the partition. */
+ public static final String PART_NAME_PREFIX = "$part";
+
/**
* Get the partition size in this parameter, or return the size of the taxonomy, which
* is smaller. (Guarantees usage of as little memory as possible at search time).
@@ -58,21 +51,21 @@ public final class PartitionsUtils {
/**
* Partition name by category ordinal
*/
- public final static String partitionNameByOrdinal(
- FacetIndexingParams iParams, CategoryListParams clParams, int ordinal) {
+ public final static String partitionNameByOrdinal(FacetIndexingParams iParams, int ordinal) {
int partition = partitionNumber(iParams, ordinal);
- return partitionName(clParams, partition);
+ return partitionName(partition);
}
- /**
- * Partition name by its number
- */
- public final static String partitionName(CategoryListParams clParams, int partition) {
- String term = clParams.getTerm().text();
+ /** Partition name by its number */
+ public final static String partitionName(int partition) {
+ // TODO would be good if this method isn't called when partitions are not enabled.
+ // perhaps through some specialization code.
if (partition == 0) {
- return term; // for backwards compatibility we do not add a partition number in this case
+ // since regular faceted search code goes through this method too,
+ // return the same value for partition 0 and when there are no partitions
+ return "";
}
- return term + partition;
+ return PART_NAME_PREFIX + Integer.toString(partition);
}
}
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java Wed Jan 16 10:09:43 2013
@@ -44,6 +44,7 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -65,6 +66,7 @@ import org.junit.BeforeClass;
*/
/** Base faceted search test. */
+@SuppressCodecs("Lucene3x")
public abstract class FacetTestBase extends LuceneTestCase {
/** Holds a search and taxonomy Directories pair. */
@@ -266,13 +268,12 @@ public abstract class FacetTestBase exte
FacetIndexingParams iParams = getFacetIndexingParams(Integer.MAX_VALUE);
String delim = String.valueOf(iParams.getFacetDelimChar());
Map<CategoryPath, Integer> res = new HashMap<CategoryPath, Integer>();
- HashSet<Term> handledTerms = new HashSet<Term>();
+ HashSet<String> handledTerms = new HashSet<String>();
for (CategoryListParams clp : iParams.getAllCategoryListParams()) {
- Term baseTerm = new Term(clp.getTerm().field());
- if (!handledTerms.add(baseTerm)) {
+ if (!handledTerms.add(clp.field)) {
continue; // already handled this term (for another list)
}
- Terms terms = MultiFields.getTerms(indexReader, baseTerm.field());
+ Terms terms = MultiFields.getTerms(indexReader, clp.field);
if (terms == null) {
continue;
}
@@ -297,7 +298,7 @@ public abstract class FacetTestBase exte
FacetResultNode topResNode = fr.getFacetResultNode();
FacetRequest freq = fr.getFacetRequest();
if (VERBOSE) {
- System.out.println(freq.getCategoryPath().toString()+ "\t\t" + topResNode);
+ System.out.println(freq.categoryPath.toString()+ "\t\t" + topResNode);
}
assertCountsAndCardinality(facetCountsTruth, topResNode, freq.getNumResults());
}
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java Wed Jan 16 10:09:43 2013
@@ -2,14 +2,9 @@ package org.apache.lucene.facet;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.facet.index.FacetFields;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.FacetsCollector;
import org.apache.lucene.facet.search.params.CountFacetRequest;
@@ -23,7 +18,6 @@ import org.apache.lucene.facet.taxonomy.
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
@@ -116,15 +110,6 @@ public class FacetTestUtils {
return collectors;
}
- public static void add(FacetIndexingParams iParams, RandomIndexWriter iw,
- TaxonomyWriter tw, String... strings) throws IOException {
- Document d = new Document();
- FacetFields facetFields = new FacetFields(tw, iParams);
- facetFields.addFields(d, Collections.singletonList(new CategoryPath(strings)));
- d.add(new TextField("content", "alpha", Field.Store.YES));
- iw.addDocument(d);
- }
-
public static class IndexTaxonomyReaderPair {
public DirectoryReader indexReader;
public DirectoryTaxonomyReader taxReader;
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestAdaptiveExample.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestAdaptiveExample.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestAdaptiveExample.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestAdaptiveExample.java Wed Jan 16 10:09:43 2013
@@ -1,10 +1,9 @@
package org.apache.lucene.facet.example;
-import org.junit.Test;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.example.ExampleResult;
import org.apache.lucene.facet.example.adaptive.AdaptiveMain;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
+import org.junit.Test;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -27,6 +26,7 @@ import org.apache.lucene.facet.example.a
* Test that the adaptive example works as expected. This test helps to verify
* that examples code is alive!
*/
+@SuppressCodecs("Lucene3x")
public class TestAdaptiveExample extends LuceneTestCase {
@Test
@@ -37,4 +37,5 @@ public class TestAdaptiveExample extends
assertEquals("Wrong number of results!",1, res.getFacetResults().size());
assertEquals("Wrong number of facets!",3, res.getFacetResults().get(0).getNumValidDescendants());
}
+
}
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestAssociationExample.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestAssociationExample.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestAssociationExample.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestAssociationExample.java Wed Jan 16 10:09:43 2013
@@ -3,6 +3,7 @@ package org.apache.lucene.facet.example;
import org.junit.Test;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.facet.example.ExampleResult;
import org.apache.lucene.facet.example.association.CategoryAssociationsMain;
import org.apache.lucene.facet.search.results.FacetResultNode;
@@ -24,10 +25,7 @@ import org.apache.lucene.facet.search.re
* limitations under the License.
*/
-/**
- * Test that the association example works as expected. This test helps to
- * verify that examples code is alive!
- */
+@SuppressCodecs("Lucene3x")
public class TestAssociationExample extends LuceneTestCase {
private static final double[] EXPECTED_INT_SUM_RESULTS = { 4, 2};
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestMultiCLExample.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestMultiCLExample.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestMultiCLExample.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestMultiCLExample.java Wed Jan 16 10:09:43 2013
@@ -6,6 +6,7 @@ import java.util.List;
import org.junit.Test;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.facet.example.multiCL.MultiCLMain;
import org.apache.lucene.facet.search.results.FacetResult;
import org.apache.lucene.facet.search.results.FacetResultNode;
@@ -27,10 +28,7 @@ import org.apache.lucene.facet.search.re
* limitations under the License.
*/
-/**
- * Test that the multi-category list example works as expected. This test helps
- * to verify that examples code is alive!
- */
+@SuppressCodecs("Lucene3x")
public class TestMultiCLExample extends LuceneTestCase {
@Test
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java?rev=1433878&r1=1433877&r2=1433878&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java Wed Jan 16 10:09:43 2013
@@ -5,6 +5,7 @@ import java.util.Iterator;
import org.junit.Test;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.facet.example.ExampleResult;
import org.apache.lucene.facet.example.simple.SimpleMain;
import org.apache.lucene.facet.search.results.FacetResult;
@@ -27,10 +28,7 @@ import org.apache.lucene.facet.search.re
* limitations under the License.
*/
-/**
- * Test that the simple example works as expected. This test helps to verify
- * that examples code is alive!
- */
+@SuppressCodecs("Lucene3x")
public class TestSimpleExample extends LuceneTestCase {
@Test