You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ja...@apache.org on 2013/05/30 09:53:46 UTC
svn commit: r1487777 [25/50] - in /lucene/dev/branches/security: ./
dev-tools/ dev-tools/eclipse/dot.settings/ dev-tools/idea/.idea/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/replicator/
dev-tools/maven/ dev-tools/maven/lucene/ dev-tools/ma...
Modified: lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/search/TopKFacetResultsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/search/TopKFacetResultsHandler.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/search/TopKFacetResultsHandler.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/search/TopKFacetResultsHandler.java Thu May 30 07:53:18 2013
@@ -5,8 +5,8 @@ import java.util.ArrayList;
import org.apache.lucene.facet.partitions.IntermediateFacetResult;
import org.apache.lucene.facet.partitions.PartitionsFacetResultsHandler;
+import org.apache.lucene.facet.taxonomy.ParallelTaxonomyArrays;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
-import org.apache.lucene.facet.taxonomy.directory.ParallelTaxonomyArrays;
import org.apache.lucene.facet.util.ResultSortUtils;
/*
@@ -256,7 +256,6 @@ public class TopKFacetResultsHandler ext
* Create a Facet Result.
* @param facetRequest Request for which this result was obtained.
* @param facetResultNode top result node for this facet result.
- * @param totalFacets - number of children of the targetFacet, up till the requested depth.
*/
TopKFacetResult(FacetRequest facetRequest, FacetResultNode facetResultNode, int totalFacets) {
super(facetRequest, facetResultNode, totalFacets);
Modified: lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java Thu May 30 07:53:18 2013
@@ -9,8 +9,8 @@ import org.apache.lucene.facet.collectio
import org.apache.lucene.facet.partitions.IntermediateFacetResult;
import org.apache.lucene.facet.partitions.PartitionsFacetResultsHandler;
import org.apache.lucene.facet.search.FacetRequest.SortOrder;
+import org.apache.lucene.facet.taxonomy.ParallelTaxonomyArrays;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
-import org.apache.lucene.facet.taxonomy.directory.ParallelTaxonomyArrays;
import org.apache.lucene.util.PriorityQueue;
/*
@@ -706,8 +706,7 @@ public class TopKInEachNodeHandler exten
value = tmp.rootNodeValue;
}
FacetResultNode root = generateNode(ordinal, value, tmp.mapToAACOs);
- return new FacetResult (tmp.facetRequest, root, tmp.totalNumOfFacetsConsidered);
-
+ return new FacetResult(tmp.facetRequest, root, tmp.totalNumOfFacetsConsidered);
}
private FacetResultNode generateNode(int ordinal, double val, IntToObjectMap<AACO> mapToAACOs) {
Modified: lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CategoryPath.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CategoryPath.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CategoryPath.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CategoryPath.java Thu May 30 07:53:18 2013
@@ -1,7 +1,5 @@
package org.apache.lucene.facet.taxonomy;
-import java.util.Arrays;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -19,6 +17,9 @@ import java.util.Arrays;
* limitations under the License.
*/
+import java.util.Arrays;
+import java.util.regex.Pattern;
+
/**
* Holds a sequence of string components, specifying the hierarchical name of a
* category.
@@ -73,7 +74,7 @@ public class CategoryPath implements Com
/** Construct from a given path, separating path components with {@code delimiter}. */
public CategoryPath(final String pathString, final char delimiter) {
- String[] comps = pathString.split(Character.toString(delimiter));
+ String[] comps = pathString.split(Pattern.quote(Character.toString(delimiter)));
if (comps.length == 1 && comps[0].isEmpty()) {
components = null;
length = 0;
@@ -121,10 +122,14 @@ public class CategoryPath implements Com
return length - other.length;
}
+ private void hasDelimiter(String offender, char delimiter) {
+ throw new IllegalArgumentException("delimiter character '" + delimiter + "' (U+" + Integer.toHexString(delimiter) + ") appears in path component \"" + offender + "\"");
+ }
+
private void noDelimiter(char[] buf, int offset, int len, char delimiter) {
for(int idx=0;idx<len;idx++) {
if (buf[offset+idx] == delimiter) {
- throw new IllegalArgumentException("delimiter character U+" + Integer.toHexString(delimiter) + " appears in path");
+ hasDelimiter(new String(buf, offset, len), delimiter);
}
}
}
@@ -237,6 +242,9 @@ public class CategoryPath implements Com
StringBuilder sb = new StringBuilder();
for (int i = 0; i < length; i++) {
+ if (components[i].indexOf(delimiter) != -1) {
+ hasDelimiter(components[i], delimiter);
+ }
sb.append(components[i]).append(delimiter);
}
sb.setLength(sb.length() - 1); // remove last delimiter
Modified: lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyReader.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyReader.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyReader.java Thu May 30 07:53:18 2013
@@ -5,7 +5,6 @@ import java.io.IOException;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.lucene.facet.taxonomy.directory.ParallelTaxonomyArrays;
import org.apache.lucene.store.AlreadyClosedException;
/*
@@ -66,6 +65,31 @@ import org.apache.lucene.store.AlreadyCl
*/
public abstract class TaxonomyReader implements Closeable {
+ /** An iterator over a category's children. */
+ public static class ChildrenIterator {
+
+ private final int[] siblings;
+ private int child;
+
+ ChildrenIterator(int child, int[] siblings) {
+ this.siblings = siblings;
+ this.child = child;
+ }
+
+ /**
+ * Return the next child ordinal, or {@link TaxonomyReader#INVALID_ORDINAL}
+ * if no more children.
+ */
+ public int next() {
+ int res = child;
+ if (child != TaxonomyReader.INVALID_ORDINAL) {
+ child = siblings[child];
+ }
+ return res;
+ }
+
+ }
+
/**
* The root category (the category with the empty path) always has the ordinal
* 0, to which we give a name ROOT_ORDINAL. {@link #getOrdinal(CategoryPath)}
@@ -168,6 +192,13 @@ public abstract class TaxonomyReader imp
*/
public abstract ParallelTaxonomyArrays getParallelTaxonomyArrays() throws IOException;
+ /** Returns an iterator over the children of the given ordinal. */
+ public ChildrenIterator getChildren(final int ordinal) throws IOException {
+ ParallelTaxonomyArrays arrays = getParallelTaxonomyArrays();
+ int child = ordinal >= 0 ? arrays.children()[ordinal] : INVALID_ORDINAL;
+ return new ChildrenIterator(child, arrays.siblings());
+ }
+
/**
* Retrieve user committed data.
*
@@ -186,25 +217,6 @@ public abstract class TaxonomyReader imp
*/
public abstract int getOrdinal(CategoryPath categoryPath) throws IOException;
- /**
- * Returns the ordinal of the parent category of the category with the given
- * ordinal, according to the following rules:
- *
- * <ul>
- * <li>If the given ordinal is the {@link #ROOT_ORDINAL}, an
- * {@link #INVALID_ORDINAL} is returned.
- * <li>If the given ordinal is a top-level category, the {@link #ROOT_ORDINAL}
- * is returned.
- * <li>If the given ordinal is an existing category, returns the ordinal of
- * its parent
- * </ul>
- *
- * @throws ArrayIndexOutOfBoundsException
- * if an invalid ordinal is given (negative or beyond the last
- * available ordinal)
- */
- public abstract int getParent(int ordinal) throws IOException;
-
/** Returns the path name of the category with the given ordinal. */
public abstract CategoryPath getPath(int ordinal) throws IOException;
@@ -234,4 +246,16 @@ public abstract class TaxonomyReader imp
refCount.incrementAndGet();
}
+ /** Expert: increments the refCount of this TaxonomyReader
+ * instance only if it has not been closed yet. Returns
+ * true on success. */
+ public final boolean tryIncRef() {
+ int count;
+ while ((count = refCount.get()) > 0) {
+ if (refCount.compareAndSet(count, count+1)) {
+ return true;
+ }
+ }
+ return false;
+ }
}
Modified: lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java Thu May 30 07:53:18 2013
@@ -7,8 +7,9 @@ import java.util.logging.Logger;
import org.apache.lucene.facet.collections.LRUHashMap;
import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.facet.taxonomy.ParallelTaxonomyArrays;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
-import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.CorruptIndexException; // javadocs
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexWriter;
@@ -62,7 +63,7 @@ public class DirectoryTaxonomyReader ext
private LRUHashMap<CategoryPath, Integer> ordinalCache;
private LRUHashMap<Integer, CategoryPath> categoryCache;
- private volatile ParallelTaxonomyArrays taxoArrays;
+ private volatile TaxonomyIndexArrays taxoArrays;
private char delimiter = Consts.DEFAULT_DELIMITER;
@@ -73,7 +74,7 @@ public class DirectoryTaxonomyReader ext
*/
DirectoryTaxonomyReader(DirectoryReader indexReader, DirectoryTaxonomyWriter taxoWriter,
LRUHashMap<CategoryPath,Integer> ordinalCache, LRUHashMap<Integer,CategoryPath> categoryCache,
- ParallelTaxonomyArrays taxoArrays) throws IOException {
+ TaxonomyIndexArrays taxoArrays) throws IOException {
this.indexReader = indexReader;
this.taxoWriter = taxoWriter;
this.taxoEpoch = taxoWriter == null ? -1 : taxoWriter.getTaxonomyEpoch();
@@ -82,7 +83,7 @@ public class DirectoryTaxonomyReader ext
this.ordinalCache = ordinalCache == null ? new LRUHashMap<CategoryPath,Integer>(DEFAULT_CACHE_VALUE) : ordinalCache;
this.categoryCache = categoryCache == null ? new LRUHashMap<Integer,CategoryPath>(DEFAULT_CACHE_VALUE) : categoryCache;
- this.taxoArrays = taxoArrays != null ? new ParallelTaxonomyArrays(indexReader, taxoArrays) : null;
+ this.taxoArrays = taxoArrays != null ? new TaxonomyIndexArrays(indexReader, taxoArrays) : null;
}
/**
@@ -130,7 +131,7 @@ public class DirectoryTaxonomyReader ext
// according to Java Concurrency in Practice, this might perform better on
// some JVMs, because the array initialization doesn't happen on the
// volatile member.
- ParallelTaxonomyArrays tmpArrays = new ParallelTaxonomyArrays(indexReader);
+ TaxonomyIndexArrays tmpArrays = new TaxonomyIndexArrays(indexReader);
taxoArrays = tmpArrays;
}
}
@@ -160,14 +161,8 @@ public class DirectoryTaxonomyReader ext
protected DirectoryTaxonomyReader doOpenIfChanged() throws IOException {
ensureOpen();
- final DirectoryReader r2;
- if (taxoWriter == null) {
- // not NRT
- r2 = DirectoryReader.openIfChanged(indexReader);
- } else {
- // NRT
- r2 = DirectoryReader.openIfChanged(indexReader, taxoWriter.getInternalIndexWriter(), false);
- }
+ // This works for both NRT and non-NRT readers (i.e. an NRT reader remains NRT).
+ final DirectoryReader r2 = DirectoryReader.openIfChanged(indexReader);
if (r2 == null) {
return null; // no changes, nothing to do
}
@@ -293,12 +288,6 @@ public class DirectoryTaxonomyReader ext
}
@Override
- public int getParent(int ordinal) throws IOException {
- ensureOpen();
- return getParallelTaxonomyArrays().parents()[ordinal];
- }
-
- @Override
public CategoryPath getPath(int ordinal) throws IOException {
ensureOpen();
Modified: lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java Thu May 30 07:53:18 2013
@@ -29,7 +29,7 @@ import org.apache.lucene.facet.taxonomy.
import org.apache.lucene.facet.taxonomy.writercache.lru.LruTaxonomyWriterCache;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.CorruptIndexException; // javadocs
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader;
@@ -44,7 +44,7 @@ import org.apache.lucene.index.TermsEnum
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.store.LockObtainFailedException; // javadocs
import org.apache.lucene.store.NativeFSLockFactory;
import org.apache.lucene.store.SimpleFSLockFactory;
import org.apache.lucene.util.BytesRef;
@@ -129,7 +129,7 @@ public class DirectoryTaxonomyWriter imp
*/
private volatile boolean cacheIsComplete;
private volatile boolean isClosed = false;
- private volatile ParallelTaxonomyArrays taxoArrays;
+ private volatile TaxonomyIndexArrays taxoArrays;
private volatile int nextID;
/** Reads the commit data from a Directory. */
@@ -225,7 +225,7 @@ public class DirectoryTaxonomyWriter imp
}
// no commit data, or no epoch in it means an old taxonomy, so set its epoch to 1, for lack
// of a better value.
- indexEpoch = epochStr == null ? 1 : Long.parseLong(epochStr);
+ indexEpoch = epochStr == null ? 1 : Long.parseLong(epochStr, 16);
}
if (openMode == OpenMode.CREATE) {
@@ -354,8 +354,7 @@ public class DirectoryTaxonomyWriter imp
@Override
public synchronized void close() throws IOException {
if (!isClosed) {
- indexWriter.setCommitData(combinedCommitData(indexWriter.getCommitData()));
- indexWriter.commit();
+ commit();
doClose();
}
}
@@ -616,7 +615,11 @@ public class DirectoryTaxonomyWriter imp
@Override
public synchronized void commit() throws IOException {
ensureOpen();
- indexWriter.setCommitData(combinedCommitData(indexWriter.getCommitData()));
+ // LUCENE-4972: if we always call setCommitData, we create empty commits
+ String epochStr = indexWriter.getCommitData().get(INDEX_EPOCH);
+ if (epochStr == null || Long.parseLong(epochStr, 16) != indexEpoch) {
+ indexWriter.setCommitData(combinedCommitData(indexWriter.getCommitData()));
+ }
indexWriter.commit();
}
@@ -626,7 +629,7 @@ public class DirectoryTaxonomyWriter imp
if (commitData != null) {
m.putAll(commitData);
}
- m.put(INDEX_EPOCH, Long.toString(indexEpoch));
+ m.put(INDEX_EPOCH, Long.toString(indexEpoch, 16));
return m;
}
@@ -647,7 +650,11 @@ public class DirectoryTaxonomyWriter imp
@Override
public synchronized void prepareCommit() throws IOException {
ensureOpen();
- indexWriter.setCommitData(combinedCommitData(indexWriter.getCommitData()));
+ // LUCENE-4972: if we always call setCommitData, we create empty commits
+ String epochStr = indexWriter.getCommitData().get(INDEX_EPOCH);
+ if (epochStr == null || Long.parseLong(epochStr, 16) != indexEpoch) {
+ indexWriter.setCommitData(combinedCommitData(indexWriter.getCommitData()));
+ }
indexWriter.prepareCommit();
}
@@ -744,7 +751,7 @@ public class DirectoryTaxonomyWriter imp
}
}
- private ParallelTaxonomyArrays getTaxoArrays() throws IOException {
+ private TaxonomyIndexArrays getTaxoArrays() throws IOException {
if (taxoArrays == null) {
synchronized (this) {
if (taxoArrays == null) {
@@ -754,7 +761,7 @@ public class DirectoryTaxonomyWriter imp
// according to Java Concurrency, this might perform better on some
// JVMs, since the object initialization doesn't happen on the
// volatile member.
- ParallelTaxonomyArrays tmpArrays = new ParallelTaxonomyArrays(reader);
+ TaxonomyIndexArrays tmpArrays = new TaxonomyIndexArrays(reader);
taxoArrays = tmpArrays;
} finally {
readerManager.release(reader);
@@ -991,9 +998,12 @@ public class DirectoryTaxonomyWriter imp
return indexWriter;
}
- /** Used by {@link DirectoryTaxonomyReader} to support NRT. */
- final long getTaxonomyEpoch() {
+ /** Expert: returns current index epoch, if this is a
+ * near-real-time reader. Used by {@link
+ * DirectoryTaxonomyReader} to support NRT.
+ *
+ * @lucene.internal */
+ public final long getTaxonomyEpoch() {
return indexEpoch;
}
-
}
Modified: lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/util/PrintTaxonomyStats.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/util/PrintTaxonomyStats.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/util/PrintTaxonomyStats.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/util/PrintTaxonomyStats.java Thu May 30 07:53:18 2013
@@ -23,8 +23,8 @@ import java.io.PrintStream;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
+import org.apache.lucene.facet.taxonomy.TaxonomyReader.ChildrenIterator;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
-import org.apache.lucene.facet.taxonomy.directory.ParallelTaxonomyArrays;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@@ -55,45 +55,40 @@ public class PrintTaxonomyStats {
}
public static void printStats(TaxonomyReader r, PrintStream out, boolean printTree) throws IOException {
- ParallelTaxonomyArrays arrays = r.getParallelTaxonomyArrays();
- //int[] parents = arrays.parents();
- int[] children = arrays.children();
- int[] siblings = arrays.siblings();
out.println(r.getSize() + " total categories.");
- int childOrd = children[TaxonomyReader.ROOT_ORDINAL];
- while(childOrd != -1) {
- CategoryPath cp = r.getPath(childOrd);
- int childOrd2 = children[childOrd];
+ ChildrenIterator it = r.getChildren(TaxonomyReader.ROOT_ORDINAL);
+ int child;
+ while ((child = it.next()) != TaxonomyReader.INVALID_ORDINAL) {
+ ChildrenIterator chilrenIt = r.getChildren(child);
int numImmediateChildren = 0;
- while(childOrd2 != -1) {
+ while (chilrenIt.next() != TaxonomyReader.INVALID_ORDINAL) {
numImmediateChildren++;
- childOrd2 = siblings[childOrd2];
}
- out.println("/" + cp + ": " + numImmediateChildren + " immediate children; " + (1+countAllChildren(r, childOrd, children, siblings)) + " total categories");
+ CategoryPath cp = r.getPath(child);
+ out.println("/" + cp + ": " + numImmediateChildren + " immediate children; " + (1+countAllChildren(r, child)) + " total categories");
if (printTree) {
- printAllChildren(out, r, childOrd, children, siblings, " ", 1);
+ printAllChildren(out, r, child, " ", 1);
}
- childOrd = siblings[childOrd];
}
}
- private static int countAllChildren(TaxonomyReader r, int ord, int[] children, int[] siblings) throws IOException {
- int childOrd = children[ord];
+ private static int countAllChildren(TaxonomyReader r, int ord) throws IOException {
int count = 0;
- while(childOrd != -1) {
- count += 1+countAllChildren(r, childOrd, children, siblings);
- childOrd = siblings[childOrd];
+ ChildrenIterator it = r.getChildren(ord);
+ int child;
+ while ((child = it.next()) != TaxonomyReader.INVALID_ORDINAL) {
+ count += 1 + countAllChildren(r, child);
}
return count;
}
- private static void printAllChildren(PrintStream out, TaxonomyReader r, int ord, int[] children, int[] siblings, String indent, int depth) throws IOException {
- int childOrd = children[ord];
- while(childOrd != -1) {
- out.println(indent + "/" + r.getPath(childOrd).components[depth]);
- printAllChildren(out, r, childOrd, children, siblings, indent + " ", depth+1);
- childOrd = siblings[childOrd];
+ private static void printAllChildren(PrintStream out, TaxonomyReader r, int ord, String indent, int depth) throws IOException {
+ ChildrenIterator it = r.getChildren(ord);
+ int child;
+ while ((child = it.next()) != TaxonomyReader.INVALID_ORDINAL) {
+ out.println(indent + "/" + r.getPath(child).components[depth]);
+ printAllChildren(out, r, child, indent + " ", depth+1);
}
}
}
Modified: lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java Thu May 30 07:53:18 2013
@@ -162,6 +162,10 @@ public class ScoredDocIdsUtils {
return docids[next];
}
+ @Override
+ public long cost() {
+ return size;
+ }
};
}
};
@@ -298,6 +302,10 @@ public class ScoredDocIdsUtils {
return ++next < maxDoc ? next : NO_MORE_DOCS;
}
+ @Override
+ public long cost() {
+ return maxDoc;
+ }
};
}
};
@@ -394,6 +402,10 @@ public class ScoredDocIdsUtils {
return next < maxDoc ? next : NO_MORE_DOCS;
}
+ @Override
+ public long cost() {
+ return maxDoc;
+ }
};
}
};
Modified: lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java Thu May 30 07:53:18 2013
@@ -24,14 +24,14 @@ public class FacetTestUtils {
public static String toSimpleString(FacetResult fr) {
StringBuilder sb = new StringBuilder();
- toSimpleString(0, sb, fr.getFacetResultNode(), "");
+ toSimpleString(fr.getFacetRequest().categoryPath.length, 0, sb, fr.getFacetResultNode(), "");
return sb.toString();
}
- private static void toSimpleString(int depth, StringBuilder sb, FacetResultNode node, String indent) {
- sb.append(indent + node.label.components[depth] + " (" + (int) node.value + ")\n");
+ private static void toSimpleString(int startLength, int depth, StringBuilder sb, FacetResultNode node, String indent) {
+ sb.append(indent + node.label.components[startLength+depth-1] + " (" + (int) node.value + ")\n");
for (FacetResultNode childNode : node.subResults) {
- toSimpleString(depth + 1, sb, childNode, indent + " ");
+ toSimpleString(startLength, depth + 1, sb, childNode, indent + " ");
}
}
Modified: lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/sampling/BaseSampleTestTopK.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/sampling/BaseSampleTestTopK.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/sampling/BaseSampleTestTopK.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/sampling/BaseSampleTestTopK.java Thu May 30 07:53:18 2013
@@ -94,7 +94,7 @@ public abstract class BaseSampleTestTopK
for (int nTrial = 0; nTrial < RETRIES; nTrial++) {
try {
// complement with sampling!
- final Sampler sampler = createSampler(nTrial, useRandomSampler);
+ final Sampler sampler = createSampler(nTrial, useRandomSampler, samplingSearchParams);
assertSampling(expectedResults, q, sampler, samplingSearchParams, false);
assertSampling(expectedResults, q, sampler, samplingSearchParams, true);
@@ -128,14 +128,20 @@ public abstract class BaseSampleTestTopK
return FacetsCollector.create(sfa);
}
- private Sampler createSampler(int nTrial, boolean useRandomSampler) {
+ private Sampler createSampler(int nTrial, boolean useRandomSampler, FacetSearchParams sParams) {
SamplingParams samplingParams = new SamplingParams();
+ /*
+ * Set sampling to Exact fixing with TakmiSampleFixer as it is not easy to
+ * validate results with amortized results.
+ */
+ samplingParams.setSampleFixer(new TakmiSampleFixer(indexReader, taxoReader, sParams));
+
final double retryFactor = Math.pow(1.01, nTrial);
+ samplingParams.setOversampleFactor(5.0 * retryFactor); // Oversampling
samplingParams.setSampleRatio(0.8 * retryFactor);
samplingParams.setMinSampleSize((int) (100 * retryFactor));
samplingParams.setMaxSampleSize((int) (10000 * retryFactor));
- samplingParams.setOversampleFactor(5.0 * retryFactor);
samplingParams.setSamplingThreshold(11000); //force sampling
Sampler sampler = useRandomSampler ?
Modified: lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/sampling/OversampleWithDepthTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/sampling/OversampleWithDepthTest.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/sampling/OversampleWithDepthTest.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/sampling/OversampleWithDepthTest.java Thu May 30 07:53:18 2013
@@ -119,7 +119,7 @@ public class OversampleWithDepthTest ext
StandardFacetsAccumulator sfa = new SamplingAccumulator(sampler, fsp, r, tr);
FacetsCollector fcWithSampling = FacetsCollector.create(sfa);
- IndexSearcher s = new IndexSearcher(r);
+ IndexSearcher s = newSearcher(r);
s.search(new MatchAllDocsQuery(), fcWithSampling);
// there's only one expected result, return just it.
Modified: lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsAggregatorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsAggregatorTest.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsAggregatorTest.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsAggregatorTest.java Thu May 30 07:53:18 2013
@@ -292,7 +292,7 @@ public class CountingFacetsAggregatorTes
// test the collector w/ FacetRequests and different numResults
DirectoryReader indexReader = DirectoryReader.open(indexDir);
TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
- IndexSearcher searcher = new IndexSearcher(indexReader);
+ IndexSearcher searcher = newSearcher(indexReader);
FacetSearchParams fsp = new FacetSearchParams(new CountFacetRequest(CP_A, NUM_CHILDREN_CP_A),
new CountFacetRequest(CP_B, NUM_CHILDREN_CP_B));
@@ -317,7 +317,7 @@ public class CountingFacetsAggregatorTes
public void testAllCounts() throws Exception {
DirectoryReader indexReader = DirectoryReader.open(indexDir);
TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
- IndexSearcher searcher = new IndexSearcher(indexReader);
+ IndexSearcher searcher = newSearcher(indexReader);
FacetSearchParams fsp = new FacetSearchParams(new CountFacetRequest(CP_A, NUM_CHILDREN_CP_A),
new CountFacetRequest(CP_B, NUM_CHILDREN_CP_B));
@@ -349,7 +349,7 @@ public class CountingFacetsAggregatorTes
public void testBigNumResults() throws Exception {
DirectoryReader indexReader = DirectoryReader.open(indexDir);
TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
- IndexSearcher searcher = new IndexSearcher(indexReader);
+ IndexSearcher searcher = newSearcher(indexReader);
FacetSearchParams fsp = new FacetSearchParams(new CountFacetRequest(CP_A, Integer.MAX_VALUE),
new CountFacetRequest(CP_B, Integer.MAX_VALUE));
@@ -373,7 +373,7 @@ public class CountingFacetsAggregatorTes
public void testNoParents() throws Exception {
DirectoryReader indexReader = DirectoryReader.open(indexDir);
TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
- IndexSearcher searcher = new IndexSearcher(indexReader);
+ IndexSearcher searcher = newSearcher(indexReader);
FacetSearchParams fsp = new FacetSearchParams(fip, new CountFacetRequest(CP_C, NUM_CHILDREN_CP_C),
new CountFacetRequest(CP_D, NUM_CHILDREN_CP_D));
FacetsCollector fc = FacetsCollector.create(randomAccumulator(fsp, indexReader, taxoReader));
Modified: lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownQueryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownQueryTest.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownQueryTest.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownQueryTest.java Thu May 30 07:53:18 2013
@@ -43,6 +43,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryUtils;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
@@ -148,6 +149,7 @@ public class DrillDownQueryTest extends
// Making sure the query yields 25 documents with the facet "a"
DrillDownQuery q = new DrillDownQuery(defaultParams);
q.add(new CategoryPath("a"));
+ QueryUtils.check(q);
TopDocs docs = searcher.search(q, 100);
assertEquals(25, docs.totalHits);
Modified: lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/FacetRequestTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/FacetRequestTest.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/FacetRequestTest.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/FacetRequestTest.java Thu May 30 07:53:18 2013
@@ -23,7 +23,7 @@ import org.junit.Test;
*/
public class FacetRequestTest extends FacetTestCase {
-
+
@Test(expected=IllegalArgumentException.class)
public void testIllegalNumResults() throws Exception {
assertNotNull(new CountFacetRequest(new CategoryPath("a", "b"), 0));
@@ -33,7 +33,7 @@ public class FacetRequestTest extends Fa
public void testIllegalCategoryPath() throws Exception {
assertNotNull(new CountFacetRequest(null, 1));
}
-
+
@Test
public void testHashAndEquals() {
CountFacetRequest fr1 = new CountFacetRequest(new CategoryPath("a"), 8);
Modified: lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java Thu May 30 07:53:18 2013
@@ -203,7 +203,7 @@ public class TestDemoFacets extends Face
FacetSearchParams fsp = new FacetSearchParams(fip,
new CountFacetRequest(new CategoryPath("a", '/'), 10));
- // Aggregatses the facet counts:
+ // Aggregate the facet counts:
FacetsCollector c = FacetsCollector.create(fsp, searcher.getIndexReader(), taxoReader);
// MatchAllDocsQuery is for "browsing" (counts facets
@@ -215,6 +215,11 @@ public class TestDemoFacets extends Face
assertEquals(1, results.size());
assertEquals(1, (int) results.get(0).getFacetResultNode().value);
+ // LUCENE-4913:
+ for(FacetResultNode childNode : results.get(0).getFacetResultNode().subResults) {
+ assertTrue(childNode.ordinal != 0);
+ }
+
searcher.getIndexReader().close();
taxoReader.close();
dir.close();
Modified: lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/TestDrillSideways.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/TestDrillSideways.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/TestDrillSideways.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/TestDrillSideways.java Thu May 30 07:53:18 2013
@@ -32,10 +32,14 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.facet.FacetTestCase;
+import org.apache.lucene.facet.FacetTestUtils;
import org.apache.lucene.facet.index.FacetFields;
import org.apache.lucene.facet.params.FacetIndexingParams;
import org.apache.lucene.facet.params.FacetSearchParams;
import org.apache.lucene.facet.search.DrillSideways.DrillSidewaysResult;
+import org.apache.lucene.facet.sortedset.SortedSetDocValuesAccumulator;
+import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetFields;
+import org.apache.lucene.facet.sortedset.SortedSetDocValuesReaderState;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
@@ -61,6 +65,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.InPlaceMergeSorter;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util._TestUtil;
@@ -115,12 +120,14 @@ public class TestDrillSideways extends F
new CountFacetRequest(new CategoryPath("Publish Date"), 10),
new CountFacetRequest(new CategoryPath("Author"), 10));
+ DrillSideways ds = new DrillSideways(searcher, taxoReader);
+
// Simple case: drill-down on a single field; in this
// case the drill-sideways + drill-down counts ==
// drill-down of just the query:
DrillDownQuery ddq = new DrillDownQuery(fsp.indexingParams, new MatchAllDocsQuery());
ddq.add(new CategoryPath("Author", "Lisa"));
- DrillSidewaysResult r = new DrillSideways(searcher, taxoReader).search(null, ddq, 10, fsp);
+ DrillSidewaysResult r = ds.search(null, ddq, 10, fsp);
assertEquals(2, r.hits.totalHits);
assertEquals(2, r.facetResults.size());
@@ -138,23 +145,26 @@ public class TestDrillSideways extends F
// just the query:
ddq = new DrillDownQuery(fsp.indexingParams);
ddq.add(new CategoryPath("Author", "Lisa"));
- r = new DrillSideways(searcher, taxoReader).search(null, ddq, 10, fsp);
+ r = ds.search(null, ddq, 10, fsp);
assertEquals(2, r.hits.totalHits);
assertEquals(2, r.facetResults.size());
// Publish Date is only drill-down, and Lisa published
// one in 2012 and one in 2010:
assertEquals("Publish Date: 2012=1 2010=1", toString(r.facetResults.get(0)));
+ assertEquals(2, r.facetResults.get(0).getNumValidDescendants());
+
// Author is drill-sideways + drill-down: Lisa
// (drill-down) published twice, and Frank/Susan/Bob
// published once:
assertEquals("Author: Lisa=2 Frank=1 Susan=1 Bob=1", toString(r.facetResults.get(1)));
+ assertEquals(4, r.facetResults.get(1).getNumValidDescendants());
// Another simple case: drill-down on on single fields
// but OR of two values
ddq = new DrillDownQuery(fsp.indexingParams, new MatchAllDocsQuery());
ddq.add(new CategoryPath("Author", "Lisa"), new CategoryPath("Author", "Bob"));
- r = new DrillSideways(searcher, taxoReader).search(null, ddq, 10, fsp);
+ r = ds.search(null, ddq, 10, fsp);
assertEquals(3, r.hits.totalHits);
assertEquals(2, r.facetResults.size());
// Publish Date is only drill-down: Lisa and Bob
@@ -169,7 +179,7 @@ public class TestDrillSideways extends F
ddq = new DrillDownQuery(fsp.indexingParams, new MatchAllDocsQuery());
ddq.add(new CategoryPath("Author", "Lisa"));
ddq.add(new CategoryPath("Publish Date", "2010"));
- r = new DrillSideways(searcher, taxoReader).search(null, ddq, 10, fsp);
+ r = ds.search(null, ddq, 10, fsp);
assertEquals(1, r.hits.totalHits);
assertEquals(2, r.facetResults.size());
// Publish Date is drill-sideways + drill-down: Lisa
@@ -187,7 +197,7 @@ public class TestDrillSideways extends F
ddq.add(new CategoryPath("Author", "Lisa"),
new CategoryPath("Author", "Bob"));
ddq.add(new CategoryPath("Publish Date", "2010"));
- r = new DrillSideways(searcher, taxoReader).search(null, ddq, 10, fsp);
+ r = ds.search(null, ddq, 10, fsp);
assertEquals(2, r.hits.totalHits);
assertEquals(2, r.facetResults.size());
// Publish Date is both drill-sideways + drill-down:
@@ -203,7 +213,7 @@ public class TestDrillSideways extends F
fsp = new FacetSearchParams(
new CountFacetRequest(new CategoryPath("Publish Date"), 10),
new CountFacetRequest(new CategoryPath("Foobar"), 10));
- r = new DrillSideways(searcher, taxoReader).search(null, ddq, 10, fsp);
+ r = ds.search(null, ddq, 10, fsp);
assertEquals(0, r.hits.totalHits);
assertEquals(2, r.facetResults.size());
assertEquals("Publish Date:", toString(r.facetResults.get(0)));
@@ -216,7 +226,7 @@ public class TestDrillSideways extends F
fsp = new FacetSearchParams(
new CountFacetRequest(new CategoryPath("Publish Date"), 10),
new CountFacetRequest(new CategoryPath("Author"), 10));
- r = new DrillSideways(searcher, taxoReader).search(null, ddq, 10, fsp);
+ r = ds.search(null, ddq, 10, fsp);
assertEquals(2, r.hits.totalHits);
assertEquals(2, r.facetResults.size());
// Publish Date is only drill-down, and Lisa published
@@ -227,13 +237,27 @@ public class TestDrillSideways extends F
// published once:
assertEquals("Author: Lisa=2 Frank=1 Susan=1 Bob=1", toString(r.facetResults.get(1)));
+ // LUCENE-4915: test drilling down on a dimension but
+ // NOT facet counting it:
+ ddq = new DrillDownQuery(fsp.indexingParams, new MatchAllDocsQuery());
+ ddq.add(new CategoryPath("Author", "Lisa"),
+ new CategoryPath("Author", "Tom"));
+ fsp = new FacetSearchParams(
+ new CountFacetRequest(new CategoryPath("Publish Date"), 10));
+ r = ds.search(null, ddq, 10, fsp);
+ assertEquals(2, r.hits.totalHits);
+ assertEquals(1, r.facetResults.size());
+ // Publish Date is only drill-down, and Lisa published
+ // one in 2012 and one in 2010:
+ assertEquals("Publish Date: 2012=1 2010=1", toString(r.facetResults.get(0)));
+
// Test main query gets null scorer:
fsp = new FacetSearchParams(
new CountFacetRequest(new CategoryPath("Publish Date"), 10),
new CountFacetRequest(new CategoryPath("Author"), 10));
ddq = new DrillDownQuery(fsp.indexingParams, new TermQuery(new Term("foobar", "baz")));
ddq.add(new CategoryPath("Author", "Lisa"));
- r = new DrillSideways(searcher, taxoReader).search(null, ddq, 10, fsp);
+ r = ds.search(null, ddq, 10, fsp);
assertEquals(0, r.hits.totalHits);
assertEquals(2, r.facetResults.size());
@@ -343,8 +367,65 @@ public class TestDrillSideways extends F
}
}
+ public void testMultipleRequestsPerDim() throws Exception {
+ Directory dir = newDirectory();
+ Directory taxoDir = newDirectory();
+ writer = new RandomIndexWriter(random(), dir);
+
+ // Writes facet ords to a separate directory from the
+ // main index:
+ taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
+
+ // Reused across documents, to add the necessary facet
+ // fields:
+ facetFields = new FacetFields(taxoWriter);
+
+ add("dim/a/x");
+ add("dim/a/y");
+ add("dim/a/z");
+ add("dim/b");
+ add("dim/c");
+ add("dim/d");
+
+ // NRT open
+ IndexSearcher searcher = newSearcher(writer.getReader());
+ writer.close();
+
+ //System.out.println("searcher=" + searcher);
+
+ // NRT open
+ TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+ taxoWriter.close();
+
+ // Two requests against the same dim:
+ FacetSearchParams fsp = new FacetSearchParams(
+ new CountFacetRequest(new CategoryPath("dim"), 10),
+ new CountFacetRequest(new CategoryPath("dim", "a"), 10));
+
+ DrillDownQuery ddq = new DrillDownQuery(fsp.indexingParams, new MatchAllDocsQuery());
+ ddq.add(new CategoryPath("dim", "a"));
+ DrillSidewaysResult r = new DrillSideways(searcher, taxoReader).search(null, ddq, 10, fsp);
+
+ assertEquals(3, r.hits.totalHits);
+ assertEquals(2, r.facetResults.size());
+ // Publish Date is only drill-down, and Lisa published
+ // one in 2012 and one in 2010:
+ assertEquals("dim: a=3 d=1 c=1 b=1", toString(r.facetResults.get(0)));
+ // Author is drill-sideways + drill-down: Lisa
+ // (drill-down) published twice, and Frank/Susan/Bob
+ // published once:
+ assertEquals("a (3)\n z (1)\n y (1)\n x (1)\n", FacetTestUtils.toSimpleString(r.facetResults.get(1)));
+
+ searcher.getIndexReader().close();
+ taxoReader.close();
+ dir.close();
+ taxoDir.close();
+ }
+
public void testRandom() throws Exception {
+ boolean canUseDV = defaultCodecSupportsSortedSet();
+
while (aChance == 0.0) {
aChance = random().nextDouble();
}
@@ -373,19 +454,21 @@ public class TestDrillSideways extends F
}
String[][] dimValues = new String[numDims][];
int valueCount = 2;
+
for(int dim=0;dim<numDims;dim++) {
Set<String> values = new HashSet<String>();
while (values.size() < valueCount) {
String s;
while (true) {
s = _TestUtil.randomRealisticUnicodeString(random());
- // We cannot include this character else the label
- // is silently truncated:
- if (s.indexOf(FacetIndexingParams.DEFAULT_FACET_DELIM_CHAR) == -1) {
+ //s = _TestUtil.randomSimpleString(random());
+ // We cannot include this character else we hit
+ // IllegalArgExc:
+ if (s.indexOf(FacetIndexingParams.DEFAULT_FACET_DELIM_CHAR) == -1 &&
+ (!canUseDV || s.indexOf('/') == -1)) {
break;
}
}
- //String s = _TestUtil.randomSimpleString(random());
if (s.length() > 0) {
values.add(s);
}
@@ -437,6 +520,9 @@ public class TestDrillSideways extends F
RandomIndexWriter w = new RandomIndexWriter(random(), d, iwc);
DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(td, IndexWriterConfig.OpenMode.CREATE);
facetFields = new FacetFields(tw);
+ SortedSetDocValuesFacetFields dvFacetFields = new SortedSetDocValuesFacetFields();
+
+ boolean doUseDV = canUseDV && random().nextBoolean();
for(Doc rawDoc : docs) {
Document doc = new Document();
@@ -450,7 +536,8 @@ public class TestDrillSideways extends F
for(int dim=0;dim<numDims;dim++) {
int dimValue = rawDoc.dims[dim];
if (dimValue != -1) {
- paths.add(new CategoryPath("dim" + dim, dimValues[dim][dimValue]));
+ CategoryPath cp = new CategoryPath("dim" + dim, dimValues[dim][dimValue]);
+ paths.add(cp);
doc.add(new StringField("dim" + dim, dimValues[dim][dimValue], Field.Store.YES));
if (VERBOSE) {
System.out.println(" dim" + dim + "=" + new BytesRef(dimValues[dim][dimValue]));
@@ -458,7 +545,8 @@ public class TestDrillSideways extends F
}
int dimValue2 = rawDoc.dims2[dim];
if (dimValue2 != -1) {
- paths.add(new CategoryPath("dim" + dim, dimValues[dim][dimValue2]));
+ CategoryPath cp = new CategoryPath("dim" + dim, dimValues[dim][dimValue2]);
+ paths.add(cp);
doc.add(new StringField("dim" + dim, dimValues[dim][dimValue2], Field.Store.YES));
if (VERBOSE) {
System.out.println(" dim" + dim + "=" + new BytesRef(dimValues[dim][dimValue2]));
@@ -466,8 +554,13 @@ public class TestDrillSideways extends F
}
}
if (!paths.isEmpty()) {
- facetFields.addFields(doc, paths);
+ if (doUseDV) {
+ dvFacetFields.addFields(doc, paths);
+ } else {
+ facetFields.addFields(doc, paths);
+ }
}
+
w.addDocument(doc);
}
@@ -499,6 +592,14 @@ public class TestDrillSideways extends F
}
IndexReader r = w.getReader();
w.close();
+
+ final SortedSetDocValuesReaderState sortedSetDVState;
+ if (doUseDV) {
+ sortedSetDVState = new SortedSetDocValuesReaderState(r);
+ } else {
+ sortedSetDVState = null;
+ }
+
if (VERBOSE) {
System.out.println("r.numDocs() = " + r.numDocs());
}
@@ -507,24 +608,43 @@ public class TestDrillSideways extends F
TaxonomyReader tr = new DirectoryTaxonomyReader(tw);
tw.close();
- List<FacetRequest> requests = new ArrayList<FacetRequest>();
- for(int i=0;i<numDims;i++) {
- requests.add(new CountFacetRequest(new CategoryPath("dim" + i), dimValues[numDims-1].length));
- }
-
- FacetSearchParams fsp = new FacetSearchParams(requests);
- IndexSearcher s = new IndexSearcher(r);
+ IndexSearcher s = newSearcher(r);
int numIters = atLeast(10);
for(int iter=0;iter<numIters;iter++) {
+
String contentToken = random().nextInt(30) == 17 ? null : randomContentToken(true);
int numDrillDown = _TestUtil.nextInt(random(), 1, Math.min(4, numDims));
- String[][] drillDowns = new String[numDims][];
if (VERBOSE) {
- System.out.println("\nTEST: iter=" + iter + " baseQuery=" + contentToken + " numDrillDown=" + numDrillDown);
+ System.out.println("\nTEST: iter=" + iter + " baseQuery=" + contentToken + " numDrillDown=" + numDrillDown + " useSortedSetDV=" + doUseDV);
}
+
+ List<FacetRequest> requests = new ArrayList<FacetRequest>();
+ while(true) {
+ for(int i=0;i<numDims;i++) {
+ // LUCENE-4915: sometimes don't request facet
+ // counts on the dim(s) we drill down on
+ if (random().nextDouble() <= 0.9) {
+ if (VERBOSE) {
+ System.out.println(" do facet request on dim=" + i);
+ }
+ requests.add(new CountFacetRequest(new CategoryPath("dim" + i), dimValues[numDims-1].length));
+ } else {
+ if (VERBOSE) {
+ System.out.println(" skip facet request on dim=" + i);
+ }
+ }
+ }
+ if (!requests.isEmpty()) {
+ break;
+ }
+ }
+ FacetSearchParams fsp = new FacetSearchParams(requests);
+ String[][] drillDowns = new String[numDims][];
+
int count = 0;
+ boolean anyMultiValuedDrillDowns = false;
while (count < numDrillDown) {
int dim = random().nextInt(numDims);
if (drillDowns[dim] == null) {
@@ -534,6 +654,7 @@ public class TestDrillSideways extends F
} else {
int orCount = _TestUtil.nextInt(random(), 1, Math.min(5, dimValues[dim].length));
drillDowns[dim] = new String[orCount];
+ anyMultiValuedDrillDowns |= orCount > 1;
for(int i=0;i<orCount;i++) {
while (true) {
String value = dimValues[dim][random().nextInt(dimValues[dim].length)];
@@ -604,7 +725,9 @@ public class TestDrillSideways extends F
filter = null;
}
- // Verify docs are always collected in order:
+ // Verify docs are always collected in order. If we
+ // had an AssertingScorer it could catch it when
+ // Weight.scoresDocsOutOfOrder lies!:
new DrillSideways(s, tr).search(ddq,
new Collector() {
int lastDocID;
@@ -630,18 +753,68 @@ public class TestDrillSideways extends F
}
}, fsp);
- SimpleFacetResult expected = slowDrillSidewaysSearch(s, docs, contentToken, drillDowns, dimValues, filter);
+ // Also separately verify that DS respects the
+ // scoreSubDocsAtOnce method, to ensure that all
+ // subScorers are on the same docID:
+ if (!anyMultiValuedDrillDowns) {
+ // Can only do this test when there are no OR'd
+ // drill-down values, beacuse in that case it's
+ // easily possible for one of the DD terms to be on
+ // a future docID:
+ new DrillSideways(s, tr) {
+ @Override
+ protected boolean scoreSubDocsAtOnce() {
+ return true;
+ }
+ }.search(ddq, new AssertingSubDocsAtOnceCollector(), fsp);
+ }
+
+ SimpleFacetResult expected = slowDrillSidewaysSearch(s, requests, docs, contentToken, drillDowns, dimValues, filter);
Sort sort = new Sort(new SortField("id", SortField.Type.STRING));
- DrillSidewaysResult actual = new DrillSideways(s, tr).search(ddq, filter, null, numDocs, sort, true, true, fsp);
+ DrillSideways ds;
+ if (doUseDV) {
+ ds = new DrillSideways(s, null) {
+ @Override
+ protected FacetsAccumulator getDrillDownAccumulator(FacetSearchParams fsp) throws IOException {
+ return new SortedSetDocValuesAccumulator(fsp, sortedSetDVState);
+ }
+
+ @Override
+ protected FacetsAccumulator getDrillSidewaysAccumulator(String dim, FacetSearchParams fsp) throws IOException {
+ return new SortedSetDocValuesAccumulator(fsp, sortedSetDVState);
+ }
+ };
+ } else {
+ ds = new DrillSideways(s, tr);
+ }
+
+ // Retrieve all facets:
+ DrillSidewaysResult actual = ds.search(ddq, filter, null, numDocs, sort, true, true, fsp);
TopDocs hits = s.search(baseQuery, numDocs);
Map<String,Float> scores = new HashMap<String,Float>();
for(ScoreDoc sd : hits.scoreDocs) {
scores.put(s.doc(sd.doc).get("id"), sd.score);
}
-
- verifyEquals(dimValues, s, expected, actual, scores);
+ if (VERBOSE) {
+ System.out.println(" verify all facets");
+ }
+ verifyEquals(requests, dimValues, s, expected, actual, scores, -1, doUseDV);
+
+ // Retrieve topN facets:
+ int topN = _TestUtil.nextInt(random(), 1, 20);
+
+ List<FacetRequest> newRequests = new ArrayList<FacetRequest>();
+ for(FacetRequest oldRequest : requests) {
+ newRequests.add(new CountFacetRequest(oldRequest.categoryPath, topN));
+ }
+ fsp = new FacetSearchParams(newRequests);
+ actual = ds.search(ddq, filter, null, numDocs, sort, true, true, fsp);
+ if (VERBOSE) {
+ System.out.println(" verify topN=" + topN);
+ }
+ verifyEquals(newRequests, dimValues, s, expected, actual, scores, topN, doUseDV);
// Make sure drill down doesn't change score:
TopDocs ddqHits = s.search(ddq, filter, numDocs);
@@ -691,9 +864,62 @@ public class TestDrillSideways extends F
private static class SimpleFacetResult {
List<Doc> hits;
int[][] counts;
+ int[] uniqueCounts;
+ }
+
+ private int[] getTopNOrds(final int[] counts, final String[] values, int topN) {
+ final int[] ids = new int[counts.length];
+ for(int i=0;i<ids.length;i++) {
+ ids[i] = i;
+ }
+
+ // Naive (on purpose, to reduce bug in tester/gold):
+ // sort all ids, then return top N slice:
+ new InPlaceMergeSorter() {
+
+ @Override
+ protected void swap(int i, int j) {
+ int id = ids[i];
+ ids[i] = ids[j];
+ ids[j] = id;
+ }
+
+ @Override
+ protected int compare(int i, int j) {
+ int counti = counts[ids[i]];
+ int countj = counts[ids[j]];
+ // Sort by count descending...
+ if (counti > countj) {
+ return -1;
+ } else if (counti < countj) {
+ return 1;
+ } else {
+ // ... then by label ascending:
+ return new BytesRef(values[ids[i]]).compareTo(new BytesRef(values[ids[j]]));
+ }
+ }
+
+ }.sort(0, ids.length);
+
+ if (topN > ids.length) {
+ topN = ids.length;
+ }
+
+ int numSet = topN;
+ for(int i=0;i<topN;i++) {
+ if (counts[ids[i]] == 0) {
+ numSet = i;
+ break;
+ }
+ }
+
+ int[] topNIDs = new int[numSet];
+ System.arraycopy(ids, 0, topNIDs, 0, topNIDs.length);
+ return topNIDs;
}
- private SimpleFacetResult slowDrillSidewaysSearch(IndexSearcher s, List<Doc> docs, String contentToken, String[][] drillDowns,
+ private SimpleFacetResult slowDrillSidewaysSearch(IndexSearcher s, List<FacetRequest> requests, List<Doc> docs,
+ String contentToken, String[][] drillDowns,
String[][] dimValues, Filter onlyEven) throws Exception {
int numDims = dimValues.length;
@@ -769,18 +995,28 @@ public class TestDrillSideways extends F
SimpleFacetResult res = new SimpleFacetResult();
res.hits = hits;
res.counts = new int[numDims][];
- for(int dim=0;dim<numDims;dim++) {
+ res.uniqueCounts = new int[numDims];
+ for (int i = 0; i < requests.size(); i++) {
+ int dim = Integer.parseInt(requests.get(i).categoryPath.components[0].substring(3));
if (drillDowns[dim] != null) {
res.counts[dim] = drillSidewaysCounts[dim].counts[dim];
} else {
res.counts[dim] = drillDownCounts.counts[dim];
}
+ int uniqueCount = 0;
+ for (int j = 0; j < res.counts[dim].length; j++) {
+ if (res.counts[dim][j] != 0) {
+ uniqueCount++;
+ }
+ }
+ res.uniqueCounts[dim] = uniqueCount;
}
return res;
}
- void verifyEquals(String[][] dimValues, IndexSearcher s, SimpleFacetResult expected, DrillSidewaysResult actual, Map<String,Float> scores) throws Exception {
+ void verifyEquals(List<FacetRequest> requests, String[][] dimValues, IndexSearcher s, SimpleFacetResult expected,
+ DrillSidewaysResult actual, Map<String,Float> scores, int topN, boolean isSortedSetDV) throws Exception {
if (VERBOSE) {
System.out.println(" verify totHits=" + expected.hits.size());
}
@@ -795,41 +1031,102 @@ public class TestDrillSideways extends F
// Score should be IDENTICAL:
assertEquals(scores.get(expected.hits.get(i).id), actual.hits.scoreDocs[i].score, 0.0f);
}
- assertEquals(expected.counts.length, actual.facetResults.size());
+
+ int numExpected = 0;
+ for(int dim=0;dim<expected.counts.length;dim++) {
+ if (expected.counts[dim] != null) {
+ numExpected++;
+ }
+ }
+
+ assertEquals(numExpected, actual.facetResults.size());
+
for(int dim=0;dim<expected.counts.length;dim++) {
+ if (expected.counts[dim] == null) {
+ continue;
+ }
+ int idx = -1;
+ for(int i=0;i<requests.size();i++) {
+ if (Integer.parseInt(requests.get(i).categoryPath.components[0].substring(3)) == dim) {
+ idx = i;
+ break;
+ }
+ }
+ assert idx != -1;
+ FacetResult fr = actual.facetResults.get(idx);
+ List<FacetResultNode> subResults = fr.getFacetResultNode().subResults;
if (VERBOSE) {
System.out.println(" dim" + dim);
System.out.println(" actual");
}
- FacetResult fr = actual.facetResults.get(dim);
+
Map<String,Integer> actualValues = new HashMap<String,Integer>();
- for(FacetResultNode childNode : fr.getFacetResultNode().subResults) {
+ idx = 0;
+ for(FacetResultNode childNode : subResults) {
actualValues.put(childNode.label.components[1], (int) childNode.value);
if (VERBOSE) {
- System.out.println(" " + new BytesRef(childNode.label.components[1]) + ": " + (int) childNode.value);
+ System.out.println(" " + idx + ": " + new BytesRef(childNode.label.components[1]) + ": " + (int) childNode.value);
+ idx++;
}
}
- if (VERBOSE) {
- System.out.println(" expected");
- }
+ if (topN != -1) {
+ int[] topNIDs = getTopNOrds(expected.counts[dim], dimValues[dim], topN);
+ if (VERBOSE) {
+ idx = 0;
+ System.out.println(" expected (sorted)");
+ for(int i=0;i<topNIDs.length;i++) {
+ int expectedOrd = topNIDs[i];
+ String value = dimValues[dim][expectedOrd];
+ System.out.println(" " + idx + ": " + new BytesRef(value) + ": " + expected.counts[dim][expectedOrd]);
+ idx++;
+ }
+ }
+ if (VERBOSE) {
+ System.out.println(" topN=" + topN + " expectedTopN=" + topNIDs.length);
+ }
- int setCount = 0;
- for(int i=0;i<dimValues[dim].length;i++) {
- String value = dimValues[dim][i];
- if (expected.counts[dim][i] != 0) {
- if (VERBOSE) {
- System.out.println(" " + new BytesRef(value) + ": " + expected.counts[dim][i]);
- }
- assertTrue(actualValues.containsKey(value));
- assertEquals(expected.counts[dim][i], actualValues.get(value).intValue());
- setCount++;
- } else {
- assertFalse(actualValues.containsKey(value));
+ assertEquals(topNIDs.length, subResults.size());
+ for(int i=0;i<topNIDs.length;i++) {
+ FacetResultNode node = subResults.get(i);
+ int expectedOrd = topNIDs[i];
+ assertEquals(expected.counts[dim][expectedOrd], (int) node.value);
+ assertEquals(2, node.label.length);
+ if (isSortedSetDV) {
+ // Tie-break facet labels are only in unicode
+ // order with SortedSetDVFacets:
+ assertEquals("value @ idx=" + i, dimValues[dim][expectedOrd], node.label.components[1]);
+ }
+ }
+ } else {
+
+ if (VERBOSE) {
+ idx = 0;
+ System.out.println(" expected (unsorted)");
+ for(int i=0;i<dimValues[dim].length;i++) {
+ String value = dimValues[dim][i];
+ if (expected.counts[dim][i] != 0) {
+ System.out.println(" " + idx + ": " + new BytesRef(value) + ": " + expected.counts[dim][i]);
+ idx++;
+ }
+ }
+ }
+
+ int setCount = 0;
+ for(int i=0;i<dimValues[dim].length;i++) {
+ String value = dimValues[dim][i];
+ if (expected.counts[dim][i] != 0) {
+ assertTrue(actualValues.containsKey(value));
+ assertEquals(expected.counts[dim][i], actualValues.get(value).intValue());
+ setCount++;
+ } else {
+ assertFalse(actualValues.containsKey(value));
+ }
}
+ assertEquals(setCount, actualValues.size());
}
- assertEquals(setCount, actualValues.size());
+ assertEquals("dim=" + dim, expected.uniqueCounts[dim], fr.getNumValidDescendants());
}
}
Modified: lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsCollector.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsCollector.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsCollector.java Thu May 30 07:53:18 2013
@@ -1,6 +1,7 @@
package org.apache.lucene.facet.search;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -16,15 +17,18 @@ import org.apache.lucene.facet.params.Ca
import org.apache.lucene.facet.params.FacetIndexingParams;
import org.apache.lucene.facet.params.FacetSearchParams;
import org.apache.lucene.facet.params.PerDimensionIndexingParams;
+import org.apache.lucene.facet.search.FacetRequest.ResultMode;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.MultiCollector;
+import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
@@ -60,7 +64,9 @@ public class TestFacetsCollector extends
FacetFields facetFields = new FacetFields(taxonomyWriter);
for(int i = atLeast(30); i > 0; --i) {
Document doc = new Document();
- doc.add(new StringField("f", "v", Store.NO));
+ if (random().nextBoolean()) { // don't match all documents
+ doc.add(new StringField("f", "v", Store.NO));
+ }
facetFields.addFields(doc, Collections.singletonList(new CategoryPath("a")));
iw.addDocument(doc);
}
@@ -80,12 +86,16 @@ public class TestFacetsCollector extends
};
FacetsCollector fc = FacetsCollector.create(fa);
TopScoreDocCollector topDocs = TopScoreDocCollector.create(10, false);
- new IndexSearcher(r).search(new MatchAllDocsQuery(), MultiCollector.wrap(fc, topDocs));
+ ConstantScoreQuery csq = new ConstantScoreQuery(new MatchAllDocsQuery());
+ csq.setBoost(2.0f);
+
+ newSearcher(r).search(csq, MultiCollector.wrap(fc, topDocs));
List<FacetResult> res = fc.getFacetResults();
- double value = res.get(0).getFacetResultNode().value;
- double expected = topDocs.topDocs().getMaxScore() * r.numDocs();
- assertEquals(expected, value, 1E-10);
+ float value = (float) res.get(0).getFacetResultNode().value;
+ TopDocs td = topDocs.topDocs();
+ int expected = (int) (td.getMaxScore() * td.totalHits);
+ assertEquals(expected, (int) value);
IOUtils.close(taxo, taxoDir, r, indexDir);
}
@@ -120,7 +130,7 @@ public class TestFacetsCollector extends
new CountFacetRequest(new CategoryPath("a"), 10),
new CountFacetRequest(new CategoryPath("b"), 10));
FacetsCollector fc = FacetsCollector.create(sParams, r, taxo);
- new IndexSearcher(r).search(new MatchAllDocsQuery(), fc);
+ newSearcher(r).search(new MatchAllDocsQuery(), fc);
for (FacetResult res : fc.getFacetResults()) {
assertEquals("unexpected count for " + res, r.maxDoc(), (int) res.getFacetResultNode().value);
@@ -172,7 +182,7 @@ public class TestFacetsCollector extends
FacetsCollector fc = FacetsCollector.create(fa);
TopScoreDocCollector topDocs = TopScoreDocCollector.create(10, false);
- new IndexSearcher(r).search(new MatchAllDocsQuery(), MultiCollector.wrap(fc, topDocs));
+ newSearcher(r).search(new MatchAllDocsQuery(), MultiCollector.wrap(fc, topDocs));
List<FacetResult> facetResults = fc.getFacetResults();
FacetResult fresA = facetResults.get(0);
@@ -184,5 +194,195 @@ public class TestFacetsCollector extends
IOUtils.close(taxo, taxoDir, r, indexDir);
}
+
+ @Test
+ public void testCountRoot() throws Exception {
+ // LUCENE-4882: FacetsAccumulator threw NPE if a FacetRequest was defined on CP.EMPTY
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+
+ TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+
+ FacetFields facetFields = new FacetFields(taxonomyWriter);
+ for(int i = atLeast(30); i > 0; --i) {
+ Document doc = new Document();
+ facetFields.addFields(doc, Arrays.asList(new CategoryPath("a"), new CategoryPath("b")));
+ iw.addDocument(doc);
+ }
+
+ taxonomyWriter.close();
+ iw.close();
+
+ DirectoryReader r = DirectoryReader.open(indexDir);
+ DirectoryTaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir);
+
+ FacetSearchParams fsp = new FacetSearchParams(new CountFacetRequest(CategoryPath.EMPTY, 10));
+
+ final FacetsAccumulator fa = random().nextBoolean() ? new FacetsAccumulator(fsp, r, taxo) : new StandardFacetsAccumulator(fsp, r, taxo);
+ FacetsCollector fc = FacetsCollector.create(fa);
+ newSearcher(r).search(new MatchAllDocsQuery(), fc);
+
+ FacetResult res = fc.getFacetResults().get(0);
+ for (FacetResultNode node : res.getFacetResultNode().subResults) {
+ assertEquals(r.numDocs(), (int) node.value);
+ }
+
+ IOUtils.close(taxo, taxoDir, r, indexDir);
+ }
+ @Test
+ public void testGetFacetResultsTwice() throws Exception {
+ // LUCENE-4893: counts were multiplied as many times as getFacetResults was called.
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+
+ TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+
+ FacetFields facetFields = new FacetFields(taxonomyWriter);
+ Document doc = new Document();
+ facetFields.addFields(doc, Arrays.asList(new CategoryPath("a/1", '/'), new CategoryPath("b/1", '/')));
+ iw.addDocument(doc);
+ taxonomyWriter.close();
+ iw.close();
+
+ DirectoryReader r = DirectoryReader.open(indexDir);
+ DirectoryTaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir);
+
+ FacetSearchParams fsp = new FacetSearchParams(
+ new CountFacetRequest(new CategoryPath("a"), 10),
+ new CountFacetRequest(new CategoryPath("b"), 10));
+ final FacetsAccumulator fa = random().nextBoolean() ? new FacetsAccumulator(fsp, r, taxo) : new StandardFacetsAccumulator(fsp, r, taxo);
+ final FacetsCollector fc = FacetsCollector.create(fa);
+ newSearcher(r).search(new MatchAllDocsQuery(), fc);
+
+ List<FacetResult> res1 = fc.getFacetResults();
+ List<FacetResult> res2 = fc.getFacetResults();
+ assertSame("calling getFacetResults twice should return the exact same result", res1, res2);
+
+ IOUtils.close(taxo, taxoDir, r, indexDir);
+ }
+
+ @Test
+ public void testReset() throws Exception {
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+
+ TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+
+ FacetFields facetFields = new FacetFields(taxonomyWriter);
+ Document doc = new Document();
+ facetFields.addFields(doc, Arrays.asList(new CategoryPath("a/1", '/'), new CategoryPath("b/1", '/')));
+ iw.addDocument(doc);
+ taxonomyWriter.close();
+ iw.close();
+
+ DirectoryReader r = DirectoryReader.open(indexDir);
+ DirectoryTaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir);
+
+ FacetSearchParams fsp = new FacetSearchParams(
+ new CountFacetRequest(new CategoryPath("a"), 10),
+ new CountFacetRequest(new CategoryPath("b"), 10));
+ final FacetsAccumulator fa = random().nextBoolean() ? new FacetsAccumulator(fsp, r, taxo) : new StandardFacetsAccumulator(fsp, r, taxo);
+ final FacetsCollector fc = FacetsCollector.create(fa);
+ // this should populate the cached results, but doing search should clear the cache
+ fc.getFacetResults();
+ newSearcher(r).search(new MatchAllDocsQuery(), fc);
+
+ List<FacetResult> res1 = fc.getFacetResults();
+ // verify that we didn't get the cached result
+ assertEquals(2, res1.size());
+ for (FacetResult res : res1) {
+ assertEquals(1, res.getFacetResultNode().subResults.size());
+ assertEquals(1, (int) res.getFacetResultNode().subResults.get(0).value);
+ }
+ fc.reset();
+ List<FacetResult> res2 = fc.getFacetResults();
+ assertNotSame("reset() should clear the cached results", res1, res2);
+
+ IOUtils.close(taxo, taxoDir, r, indexDir);
+ }
+
+ @Test
+ public void testParentOrdinal() throws Exception {
+ // LUCENE-4913: root ordinal was always 0 when all children were requested
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+
+ TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+
+ FacetFields facetFields = new FacetFields(taxonomyWriter);
+ Document doc = new Document();
+ facetFields.addFields(doc, Arrays.asList(new CategoryPath("a/1", '/')));
+ iw.addDocument(doc);
+ taxonomyWriter.close();
+ iw.close();
+
+ DirectoryReader r = DirectoryReader.open(indexDir);
+ DirectoryTaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir);
+
+ // assert IntFacetResultHandler
+ FacetSearchParams fsp = new FacetSearchParams(new CountFacetRequest(new CategoryPath("a"), 10));
+ FacetsAccumulator fa = random().nextBoolean() ? new FacetsAccumulator(fsp, r, taxo) : new StandardFacetsAccumulator(fsp, r, taxo);
+ FacetsCollector fc = FacetsCollector.create(fa);
+ newSearcher(r).search(new MatchAllDocsQuery(), fc);
+ assertTrue("invalid ordinal for child node: 0", 0 != fc.getFacetResults().get(0).getFacetResultNode().subResults.get(0).ordinal);
+
+ // assert IntFacetResultHandler
+ fsp = new FacetSearchParams(new SumScoreFacetRequest(new CategoryPath("a"), 10));
+ if (random().nextBoolean()) {
+ fa = new FacetsAccumulator(fsp, r, taxo) {
+ @Override
+ public FacetsAggregator getAggregator() {
+ return new SumScoreFacetsAggregator();
+ }
+ };
+ } else {
+ fa = new StandardFacetsAccumulator(fsp, r, taxo);
+ }
+ fc = FacetsCollector.create(fa);
+ newSearcher(r).search(new MatchAllDocsQuery(), fc);
+ assertTrue("invalid ordinal for child node: 0", 0 != fc.getFacetResults().get(0).getFacetResultNode().subResults.get(0).ordinal);
+
+ IOUtils.close(taxo, taxoDir, r, indexDir);
+ }
+
+ @Test
+ public void testNumValidDescendants() throws Exception {
+ // LUCENE-4885: FacetResult.numValidDescendants was not set properly by FacetsAccumulator
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+
+ TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(taxoDir);
+ IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+
+ FacetFields facetFields = new FacetFields(taxonomyWriter);
+ for (int i = 0; i < 10; i++) {
+ Document doc = new Document();
+ facetFields.addFields(doc, Arrays.asList(new CategoryPath("a", Integer.toString(i))));
+ iw.addDocument(doc);
+ }
+
+ taxonomyWriter.close();
+ iw.close();
+
+ DirectoryReader r = DirectoryReader.open(indexDir);
+ DirectoryTaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir);
+
+ CountFacetRequest cfr = new CountFacetRequest(new CategoryPath("a"), 2);
+ cfr.setResultMode(random().nextBoolean() ? ResultMode.GLOBAL_FLAT : ResultMode.PER_NODE_IN_TREE);
+ FacetSearchParams fsp = new FacetSearchParams(cfr);
+ final FacetsAccumulator fa = random().nextBoolean() ? new FacetsAccumulator(fsp, r, taxo) : new StandardFacetsAccumulator(fsp, r, taxo);
+ FacetsCollector fc = FacetsCollector.create(fa);
+ newSearcher(r).search(new MatchAllDocsQuery(), fc);
+
+ FacetResult res = fc.getFacetResults().get(0);
+ assertEquals(10, res.getNumValidDescendants());
+
+ IOUtils.close(taxo, taxoDir, r, indexDir);
+ }
+
}
Modified: lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java Thu May 30 07:53:18 2013
@@ -93,7 +93,7 @@ public class TestStandardFacetsAccumulat
DirectoryReader indexReader = DirectoryReader.open(indexDir);
TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
- IndexSearcher indexSearcher = new IndexSearcher(indexReader);
+ IndexSearcher indexSearcher = newSearcher(indexReader);
// search for "f:a", only segments 1 and 3 should match results
Query q = new TermQuery(new Term("f", "a"));
Modified: lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestCategoryPath.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestCategoryPath.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestCategoryPath.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestCategoryPath.java Thu May 30 07:53:18 2013
@@ -216,5 +216,63 @@ public class TestCategoryPath extends Fa
assertNotNull(new CategoryPath("test/", '/'));
}
+
+ @Test
+ public void testInvalidDelimChar() throws Exception {
+ // Make sure CategoryPath doesn't silently corrupt:
+ char[] buf = new char[100];
+ CategoryPath cp = new CategoryPath("foo/bar");
+ try {
+ cp.toString();
+ fail("expected exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ try {
+ cp.copyFullPath(buf, 0, '/');
+ fail("expected exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ cp = new CategoryPath("abc", "foo/bar");
+ try {
+ cp.toString();
+ fail("expected exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ try {
+ cp.copyFullPath(buf, 0, '/');
+ fail("expected exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ cp = new CategoryPath("foo:bar");
+ try {
+ cp.toString(':');
+ fail("expected exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ try {
+ cp.copyFullPath(buf, 0, ':');
+ fail("expected exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ cp = new CategoryPath("abc", "foo:bar");
+ try {
+ cp.toString(':');
+ fail("expected exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ try {
+ cp.copyFullPath(buf, 0, ':');
+ fail("expected exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyCombined.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyCombined.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyCombined.java (original)
+++ lucene/dev/branches/security/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyCombined.java Thu May 30 07:53:18 2013
@@ -11,7 +11,6 @@ import org.apache.lucene.facet.FacetTest
import org.apache.lucene.facet.SlowRAMDirectory;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
-import org.apache.lucene.facet.taxonomy.directory.ParallelTaxonomyArrays;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.RAMDirectory;
@@ -35,7 +34,6 @@ import org.junit.Test;
* limitations under the License.
*/
-// TODO: remove this suppress if we fix the TaxoWriter Codec to a non-default (see todo in DirTW)
@SuppressCodecs("SimpleText")
public class TestTaxonomyCombined extends FacetTestCase {
@@ -305,7 +303,7 @@ public class TestTaxonomyCombined extend
TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);
assertEquals(1, tr.getSize());
assertEquals(0, tr.getPath(0).length);
- assertEquals(TaxonomyReader.INVALID_ORDINAL, tr.getParent(0));
+ assertEquals(TaxonomyReader.INVALID_ORDINAL, tr.getParallelTaxonomyArrays().parents()[0]);
assertEquals(0, tr.getOrdinal(CategoryPath.EMPTY));
tr.close();
indexDir.close();
@@ -324,7 +322,7 @@ public class TestTaxonomyCombined extend
TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);
assertEquals(1, tr.getSize());
assertEquals(0, tr.getPath(0).length);
- assertEquals(TaxonomyReader.INVALID_ORDINAL, tr.getParent(0));
+ assertEquals(TaxonomyReader.INVALID_ORDINAL, tr.getParallelTaxonomyArrays().parents()[0]);
assertEquals(0, tr.getOrdinal(CategoryPath.EMPTY));
tw.close();
tr.close();
@@ -404,12 +402,13 @@ public class TestTaxonomyCombined extend
TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);
// check that the parent of the root ordinal is the invalid ordinal:
- assertEquals(TaxonomyReader.INVALID_ORDINAL, tr.getParent(0));
+ int[] parents = tr.getParallelTaxonomyArrays().parents();
+ assertEquals(TaxonomyReader.INVALID_ORDINAL, parents[0]);
// check parent of non-root ordinals:
for (int ordinal=1; ordinal<tr.getSize(); ordinal++) {
CategoryPath me = tr.getPath(ordinal);
- int parentOrdinal = tr.getParent(ordinal);
+ int parentOrdinal = parents[ordinal];
CategoryPath parent = tr.getPath(parentOrdinal);
if (parent==null) {
fail("Parent of "+ordinal+" is "+parentOrdinal+
@@ -423,26 +422,6 @@ public class TestTaxonomyCombined extend
}
}
- // check parent of of invalid ordinals:
- try {
- tr.getParent(-1);
- fail("getParent for -1 should throw exception");
- } catch (ArrayIndexOutOfBoundsException e) {
- // ok
- }
- try {
- tr.getParent(TaxonomyReader.INVALID_ORDINAL);
- fail("getParent for INVALID_ORDINAL should throw exception");
- } catch (ArrayIndexOutOfBoundsException e) {
- // ok
- }
- try {
- int parent = tr.getParent(tr.getSize());
- fail("getParent for getSize() should throw exception, but returned "+parent);
- } catch (ArrayIndexOutOfBoundsException e) {
- // ok
- }
-
tr.close();
indexDir.close();
}
@@ -535,26 +514,6 @@ public class TestTaxonomyCombined extend
}
}
- /** Tests TaxonomyReader's getParentArray() method. We do not test this
- method directly, but rather just compare its results to those from
- other methods (which we have already tested above).
- */
- @Test
- public void testReaderParentArray() throws Exception {
- Directory indexDir = newDirectory();
- TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
- fillTaxonomy(tw);
- tw.close();
- TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);
- int[] parents = tr.getParallelTaxonomyArrays().parents();
- assertEquals(tr.getSize(), parents.length);
- for (int i=0; i<tr.getSize(); i++) {
- assertEquals(tr.getParent(i), parents[i]);
- }
- tr.close();
- indexDir.close();
- }
-
/**
* Test TaxonomyReader's child browsing method, getChildrenArrays()
* This only tests for correctness of the data on one example - we have
@@ -638,10 +597,11 @@ public class TestTaxonomyCombined extend
assertEquals(tr.getSize(), olderSiblingArray.length);
// test that the "youngest child" of every category is indeed a child:
+ int[] parents = tr.getParallelTaxonomyArrays().parents();
for (int i=0; i<tr.getSize(); i++) {
int youngestChild = children[i];
if (youngestChild != TaxonomyReader.INVALID_ORDINAL) {
- assertEquals(i, tr.getParent(youngestChild));
+ assertEquals(i, parents[youngestChild]);
}
}
@@ -658,7 +618,7 @@ public class TestTaxonomyCombined extend
if (sibling == TaxonomyReader.INVALID_ORDINAL) {
continue;
}
- assertEquals(tr.getParent(i), tr.getParent(sibling));
+ assertEquals(parents[i], parents[sibling]);
}
// And now for slightly more complex (and less "invariant-like"...)
@@ -670,7 +630,7 @@ public class TestTaxonomyCombined extend
// Find the really youngest child:
int j;
for (j=tr.getSize()-1; j>i; j--) {
- if (tr.getParent(j)==i) {
+ if (parents[j]==i) {
break; // found youngest child
}
}
@@ -687,7 +647,7 @@ public class TestTaxonomyCombined extend
// Find the youngest older sibling:
int j;
for (j=i-1; j>=0; j--) {
- if (tr.getParent(j)==tr.getParent(i)) {
+ if (parents[j]==parents[i]) {
break; // found youngest older sibling
}
}
@@ -879,47 +839,21 @@ public class TestTaxonomyCombined extend
tw.commit();
TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);
- int author = 1;
-
- // getParent() and getSize() test:
- try {
- tr.getParent(author);
- fail("Initially, getParent for "+author+" should throw exception");
- } catch (ArrayIndexOutOfBoundsException e) {
- // ok
- }
assertEquals(1, tr.getSize()); // the empty taxonomy has size 1 (the root)
tw.addCategory(new CategoryPath("Author"));
- try {
- tr.getParent(author);
- fail("Before commit() and refresh(), getParent for "+author+" should still throw exception");
- } catch (ArrayIndexOutOfBoundsException e) {
- // ok
- }
assertEquals(1, tr.getSize()); // still root only...
assertNull(TaxonomyReader.openIfChanged(tr)); // this is not enough, because tw.commit() hasn't been done yet
- try {
- tr.getParent(author);
- fail("Before commit() and refresh(), getParent for "+author+" should still throw exception");
- } catch (ArrayIndexOutOfBoundsException e) {
- // ok
- }
assertEquals(1, tr.getSize()); // still root only...
tw.commit();
- try {
- tr.getParent(author);
- fail("Before refresh(), getParent for "+author+" should still throw exception");
- } catch (ArrayIndexOutOfBoundsException e) {
- // ok
- }
assertEquals(1, tr.getSize()); // still root only...
TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(tr);
assertNotNull(newTaxoReader);
tr.close();
tr = newTaxoReader;
+ int author = 1;
try {
- assertEquals(TaxonomyReader.ROOT_ORDINAL, tr.getParent(author));
+ assertEquals(TaxonomyReader.ROOT_ORDINAL, tr.getParallelTaxonomyArrays().parents()[author]);
// ok
} catch (ArrayIndexOutOfBoundsException e) {
fail("After category addition, commit() and refresh(), getParent for "+author+" should NOT throw exception");
@@ -937,9 +871,10 @@ public class TestTaxonomyCombined extend
assertNotNull(newTaxoReader);
tr.close();
tr = newTaxoReader;
- assertEquals(author, tr.getParent(dawkins));
- assertEquals(TaxonomyReader.ROOT_ORDINAL, tr.getParent(author));
- assertEquals(TaxonomyReader.INVALID_ORDINAL, tr.getParent(TaxonomyReader.ROOT_ORDINAL));
+ int[] parents = tr.getParallelTaxonomyArrays().parents();
+ assertEquals(author, parents[dawkins]);
+ assertEquals(TaxonomyReader.ROOT_ORDINAL, parents[author]);
+ assertEquals(TaxonomyReader.INVALID_ORDINAL, parents[TaxonomyReader.ROOT_ORDINAL]);
assertEquals(3, tr.getSize());
tw.close();
tr.close();