You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by al...@apache.org on 2014/04/01 22:10:18 UTC
svn commit: r1583771 - in /jackrabbit/oak/trunk:
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/
oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/
oak-core/src/test/java/org/apache/jackrabbit/oak/spi/query/ oak-lucen...
Author: alexparvulescu
Date: Tue Apr 1 20:10:17 2014
New Revision: 1583771
URL: http://svn.apache.org/r1583771
Log:
OAK-1654 Composite index aggregates
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/spi/query/CursorsTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java?rev=1583771&r1=1583770&r2=1583771&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java Tue Apr 1 20:10:17 2014
@@ -16,15 +16,15 @@
*/
package org.apache.jackrabbit.oak.plugins.index.aggregate;
-import java.util.ArrayList;
+
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.jackrabbit.oak.api.PropertyValue;
-import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.query.fulltext.FullTextAnd;
import org.apache.jackrabbit.oak.query.fulltext.FullTextExpression;
import org.apache.jackrabbit.oak.query.fulltext.FullTextOr;
@@ -32,14 +32,17 @@ import org.apache.jackrabbit.oak.query.f
import org.apache.jackrabbit.oak.query.fulltext.FullTextVisitor;
import org.apache.jackrabbit.oak.query.index.FilterImpl;
import org.apache.jackrabbit.oak.spi.query.Cursor;
+import org.apache.jackrabbit.oak.spi.query.Cursors;
import org.apache.jackrabbit.oak.spi.query.Cursors.AbstractCursor;
import org.apache.jackrabbit.oak.spi.query.Filter;
import org.apache.jackrabbit.oak.spi.query.IndexRow;
import org.apache.jackrabbit.oak.spi.query.QueryIndex.FulltextQueryIndex;
import org.apache.jackrabbit.oak.spi.state.NodeState;
+import com.google.common.base.Function;
import com.google.common.base.Predicates;
import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
/**
* A virtual full-text that can aggregate nodes based on aggregate definitions.
@@ -58,7 +61,13 @@ public class AggregateIndex implements F
if (baseIndex == null) {
return Double.POSITIVE_INFINITY;
}
- return baseIndex.getCost(filter, rootState) - 0.05;
+ double localCost = Double.POSITIVE_INFINITY;
+ FullTextExpression e = filter.getFullTextConstraint();
+ if (e != null && hasCompositeExpression(e)) {
+ localCost = flattenCost(e, filter, baseIndex, rootState);
+ }
+ double baseCost = baseIndex.getCost(filter, rootState);
+ return Math.min(localCost, baseCost) - 0.05;
}
@Override
@@ -67,70 +76,172 @@ public class AggregateIndex implements F
if (baseIndex.getNodeAggregator() == null) {
return baseIndex.query(filter, rootState);
}
- return new AggregationCursor(baseIndex.query(
- newAggregationFilter(filter), rootState),
- baseIndex.getNodeAggregator(), rootState);
+ return newCursor(filter, baseIndex, rootState);
}
- private static Filter newAggregationFilter(Filter filter) {
- FilterImpl f = new FilterImpl(filter);
- // disables node type checks for now
- f.setMatchesAllTypes(true);
+ private static Cursor newCursor(Filter f, FulltextQueryIndex index,
+ NodeState state) {
+ FullTextExpression e = f.getFullTextConstraint();
+ if (hasCompositeExpression(e)) {
+ Cursor c = flatten(e, f, index, state);
+ if (c != null) {
+ return c;
+ }
+ }
+ return new AggregationCursor(index.query(newAggregationFilter(f, null),
+ state), index.getNodeAggregator(), state);
+ }
+
+ private static boolean hasCompositeExpression(FullTextExpression ft) {
+ if (ft == null) {
+ return false;
+ }
+ final AtomicReference<Boolean> composite = new AtomicReference<Boolean>();
+ composite.set(false);
- // TODO OAK-828
- // FullTextExpression constraint = filter.getFullTextConstraint();
- // constraint = getFlatConstraint(constraint);
- // f.setFullTextConstraint(constraint);
+ ft.accept(new FullTextVisitor() {
- return f;
+ @Override
+ public boolean visit(FullTextTerm term) {
+ return true;
+ }
+
+ @Override
+ public boolean visit(FullTextAnd and) {
+ composite.set(true);
+ return true;
+ }
+
+ @Override
+ public boolean visit(FullTextOr or) {
+ composite.set(true);
+ return true;
+ }
+ });
+ return composite.get() && !hasNegativeContains(ft);
}
- static FullTextExpression getFlatConstraint(
- FullTextExpression constraint) {
+ private static boolean hasNegativeContains(FullTextExpression ft) {
+ if (ft == null) {
+ return false;
+ }
+ final AtomicReference<Boolean> hasNegative = new AtomicReference<Boolean>();
+ hasNegative.set(false);
+
+ ft.accept(new FullTextVisitor.FullTextVisitorBase() {
+
+ @Override
+ public boolean visit(FullTextTerm term) {
+ if (term.isNot()) {
+ hasNegative.set(true);
+ }
+ return true;
+ }
+
+ });
+ return hasNegative.get();
+ }
+
+ private static Cursor flatten(FullTextExpression constraint,
+ final Filter filter, final FulltextQueryIndex index,
+ final NodeState state) {
if (constraint == null) {
return null;
}
- final AtomicReference<FullTextExpression> result = new AtomicReference<FullTextExpression>();
+ final AtomicReference<Cursor> result = new AtomicReference<Cursor>();
constraint.accept(new FullTextVisitor() {
-
+
@Override
public boolean visit(FullTextTerm term) {
- String p = term.getPropertyName();
- if (p != null) {
- if (PathUtils.getDepth(p) > 1) {
- // remove indirection
- String name = PathUtils.getName(p);
- term = new FullTextTerm(name, term);
- }
+ result.set(filterToCursor(newAggregationFilter(filter, term),
+ index, state));
+ return true;
+ }
+
+ @Override
+ public boolean visit(FullTextAnd and) {
+ Iterator<FullTextExpression> iterator = and.list.iterator();
+ Cursor c = flatten(iterator.next(), filter, index, state);
+ while (iterator.hasNext()) {
+ FullTextExpression input = iterator.next();
+ Cursor newC = flatten(input, filter, index, state);
+ c = Cursors.newIntersectionCursor(c, newC,
+ filter.getQueryEngineSettings());
}
- result.set(term);
+ result.set(c);
+ return true;
+ }
+
+ @Override
+ public boolean visit(FullTextOr or) {
+ List<Cursor> cursors = Lists.transform(or.list,
+ new Function<FullTextExpression, Cursor>() {
+ @Override
+ public Cursor apply(FullTextExpression input) {
+ return flatten(input, filter, index, state);
+ }
+ });
+ result.set(Cursors.newConcatCursor(cursors,
+ filter.getQueryEngineSettings()));
+ return true;
+ }
+ });
+ return result.get();
+ }
+
+ private static double flattenCost(FullTextExpression constraint,
+ final Filter filter, final FulltextQueryIndex index,
+ final NodeState state) {
+ if (constraint == null) {
+ return Double.POSITIVE_INFINITY;
+ }
+ final AtomicReference<Double> result = new AtomicReference<Double>();
+ result.set(0d);
+ constraint.accept(new FullTextVisitor() {
+
+ @Override
+ public boolean visit(FullTextTerm term) {
+ result.set(result.get() + index.getCost(newAggregationFilter(filter, term), state));
return true;
}
@Override
public boolean visit(FullTextAnd and) {
- ArrayList<FullTextExpression> list = new ArrayList<FullTextExpression>();
- for (FullTextExpression e : and.list) {
- list.add(getFlatConstraint(e));
+ for (FullTextExpression input : and.list) {
+ double d = flattenCost(input, filter, index, state);
+ result.set(result.get() + d);
}
- result.set(new FullTextAnd(list));
return true;
}
@Override
public boolean visit(FullTextOr or) {
- ArrayList<FullTextExpression> list = new ArrayList<FullTextExpression>();
- for (FullTextExpression e : or.list) {
- list.add(getFlatConstraint(e));
+ for (FullTextExpression input : or.list) {
+ double d = flattenCost(input, filter, index, state);
+ result.set(result.get() + d);
}
- result.set(new FullTextOr(list));
return true;
}
-
});
return result.get();
}
+ private static Cursor filterToCursor(Filter f, FulltextQueryIndex index,
+ NodeState state) {
+ return new AggregationCursor(index.query(f, state),
+ index.getNodeAggregator(), state);
+ }
+
+ private static Filter newAggregationFilter(Filter filter, FullTextExpression exp) {
+ FilterImpl f = new FilterImpl(filter);
+ // disables node type checks for now
+ f.setMatchesAllTypes(true);
+ if (exp != null) {
+ f.setFullTextConstraint(exp);
+ }
+ return f;
+ }
+
@Override
public String getPlan(Filter filter, NodeState rootState) {
if (baseIndex == null) {
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java?rev=1583771&r1=1583770&r2=1583771&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java Tue Apr 1 20:10:17 2014
@@ -20,6 +20,7 @@ import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import javax.annotation.Nullable;
@@ -57,6 +58,10 @@ public class Cursors {
return new IntersectionCursor(a, b, settings);
}
+ public static Cursor newConcatCursor(List<Cursor> cursors, QueryEngineSettings settings) {
+ return new ConcatCursor(cursors, settings);
+ }
+
/**
* Creates a {@link Cursor} over paths.
*
@@ -361,9 +366,13 @@ public class Cursors {
if (!init) {
fetchNext();
init = true;
+ if (closed) {
+ throw new IllegalStateException("This cursor is closed");
+ }
}
IndexRow result = current;
- fetchNext();
+ // fetchNext();
+ init = false;
return result;
}
@@ -402,8 +411,7 @@ public class Cursors {
}
secondSet.put(p2, s);
FilterIterators.checkMemoryLimit(secondSet.size(), settings.getLimitInMemory());
- }
- closed = true;
+ }
}
}
@@ -413,5 +421,75 @@ public class Cursors {
}
}
-
+
+ /**
+ * A cursor that combines multiple cursors into a single cursor.
+ */
+ private static class ConcatCursor extends AbstractCursor {
+
+ private final HashSet<String> seen = new HashSet<String>();
+ private final List<Cursor> cursors;
+ private final QueryEngineSettings settings;
+ private boolean init;
+ private boolean closed;
+
+ private Cursor currentCursor;
+ private IndexRow current;
+
+ ConcatCursor(List<Cursor> cursors, QueryEngineSettings settings) {
+ this.cursors = cursors;
+ this.settings = settings;
+ this.currentCursor = cursors.remove(0);
+ }
+
+ @Override
+ public IndexRow next() {
+ if (closed) {
+ throw new IllegalStateException("This cursor is closed");
+ }
+ if (!init) {
+ fetchNext();
+ init = true;
+ }
+ IndexRow result = current;
+ fetchNext();
+ return result;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (!closed && !init) {
+ fetchNext();
+ init = true;
+ }
+ return !closed;
+ }
+
+ private void fetchNext() {
+ while (true) {
+ while (!currentCursor.hasNext()) {
+ if (cursors.isEmpty()) {
+ closed = true;
+ return;
+ } else {
+ currentCursor = cursors.remove(0);
+ }
+ }
+ IndexRow c = currentCursor.next();
+ String p = c.getPath();
+ if (seen.contains(p)) {
+ continue;
+ }
+ current = c;
+ markSeen(p);
+ return;
+ }
+ }
+
+ private void markSeen(String path) {
+ seen.add(path);
+ FilterIterators.checkMemoryLimit(seen.size(), settings.getLimitInMemory());
+ }
+
+ }
}
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/spi/query/CursorsTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/spi/query/CursorsTest.java?rev=1583771&r1=1583770&r2=1583771&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/spi/query/CursorsTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/spi/query/CursorsTest.java Tue Apr 1 20:10:17 2014
@@ -45,7 +45,7 @@ public class CursorsTest {
@Test
public void intersectionCursorExceptions() {
QueryEngineSettings s = new QueryEngineSettings();
- Cursor a = new SimpleCursor("1:", "/b", "/c", "/e", "/e", "/c");
+ Cursor a = new SimpleCursor("1:", "/x", "/b", "/c", "/e", "/e", "/c");
Cursor b = new SimpleCursor("2:", "/a", "/c", "/d", "/b", "/c");
Cursor c = Cursors.newIntersectionCursor(a, b, s);
c.next();
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java?rev=1583771&r1=1583770&r2=1583771&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java Tue Apr 1 20:10:17 2014
@@ -34,11 +34,12 @@ import org.apache.jackrabbit.oak.api.Typ
import org.apache.jackrabbit.oak.plugins.index.aggregate.AggregateIndexProvider;
import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator;
import org.apache.jackrabbit.oak.plugins.index.aggregate.SimpleNodeAggregator;
+
import static org.apache.jackrabbit.oak.plugins.memory.BinaryPropertyState.binaryProperty;
+
import org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent;
import org.apache.jackrabbit.oak.query.AbstractQueryTest;
import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider;
-import org.junit.Ignore;
import org.junit.Test;
import com.google.common.collect.ImmutableList;
@@ -361,7 +362,6 @@ public class LuceneIndexAggregationTest
}
@Test
- @Ignore("OAK-828")
public void testDifferentNodes() throws Exception {
Tree folder = root.getTree("/").addChild("myFolder");