You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2015/06/22 06:30:42 UTC
svn commit: r1686772 - in /jackrabbit/oak/trunk:
oak-core/src/main/java/org/apache/jackrabbit/oak/query/
oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/
oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/
oak-lucene/s...
Author: amitj
Date: Mon Jun 22 04:30:41 2015
New Revision: 1686772
URL: http://svn.apache.org/r1686772
Log:
OAK-2943: Support measure for union queries
Added a measuring iterator and support for measuring of union queries by merging the underlying queries.
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/UnionQueryImpl.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/MultiPropertyOrTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java?rev=1686772&r1=1686771&r2=1686772&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java Mon Jun 22 04:30:41 2015
@@ -21,8 +21,13 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
import org.apache.jackrabbit.oak.api.PropertyValue;
import org.apache.jackrabbit.oak.api.Tree;
import org.apache.jackrabbit.oak.api.Type;
@@ -470,7 +475,7 @@ public class QueryImpl implements Query
logDebug("query execute " + statement);
logDebug("query plan " + getPlan());
}
- RowIterator rowIt = new RowIterator(context.getBaseState());
+ final RowIterator rowIt = new RowIterator(context.getBaseState());
Comparator<ResultRowImpl> orderBy;
boolean sortUsingIndex = false;
// TODO add issue about order by optimization for multiple selectors
@@ -511,41 +516,34 @@ public class QueryImpl implements Query
} else {
orderBy = ResultRowImpl.getComparator(orderings);
}
- Iterator<ResultRowImpl> it =
+ Iterator<ResultRowImpl> it =
FilterIterators.newCombinedFilter(rowIt, distinct, limit, offset, orderBy, settings);
if (measure) {
- // run the query
- while (it.hasNext()) {
- it.next();
- }
- columns = new ColumnImpl[] {
- new ColumnImpl("measure", "selector", "selector"),
- new ColumnImpl("measure", "scanCount", "scanCount")
+ // return the measuring iterator delegating the readCounts to the rowIterator
+ it = new MeasuringIterator(this, it) {
+ @Override
+ protected void setColumns(ColumnImpl[] col) {
+ columns = col;
+ }
+
+ @Override
+ protected long getReadCount() {
+ return rowIt.getReadCount();
+ }
+
+ @Override
+ protected Map<String, Long> getSelectorScanCount() {
+ Map<String, Long> selectorReadCounts = Maps.newHashMap();
+ for (SelectorImpl selector : selectors) {
+ selectorReadCounts.put(selector.getSelectorName(), selector.getScanCount());
+ }
+ return selectorReadCounts;
+ }
};
- ArrayList<ResultRowImpl> list = new ArrayList<ResultRowImpl>();
- ResultRowImpl r = new ResultRowImpl(this,
- Tree.EMPTY_ARRAY,
- new PropertyValue[] {
- PropertyValues.newString("query"),
- PropertyValues.newLong(rowIt.getReadCount())
- },
- null, null);
- list.add(r);
- for (SelectorImpl selector : selectors) {
- r = new ResultRowImpl(this,
- Tree.EMPTY_ARRAY,
- new PropertyValue[] {
- PropertyValues.newString(selector.getSelectorName()),
- PropertyValues.newLong(selector.getScanCount()),
- },
- null, null);
- list.add(r);
- }
- it = list.iterator();
}
return it;
}
-
+
@Override
public String getPlan() {
return source.getPlan(context.getBaseState());
@@ -601,7 +599,7 @@ public class QueryImpl implements Query
}
estimatedCost = result.prepare().getEstimatedCost();
source = result;
-
+
}
private static SourceImpl buildJoin(SourceImpl result, SourceImpl last, List<JoinConditionImpl> conditions) {
@@ -636,6 +634,98 @@ public class QueryImpl implements Query
return source.createFilter(preparing);
}
+
+ /**
+ * Abstract decorating iterator for measure queries. The iterator delegates to the underlying actual
+ * query iterator to lazily execute and return counts.
+ */
+ abstract static class MeasuringIterator extends AbstractIterator<ResultRowImpl> {
+ private Iterator<ResultRowImpl> delegate;
+ private Query query;
+ private List<ResultRowImpl> results;
+ private boolean init;
+
+ MeasuringIterator(Query query, Iterator<ResultRowImpl> delegate) {
+ this.query = query;
+ this.delegate = delegate;
+ results = Lists.newArrayList();
+ }
+
+ @Override
+ protected ResultRowImpl computeNext() {
+ if (!init) {
+ getRows();
+ }
+
+ if (!results.isEmpty()) {
+ return results.remove(0);
+ } else {
+ return endOfData();
+ }
+ }
+
+ void getRows() {
+ // run the query
+ while (delegate.hasNext()) {
+ delegate.next();
+ }
+
+ ColumnImpl[] columns = new ColumnImpl[] {
+ new ColumnImpl("measure", "selector", "selector"),
+ new ColumnImpl("measure", "scanCount", "scanCount")
+ };
+ setColumns(columns);
+
+ ResultRowImpl r = new ResultRowImpl(query,
+ Tree.EMPTY_ARRAY,
+ new PropertyValue[] {
+ PropertyValues.newString("query"),
+ PropertyValues.newLong(getReadCount())
+ },
+ null, null);
+ results.add(r);
+
+ Map<String, Long> selectorScanCount = getSelectorScanCount();
+ for (String selector : selectorScanCount.keySet()) {
+ r = new ResultRowImpl(query,
+ Tree.EMPTY_ARRAY,
+ new PropertyValue[] {
+ PropertyValues.newString(selector),
+ PropertyValues.newLong(selectorScanCount.get(selector)),
+ },
+ null, null);
+ results.add(r);
+ }
+ init = true;
+ }
+
+ /**
+ * Set the measure specific columns in the query object
+ * @param columns the measure specific columns
+ */
+ protected abstract void setColumns(ColumnImpl[] columns);
+
+ /**
+ * Retrieve the selector scan count
+ * @return map of selector to scan count
+ */
+ protected abstract Map<String, Long> getSelectorScanCount();
+
+ /**
+ * Retrieve the query read count
+ * @return count
+ */
+ protected abstract long getReadCount();
+
+ /**
+ * Retrieves the actual query iterator
+ * @return the delegate
+ */
+ protected Iterator<ResultRowImpl> getDelegate() {
+ return delegate;
+ }
+ }
+
/**
* An iterator over result rows.
*/
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/UnionQueryImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/UnionQueryImpl.java?rev=1686772&r1=1686771&r2=1686772&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/UnionQueryImpl.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/UnionQueryImpl.java Mon Jun 22 04:30:41 2015
@@ -19,6 +19,9 @@ import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.Maps;
import org.apache.jackrabbit.oak.api.PropertyValue;
import org.apache.jackrabbit.oak.api.Result;
@@ -26,6 +29,7 @@ import org.apache.jackrabbit.oak.api.Res
import org.apache.jackrabbit.oak.api.Tree;
import org.apache.jackrabbit.oak.query.ast.ColumnImpl;
import org.apache.jackrabbit.oak.query.ast.OrderingImpl;
+import org.apache.jackrabbit.oak.query.QueryImpl.MeasuringIterator;
import org.apache.jackrabbit.oak.spi.query.PropertyValues;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -128,6 +132,9 @@ public class UnionQueryImpl implements Q
@Override
public ColumnImpl[] getColumns() {
+ if (columns != null) {
+ return columns;
+ }
return left.getColumns();
}
@@ -258,14 +265,60 @@ public class UnionQueryImpl implements Q
LOG.debug("query union plan {}", getPlan());
}
}
- Iterator<ResultRowImpl> it = Iterators.concat(left.getRows(), right.getRows());
- if (measure) {
- // both queries measure themselves
- return it;
- }
boolean distinct = !unionAll;
Comparator<ResultRowImpl> orderBy = ResultRowImpl.getComparator(orderings);
- it = FilterIterators.newCombinedFilter(it, distinct, limit, offset, orderBy, settings);
+
+ Iterator<ResultRowImpl> it;
+ final Iterator<ResultRowImpl> leftRows = left.getRows();
+ final Iterator<ResultRowImpl> rightRows = right.getRows();
+ Iterator<ResultRowImpl> leftIter = leftRows;
+ Iterator<ResultRowImpl> rightIter = rightRows;
+
+ // if measure retrieve the backing delegate iterator instead
+ if (measure) {
+ leftIter = ((MeasuringIterator) leftRows).getDelegate();
+ rightIter = ((MeasuringIterator) rightRows).getDelegate();
+ }
+
+ it = FilterIterators
+ .newCombinedFilter(Iterators.concat(leftIter, rightIter), distinct, limit, offset, orderBy, settings);
+
+ if (measure) {
+ // return the measuring iterator for the union
+ it = new MeasuringIterator(this, it) {
+ MeasuringIterator left = (MeasuringIterator) leftRows;
+ MeasuringIterator right = (MeasuringIterator) rightRows;
+
+ @Override
+ protected void setColumns(ColumnImpl[] cols) {
+ columns = cols;
+ left.setColumns(cols);
+ right.setColumns(cols);
+ }
+
+ @Override
+ protected Map<String, Long> getSelectorScanCount() {
+ // Merge the 2 maps from the left and right queries to get the selector counts
+ Map<String, Long> leftSelectorScan = left.getSelectorScanCount();
+ Map<String, Long> rightSelectorScan = right.getSelectorScanCount();
+ Map<String, Long> unionScan = Maps.newHashMap(leftSelectorScan);
+ for (String key : rightSelectorScan.keySet()) {
+ if (unionScan.containsKey(key)) {
+ unionScan.put(key, rightSelectorScan.get(key) + unionScan.get(key));
+ } else {
+ unionScan.put(key, rightSelectorScan.get(key));
+ }
+ }
+ return unionScan;
+ }
+
+ @Override
+ protected long getReadCount() {
+ return left.getReadCount() + right.getReadCount();
+ }
+ };
+ }
+
return it;
}
@@ -273,5 +326,4 @@ public class UnionQueryImpl implements Q
public void setInternal(boolean isInternal) {
this.isInternal = isInternal;
}
-
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java?rev=1686772&r1=1686771&r2=1686772&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java Mon Jun 22 04:30:41 2015
@@ -28,8 +28,8 @@ import org.apache.jackrabbit.oak.query.x
*/
public class Statement {
- private boolean explain;
- private boolean measure;
+ boolean explain;
+ boolean measure;
/**
* The selector to get the columns from (the selector used in the select
@@ -52,9 +52,6 @@ public class Statement {
public Statement optimize() {
ignoreOrderByScoreDesc();
- if (explain || measure) {
- return this;
- }
if (where == null) {
return this;
}
@@ -80,6 +77,9 @@ public class Statement {
}
union.orderList = orderList;
union.xpathQuery = xpathQuery;
+ union.measure = measure;
+ union.explain = explain;
+
return union;
}
@@ -249,6 +249,12 @@ public class Statement {
@Override
public String toString() {
StringBuilder buff = new StringBuilder();
+ // explain | measure ...
+ if (this.explain) {
+ buff.append("explain ");
+ } else if (measure) {
+ buff.append("measure ");
+ }
buff.append(s1).append(" union ").append(s2);
// order by ...
if (orderList != null && !orderList.isEmpty()) {
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/MultiPropertyOrTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/MultiPropertyOrTest.java?rev=1686772&r1=1686771&r2=1686772&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/MultiPropertyOrTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/MultiPropertyOrTest.java Mon Jun 22 04:30:41 2015
@@ -16,17 +16,29 @@
*/
package org.apache.jackrabbit.oak.plugins.index.property;
+import static com.google.common.collect.ImmutableSet.of;
+import static org.apache.jackrabbit.oak.api.QueryEngine.NO_MAPPINGS;
+import static org.apache.jackrabbit.oak.api.Type.STRINGS;
+import static org.apache.jackrabbit.oak.plugins.memory.PropertyStates.createProperty;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
+import static org.junit.matchers.JUnitMatchers.containsString;
+import java.text.ParseException;
import java.util.List;
+import java.util.Random;
import javax.annotation.Nonnull;
import javax.jcr.query.Query;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import org.apache.jackrabbit.oak.Oak;
import org.apache.jackrabbit.oak.api.ContentRepository;
+import org.apache.jackrabbit.oak.api.PropertyValue;
+import org.apache.jackrabbit.oak.api.ResultRow;
import org.apache.jackrabbit.oak.api.Tree;
import org.apache.jackrabbit.oak.plugins.index.IndexUtils;
import org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent;
@@ -50,7 +62,7 @@ public class MultiPropertyOrTest extends
NodeBuilder index = IndexUtils.getOrCreateOakIndex(builder);
IndexUtils.createIndexDefinition(
index, "xyz", true, false,
- ImmutableList.<String>of("x", "y", "z"), null);
+ ImmutableList.of("x", "y", "z"), null);
}
})
.with(new OpenSecurityProvider())
@@ -102,4 +114,51 @@ public class MultiPropertyOrTest extends
ImmutableList.of("/a", "/b"));
setTraversalEnabled(false);
}
+
+ @Test
+ public void unionSortResultCount() throws Exception {
+ // create test data
+ Tree test = root.getTree("/").addChild("test");
+ root.commit();
+
+ List<Integer> nodes = Lists.newArrayList();
+ Random r = new Random();
+ int seed = -2;
+ for (int i = 0; i < 1000; i++) {
+ Tree a = test.addChild("a" + i);
+ a.setProperty("x", "fooa");
+ seed += 2;
+ int num = r.nextInt(100);
+ a.setProperty("z", num);
+ nodes.add(num);
+ }
+
+ seed = -1;
+ for (int i = 0; i < 1000; i++) {
+ Tree a = test.addChild("b" + i);
+ a.setProperty("y", "foob");
+ seed += 2;
+ int num = 100 + r.nextInt(100);
+ a.setProperty("z", num);
+ nodes.add(num);
+ }
+ root.commit();
+
+ // scan count scans the whole result set
+ String query =
+ "measure /jcr:root//element(*, nt:base)[(@x = 'fooa' or @y = 'foob')] order by @z";
+ assertThat(measureWithLimit(query, XPATH, 100), containsString("scanCount: 2000"));
+ }
+
+ private String measureWithLimit(String query, String lang, int limit) throws ParseException {
+ List<? extends ResultRow> result = Lists.newArrayList(
+ qe.executeQuery(query, lang, limit, 0, Maps.<String, PropertyValue>newHashMap(),
+ NO_MAPPINGS).getRows());
+
+ String measure = "";
+ if (result.size() > 0) {
+ measure = result.get(0).toString();
+ }
+ return measure;
+ }
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java?rev=1686772&r1=1686771&r2=1686772&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java Mon Jun 22 04:30:41 2015
@@ -1358,6 +1358,46 @@ public class LucenePropertyIndexTest ext
assertQuery(queryString, "xpath", asList("/test/b"));
}
+ @Test
+ public void unionSortResultCount() throws Exception {
+ // Index Definition
+ Tree idx = createIndex("test1", of("propa", "propb", "propc"));
+ idx.setProperty(createProperty(ORDERED_PROP_NAMES, of("propc"), STRINGS));
+ useV2(idx);
+
+ // create test data
+ Tree test = root.getTree("/").addChild("test");
+ root.commit();
+
+ List<Integer> nodes = Lists.newArrayList();
+ Random r = new Random();
+ int seed = -2;
+ for (int i = 0; i < 1000; i++) {
+ Tree a = test.addChild("a" + i);
+ a.setProperty("propa", "fooa");
+ seed += 2;
+ int num = r.nextInt(100);
+ a.setProperty("propc", num);
+ nodes.add(num);
+ }
+
+ seed = -1;
+ for (int i = 0; i < 1000; i++) {
+ Tree a = test.addChild("b" + i);
+ a.setProperty("propb", "foob");
+ seed += 2;
+ int num = 100 + r.nextInt(100);
+ a.setProperty("propc", num);
+ nodes.add(num);
+ }
+ root.commit();
+
+ // scan count scans the whole result set
+ String query =
+ "measure /jcr:root//element(*, nt:base)[(@propa = 'fooa' or @propb = 'foob')] order by @propc";
+ assertThat(measureWithLimit(query, XPATH, 100), containsString("scanCount: 200"));
+ }
+
private Tree createFileNode(Tree tree, String name, String content, String mimeType){
return createFileNode(tree, name, new ArrayBasedBlob(content.getBytes()), mimeType);
}