You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2016/05/10 04:06:19 UTC
svn commit: r1743094 - in /jackrabbit/oak/branches/1.2: ./ oak-blob-cloud/
oak-core/src/main/java/org/apache/jackrabbit/oak/query/
oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/
oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/in...
Author: amitj
Date: Tue May 10 04:06:19 2016
New Revision: 1743094
URL: http://svn.apache.org/viewvc?rev=1743094&view=rev
Log:
OAK-2618 Improve performance of queries with ORDER BY and multiple OR filters
Merge revisions 1686772,1686790 from trunk
Modified:
jackrabbit/oak/branches/1.2/ (props changed)
jackrabbit/oak/branches/1.2/oak-blob-cloud/pom.xml
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/Query.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/UnionQueryImpl.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/MultiPropertyOrTest.java
jackrabbit/oak/branches/1.2/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
Propchange: jackrabbit/oak/branches/1.2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 10 04:06:19 2016
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1672350,1672468,1672537,1672603,1672611,1672642,1672644,1672834-1672835,1673351,1673410,1673414-1673415,1673436,1673644,1673662-1673664,1673669,1673695,1673713,1673738,1673787,1673791,1674046,1674065,1674075,1674107,1674228,1674780,1674880,1675054-1675055,1675319,1675332,1675354,1675357,1675382,1675555,1675566,1675593,1676198,1676237,1676407,1676458,1676539,1676670,1676693,1676703,1676725,1677579,1677581,1677609,1677611,1677774,1677788,1677797,1677804,1677806,1677939,1677991,1678023,1678095-1678096,1678124,1678171,1678173,1678202,1678211,1678323,1678758,1678938,1678954,1679144,1679165,1679191,1679232,1679235,1679503,1679958,1679961,1680170,1680172,1680182,1680222,1680232,1680236,1680461,1680633,1680643,1680747,1680805-1680806,1680903,1681282,1681767,1681918,1681921,1681955,1682042,1682218,1682235,1682437,1682488,1682494,1682555,1682855,1682904,1683059,1683089,1683213,1683249,1683259,1683278,1683323,1683687,1683700,1684174-1684175,1684186,1684376,1684442,1684561
,1684570,1684601,1684618,1684669,1684820,1684868,1684894,1685023,1685075,1685370,1685541,1685552,1685589-1685590,1685840,1685964,1685977,1685989,1685999,1686003,1686023,1686032,1686097,1686162,1686229,1686234,1686253,1686414,1686780,1686854,1686857,1686971,1687053-1687055,1687175,1687196,1687198,1687220,1687239-1687240,1687301,1687441,1687553,1688089-1688090,1688172,1688179,1688349,1688421,1688436,1688453,1688616,1688622,1688634,1688636,1688817,1689003-1689004,1689008,1689577,1689581,1689623,1689810,1689828,1689831,1689833,1689903,1690017,1690043,1690047,1690057,1690247,1690249,1690634-1690637,1690650,1690669,1690674,1690885,1690941,1691139,1691151,1691159,1691167,1691183,1691188,1691210,1691280,1691307,1691331-1691333,1691345,1691384-1691385,1691401,1691509,1692133-1692134,1692156,1692250,1692274,1692363,1692382,1692478,1692955,1693002,1693030,1693050,1693209,1693401,1693421,1693525-1693526,1694007,1694393-1694394,1694651,1694653-1694654,1695032,1695050,1695122,1695280,1695299,1695
420,1695457,1695482,1695492,1695507,1695521,1695540,1695571,1695905,1696190,1696194,1696242,1696285,1696375,1696522,1696578,1696759,1696916,1697363,1697373,1697410,1697582,1697589,1697616,1697672,1697896,1700191,1700231,1700397,1700403,1700506,1700571,1700718,1700720,1700727,1700749,1700769,1700775,1701065,1701619,1701733,1701743,1701750,1701768,1701806,1701810,1701814,1701907,1701948,1701955,1701959,1701965,1701986,1702014,1702022,1702045,1702051,1702241,1702272,1702371,1702387,1702405,1702423,1702426,1702428,1702860,1702866,1702942,1702960,1703212,1703382,1703395,1703411,1703428,1703430,1703568,1703592,1703758,1703858,1703878,1704256,1704282,1704285,1704457,1704479,1704490,1704614,1704629,1704636,1704655,1704670,1704886,1705005,1705027,1705043,1705055,1705250,1705268,1705273,1705323,1705677,1705701,1705871,1705992,1705998,1706009,1706037,1706059,1706212,1706218,1706270,1706764,1706772,1707049,1707189,1707191,1707331,1707435,1707509,1708049,1708105,1708307,1708315,1708546,1708592,1
708766,1709012,1709852,1709978,1710013,1710031,1710049,1710205,1710242,1710559,1710575,1710590,1710614,1710637,1710789,1710800,1710811,1710816,1710972,1711248,1711282,1711296,1711405,1711498,1711654,1712018,1712042,1712319,1712490,1712531,1712730,1712785,1712963,1713008,1713439,1713461,1713580,1713586,1713599-1713600,1713626,1713698,1713803,1713809,1714034,1714061,1714084,1714170,1714213,1714229,1714238,1714519-1714520,1714543-1714544,1714730,1714739,1714779,1714956,1714961,1715010,1715092,1715191,1715346,1715716,1715767,1715771,1715888,1715898,1716100,1716178,1716426,1716576,1716588-1716589,1716596,1716616,1716703,1716712,1716815,1716823,1716830,1716883,1717203,1717277,1717410,1717462,1717632,1717768-1717769,1717784,1717789,1717988,1718528,1718533,1718547-1718548,1718626,1718646,1718772,1718801-1718802,1718895,1719111,1719288,1719869,1720335,1720350,1720354,1720500,1721160,1721172,1721337,1722141,1722832,1723227,1723239,1723241,1723251,1723254,1723333,1723347,1723350,1723565,172358
4,1723713,1723731,1724026,1724057,1724186,1724210,1724401,1724628,1724631,1725216,1725477,1725515,1725555,1725941,1725960,1726232,1726237,1726570,1726579,1726585-1726586,1726621,1726795,1726797,1726809,1726812,1726981,1726993,1727026,1727254,1727331,1727350,1727358,1727429,1727476,1727508,1727515-1727518,1727893,1728037,1728041,1728070,1728114,1728281,1728443,1728642,1729200,1729505,1729599,1729957,1729979,1730216,1730527,1730581,1730629,1730801,1731627,1731647-1731648,1731789,1731797,1732131,1732268,1732278,1732647-1732648,1732864,1733929,1734230,1734254,1735052,1735405,1735484,1735588,1736176,1737309-1737310,1737334,1737349,1738833,1738950,1738957,1739894,1740971,1741032,1741339,1741343,1742520,1742888,1742916
+/jackrabbit/oak/trunk:1672350,1672468,1672537,1672603,1672611,1672642,1672644,1672834-1672835,1673351,1673410,1673414-1673415,1673436,1673644,1673662-1673664,1673669,1673695,1673713,1673738,1673787,1673791,1674046,1674065,1674075,1674107,1674228,1674780,1674880,1675054-1675055,1675319,1675332,1675354,1675357,1675382,1675555,1675566,1675593,1676198,1676237,1676407,1676458,1676539,1676670,1676693,1676703,1676725,1677579,1677581,1677609,1677611,1677774,1677788,1677797,1677804,1677806,1677939,1677991,1678023,1678095-1678096,1678124,1678171,1678173,1678202,1678211,1678323,1678758,1678938,1678954,1679144,1679165,1679191,1679232,1679235,1679503,1679958,1679961,1680170,1680172,1680182,1680222,1680232,1680236,1680461,1680633,1680643,1680747,1680805-1680806,1680903,1681282,1681767,1681918,1681921,1681955,1682042,1682218,1682235,1682437,1682488,1682494,1682555,1682855,1682904,1683059,1683089,1683213,1683249,1683259,1683278,1683323,1683687,1683700,1684174-1684175,1684186,1684376,1684442,1684561
,1684570,1684601,1684618,1684669,1684820,1684868,1684894,1685023,1685075,1685370,1685541,1685552,1685589-1685590,1685840,1685964,1685977,1685989,1685999,1686003,1686023,1686032,1686097,1686162,1686229,1686234,1686253,1686414,1686772,1686780,1686790,1686854,1686857,1686971,1687053-1687055,1687175,1687196,1687198,1687220,1687239-1687240,1687301,1687441,1687553,1688089-1688090,1688172,1688179,1688349,1688421,1688436,1688453,1688616,1688622,1688634,1688636,1688817,1689003-1689004,1689008,1689577,1689581,1689623,1689810,1689828,1689831,1689833,1689903,1690017,1690043,1690047,1690057,1690247,1690249,1690634-1690637,1690650,1690669,1690674,1690885,1690941,1691139,1691151,1691159,1691167,1691183,1691188,1691210,1691280,1691307,1691331-1691333,1691345,1691384-1691385,1691401,1691509,1692133-1692134,1692156,1692250,1692274,1692363,1692382,1692478,1692955,1693002,1693030,1693050,1693209,1693401,1693421,1693525-1693526,1694007,1694393-1694394,1694651,1694653-1694654,1695032,1695050,1695122,1695
280,1695299,1695420,1695457,1695482,1695492,1695507,1695521,1695540,1695571,1695905,1696190,1696194,1696242,1696285,1696375,1696522,1696578,1696759,1696916,1697363,1697373,1697410,1697582,1697589,1697616,1697672,1697896,1700191,1700231,1700397,1700403,1700506,1700571,1700718,1700720,1700727,1700749,1700769,1700775,1701065,1701619,1701733,1701743,1701750,1701768,1701806,1701810,1701814,1701907,1701948,1701955,1701959,1701965,1701986,1702014,1702022,1702045,1702051,1702241,1702272,1702371,1702387,1702405,1702423,1702426,1702428,1702860,1702866,1702942,1702960,1703212,1703382,1703395,1703411,1703428,1703430,1703568,1703592,1703758,1703858,1703878,1704256,1704282,1704285,1704457,1704479,1704490,1704614,1704629,1704636,1704655,1704670,1704886,1705005,1705027,1705043,1705055,1705250,1705268,1705273,1705323,1705677,1705701,1705871,1705992,1705998,1706009,1706037,1706059,1706212,1706218,1706270,1706764,1706772,1707049,1707189,1707191,1707331,1707435,1707509,1708049,1708105,1708307,1708315,1
708546,1708592,1708766,1709012,1709852,1709978,1710013,1710031,1710049,1710205,1710242,1710559,1710575,1710590,1710614,1710637,1710789,1710800,1710811,1710816,1710972,1711248,1711282,1711296,1711405,1711498,1711654,1712018,1712042,1712319,1712490,1712531,1712730,1712785,1712963,1713008,1713439,1713461,1713580,1713586,1713599-1713600,1713626,1713698,1713803,1713809,1714034,1714061,1714084,1714170,1714213,1714229,1714238,1714519-1714520,1714543-1714544,1714730,1714739,1714779,1714956,1714961,1715010,1715092,1715191,1715346,1715716,1715767,1715771,1715888,1715898,1716100,1716178,1716426,1716576,1716588-1716589,1716596,1716616,1716703,1716712,1716815,1716823,1716830,1716883,1717203,1717277,1717410,1717462,1717632,1717768-1717769,1717784,1717789,1717988,1718528,1718533,1718547-1718548,1718626,1718646,1718772,1718801-1718802,1718895,1719111,1719288,1719869,1720335,1720350,1720354,1720500,1721160,1721172,1721337,1722141,1722832,1723227,1723239,1723241,1723251,1723254,1723333,1723347,172335
0,1723565,1723584,1723713,1723731,1724026,1724057,1724186,1724210,1724401,1724628,1724631,1725216,1725477,1725515,1725555,1725941,1725960,1726232,1726237,1726570,1726579,1726585-1726586,1726621,1726795,1726797,1726809,1726812,1726981,1726993,1727026,1727254,1727331,1727350,1727358,1727429,1727476,1727508,1727515-1727518,1727893,1728037,1728041,1728070,1728114,1728281,1728443,1728642,1729200,1729505,1729599,1729957,1729979,1730216,1730527,1730581,1730629,1730801,1731627,1731647-1731648,1731789,1731797,1732131,1732268,1732278,1732647-1732648,1732864,1733929,1734230,1734254,1735052,1735405,1735484,1735588,1736176,1737309-1737310,1737334,1737349,1738833,1738950,1738957,1739894,1740971,1741032,1741339,1741343,1742520,1742888,1742916
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.2/oak-blob-cloud/pom.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-blob-cloud/pom.xml?rev=1743094&r1=1743093&r2=1743094&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-blob-cloud/pom.xml (original)
+++ jackrabbit/oak/branches/1.2/oak-blob-cloud/pom.xml Tue May 10 04:06:19 2016
@@ -120,7 +120,7 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
- <version>1.10.27</version>
+ <version>1.10.76</version>
</dependency>
<dependency>
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/Query.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/Query.java?rev=1743094&r1=1743093&r2=1743094&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/Query.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/Query.java Tue May 10 04:06:19 2016
@@ -16,6 +16,7 @@ package org.apache.jackrabbit.oak.query;
import java.util.Iterator;
import java.util.List;
+import aQute.bnd.annotation.ProviderType;
import org.apache.jackrabbit.oak.api.PropertyValue;
import org.apache.jackrabbit.oak.api.Result;
import org.apache.jackrabbit.oak.api.Tree;
@@ -29,6 +30,7 @@ import org.apache.jackrabbit.oak.query.a
* initialize the bind variable map. If the query is re-executed, a new instance
* is created.
*/
+@ProviderType
public interface Query {
void setExecutionContext(ExecutionContext context);
@@ -109,4 +111,11 @@ public interface Query {
boolean isMeasureOrExplainEnabled();
void setInternal(boolean internal);
+
+ /**
+ * Returns whether the results will be sorted by index. The query must already be prepared.
+ *
+ * @return if sorted by index
+ */
+ boolean isSortedByIndex();
}
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java?rev=1743094&r1=1743093&r2=1743094&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java Tue May 10 04:06:19 2016
@@ -21,8 +21,13 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
import org.apache.jackrabbit.oak.api.PropertyValue;
import org.apache.jackrabbit.oak.api.Tree;
import org.apache.jackrabbit.oak.api.Type;
@@ -153,6 +158,8 @@ public class QueryImpl implements Query
private boolean prepared;
private ExecutionContext context;
+ private boolean isSortedByIndex;
+
private final NamePathMapper namePathMapper;
private double estimatedCost;
@@ -475,23 +482,62 @@ public class QueryImpl implements Query
logDebug("query execute " + statement);
logDebug("query plan " + getPlan());
}
- RowIterator rowIt = new RowIterator(context.getBaseState());
+ final RowIterator rowIt = new RowIterator(context.getBaseState());
Comparator<ResultRowImpl> orderBy;
- boolean sortUsingIndex = false;
+ if (isSortedByIndex) {
+ orderBy = null;
+ } else {
+ orderBy = ResultRowImpl.getComparator(orderings);
+ }
+ Iterator<ResultRowImpl> it =
+ FilterIterators.newCombinedFilter(rowIt, distinct, limit, offset, orderBy, settings);
+ if (measure) {
+ // return the measuring iterator delegating the readCounts to the rowIterator
+ it = new MeasuringIterator(this, it) {
+ @Override
+ protected void setColumns(ColumnImpl[] col) {
+ columns = col;
+ }
+
+ @Override
+ protected long getReadCount() {
+ return rowIt.getReadCount();
+ }
+
+ @Override
+ protected Map<String, Long> getSelectorScanCount() {
+ Map<String, Long> selectorReadCounts = Maps.newHashMap();
+ for (SelectorImpl selector : selectors) {
+ selectorReadCounts.put(selector.getSelectorName(), selector.getScanCount());
+ }
+ return selectorReadCounts;
+ }
+ };
+ }
+ return it;
+ }
+
+ @Override
+ public boolean isSortedByIndex() {
+ return isSortedByIndex;
+ }
+
+ private boolean canSortByIndex() {
+ boolean canSortByIndex = false;
// TODO add issue about order by optimization for multiple selectors
if (orderings != null && selectors.size() == 1) {
IndexPlan plan = selectors.get(0).getExecutionPlan().getIndexPlan();
if (plan != null) {
List<OrderEntry> list = plan.getSortOrder();
if (list != null && list.size() == orderings.length) {
- sortUsingIndex = true;
+ canSortByIndex = true;
for (int i = 0; i < list.size(); i++) {
OrderEntry e = list.get(i);
OrderingImpl o = orderings[i];
DynamicOperandImpl op = o.getOperand();
if (!(op instanceof PropertyValueImpl)) {
// ordered by a function: currently not supported
- sortUsingIndex = false;
+ canSortByIndex = false;
break;
}
// we only have one selector, so no need to check that
@@ -499,58 +545,21 @@ public class QueryImpl implements Query
String pn = ((PropertyValueImpl) op).getPropertyName();
if (!pn.equals(e.getPropertyName())) {
// ordered by another property
- sortUsingIndex = false;
+ canSortByIndex = false;
break;
}
if (o.isDescending() != (e.getOrder() == Order.DESCENDING)) {
// ordered ascending versus descending
- sortUsingIndex = false;
+ canSortByIndex = false;
break;
}
}
}
}
}
- if (sortUsingIndex) {
- orderBy = null;
- } else {
- orderBy = ResultRowImpl.getComparator(orderings);
- }
- Iterator<ResultRowImpl> it =
- FilterIterators.newCombinedFilter(rowIt, distinct, limit, offset, orderBy, settings);
- if (measure) {
- // run the query
- while (it.hasNext()) {
- it.next();
- }
- columns = new ColumnImpl[] {
- new ColumnImpl("measure", "selector", "selector"),
- new ColumnImpl("measure", "scanCount", "scanCount")
- };
- ArrayList<ResultRowImpl> list = new ArrayList<ResultRowImpl>();
- ResultRowImpl r = new ResultRowImpl(this,
- Tree.EMPTY_ARRAY,
- new PropertyValue[] {
- PropertyValues.newString("query"),
- PropertyValues.newLong(rowIt.getReadCount())
- },
- null, null);
- list.add(r);
- for (SelectorImpl selector : selectors) {
- r = new ResultRowImpl(this,
- Tree.EMPTY_ARRAY,
- new PropertyValue[] {
- PropertyValues.newString(selector.getSelectorName()),
- PropertyValues.newLong(selector.getScanCount()),
- },
- null, null);
- list.add(r);
- }
- it = list.iterator();
- }
- return it;
+ return canSortByIndex;
}
-
+
@Override
public String getPlan() {
return source.getPlan(context.getBaseState());
@@ -573,6 +582,7 @@ public class QueryImpl implements Query
if (sources.size() <= 1) {
// simple case (no join)
estimatedCost = source.prepare().getEstimatedCost();
+ isSortedByIndex = canSortByIndex();
return;
}
@@ -606,7 +616,7 @@ public class QueryImpl implements Query
}
estimatedCost = result.prepare().getEstimatedCost();
source = result;
-
+ isSortedByIndex = canSortByIndex();
}
private static SourceImpl buildJoin(SourceImpl result, SourceImpl last, List<JoinConditionImpl> conditions) {
@@ -641,6 +651,98 @@ public class QueryImpl implements Query
return source.createFilter(preparing);
}
+
+ /**
+ * Abstract decorating iterator for measure queries. The iterator delegates to the underlying actual
+ * query iterator to lazily execute and return counts.
+ */
+ abstract static class MeasuringIterator extends AbstractIterator<ResultRowImpl> {
+ private Iterator<ResultRowImpl> delegate;
+ private Query query;
+ private List<ResultRowImpl> results;
+ private boolean init;
+
+ MeasuringIterator(Query query, Iterator<ResultRowImpl> delegate) {
+ this.query = query;
+ this.delegate = delegate;
+ results = Lists.newArrayList();
+ }
+
+ @Override
+ protected ResultRowImpl computeNext() {
+ if (!init) {
+ getRows();
+ }
+
+ if (!results.isEmpty()) {
+ return results.remove(0);
+ } else {
+ return endOfData();
+ }
+ }
+
+ void getRows() {
+ // run the query
+ while (delegate.hasNext()) {
+ delegate.next();
+ }
+
+ ColumnImpl[] columns = new ColumnImpl[] {
+ new ColumnImpl("measure", "selector", "selector"),
+ new ColumnImpl("measure", "scanCount", "scanCount")
+ };
+ setColumns(columns);
+
+ ResultRowImpl r = new ResultRowImpl(query,
+ Tree.EMPTY_ARRAY,
+ new PropertyValue[] {
+ PropertyValues.newString("query"),
+ PropertyValues.newLong(getReadCount())
+ },
+ null, null);
+ results.add(r);
+
+ Map<String, Long> selectorScanCount = getSelectorScanCount();
+ for (String selector : selectorScanCount.keySet()) {
+ r = new ResultRowImpl(query,
+ Tree.EMPTY_ARRAY,
+ new PropertyValue[] {
+ PropertyValues.newString(selector),
+ PropertyValues.newLong(selectorScanCount.get(selector)),
+ },
+ null, null);
+ results.add(r);
+ }
+ init = true;
+ }
+
+ /**
+ * Set the measure specific columns in the query object
+ * @param columns the measure specific columns
+ */
+ protected abstract void setColumns(ColumnImpl[] columns);
+
+ /**
+ * Retrieve the selector scan count
+ * @return map of selector to scan count
+ */
+ protected abstract Map<String, Long> getSelectorScanCount();
+
+ /**
+ * Retrieve the query read count
+ * @return count
+ */
+ protected abstract long getReadCount();
+
+ /**
+ * Retrieves the actual query iterator
+ * @return the delegate
+ */
+ protected Iterator<ResultRowImpl> getDelegate() {
+ return delegate;
+ }
+ }
+
/**
* An iterator over result rows.
*/
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/UnionQueryImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/UnionQueryImpl.java?rev=1743094&r1=1743093&r2=1743094&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/UnionQueryImpl.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/UnionQueryImpl.java Tue May 10 04:06:19 2016
@@ -19,6 +19,10 @@ import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
import org.apache.jackrabbit.oak.api.PropertyValue;
import org.apache.jackrabbit.oak.api.Result;
@@ -26,6 +30,7 @@ import org.apache.jackrabbit.oak.api.Res
import org.apache.jackrabbit.oak.api.Tree;
import org.apache.jackrabbit.oak.query.ast.ColumnImpl;
import org.apache.jackrabbit.oak.query.ast.OrderingImpl;
+import org.apache.jackrabbit.oak.query.QueryImpl.MeasuringIterator;
import org.apache.jackrabbit.oak.spi.query.PropertyValues;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -134,6 +139,9 @@ public class UnionQueryImpl implements Q
@Override
public ColumnImpl[] getColumns() {
+ if (columns != null) {
+ return columns;
+ }
return left.getColumns();
}
@@ -264,14 +272,66 @@ public class UnionQueryImpl implements Q
LOG.debug("query union plan {}", getPlan());
}
}
- Iterator<ResultRowImpl> it = Iterators.concat(left.getRows(), right.getRows());
- if (measure) {
- // both queries measure themselves
- return it;
- }
boolean distinct = !unionAll;
Comparator<ResultRowImpl> orderBy = ResultRowImpl.getComparator(orderings);
- it = FilterIterators.newCombinedFilter(it, distinct, limit, offset, orderBy, settings);
+
+ Iterator<ResultRowImpl> it;
+ final Iterator<ResultRowImpl> leftRows = left.getRows();
+ final Iterator<ResultRowImpl> rightRows = right.getRows();
+ Iterator<ResultRowImpl> leftIter = leftRows;
+ Iterator<ResultRowImpl> rightIter = rightRows;
+
+ // if measure retrieve the backing delegate iterator instead
+ if (measure) {
+ leftIter = ((MeasuringIterator) leftRows).getDelegate();
+ rightIter = ((MeasuringIterator) rightRows).getDelegate();
+ }
+ // Since sorted by index use a merge iterator
+ if (isSortedByIndex()) {
+ it = FilterIterators
+ .newCombinedFilter(Iterators.mergeSorted(ImmutableList.of(leftIter, rightIter), orderBy), distinct,
+ limit, offset, null, settings);
+ } else {
+ it = FilterIterators
+ .newCombinedFilter(Iterators.concat(leftIter, rightIter), distinct, limit, offset, orderBy, settings);
+ }
+
+ if (measure) {
+ // return the measuring iterator for the union
+ it = new MeasuringIterator(this, it) {
+ MeasuringIterator left = (MeasuringIterator) leftRows;
+ MeasuringIterator right = (MeasuringIterator) rightRows;
+
+ @Override
+ protected void setColumns(ColumnImpl[] cols) {
+ columns = cols;
+ left.setColumns(cols);
+ right.setColumns(cols);
+ }
+
+ @Override
+ protected Map<String, Long> getSelectorScanCount() {
+ // Merge the 2 maps from the left and right queries to get the selector counts
+ Map<String, Long> leftSelectorScan = left.getSelectorScanCount();
+ Map<String, Long> rightSelectorScan = right.getSelectorScanCount();
+ Map<String, Long> unionScan = Maps.newHashMap(leftSelectorScan);
+ for (String key : rightSelectorScan.keySet()) {
+ if (unionScan.containsKey(key)) {
+ unionScan.put(key, rightSelectorScan.get(key) + unionScan.get(key));
+ } else {
+ unionScan.put(key, rightSelectorScan.get(key));
+ }
+ }
+ return unionScan;
+ }
+
+ @Override
+ protected long getReadCount() {
+ return left.getReadCount() + right.getReadCount();
+ }
+ };
+ }
+
return it;
}
@@ -279,5 +339,9 @@ public class UnionQueryImpl implements Q
public void setInternal(boolean isInternal) {
this.isInternal = isInternal;
}
-
+
+ @Override
+ public boolean isSortedByIndex() {
+ return left.isSortedByIndex() && right.isSortedByIndex();
+ }
}
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java?rev=1743094&r1=1743093&r2=1743094&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java Tue May 10 04:06:19 2016
@@ -28,8 +28,8 @@ import org.apache.jackrabbit.oak.query.x
*/
public class Statement {
- private boolean explain;
- private boolean measure;
+ boolean explain;
+ boolean measure;
/**
* The selector to get the columns from (the selector used in the select
@@ -52,9 +52,6 @@ public class Statement {
public Statement optimize() {
ignoreOrderByScoreDesc();
- if (explain || measure) {
- return this;
- }
if (where == null) {
return this;
}
@@ -80,6 +77,9 @@ public class Statement {
}
union.orderList = orderList;
union.xpathQuery = xpathQuery;
+ union.measure = measure;
+ union.explain = explain;
+
return union;
}
@@ -249,6 +249,12 @@ public class Statement {
@Override
public String toString() {
StringBuilder buff = new StringBuilder();
+ // explain | measure ...
+ if (this.explain) {
+ buff.append("explain ");
+ } else if (measure) {
+ buff.append("measure ");
+ }
buff.append(s1).append(" union ").append(s2);
// order by ...
if (orderList != null && !orderList.isEmpty()) {
Modified: jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/MultiPropertyOrTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/MultiPropertyOrTest.java?rev=1743094&r1=1743093&r2=1743094&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/MultiPropertyOrTest.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/MultiPropertyOrTest.java Tue May 10 04:06:19 2016
@@ -16,17 +16,27 @@
*/
package org.apache.jackrabbit.oak.plugins.index.property;
+import static java.util.Arrays.asList;
+import static org.apache.jackrabbit.oak.api.QueryEngine.NO_MAPPINGS;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
+import static org.junit.matchers.JUnitMatchers.containsString;
+import java.text.ParseException;
import java.util.List;
+import java.util.Random;
import javax.annotation.Nonnull;
import javax.jcr.query.Query;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import org.apache.jackrabbit.oak.Oak;
import org.apache.jackrabbit.oak.api.ContentRepository;
+import org.apache.jackrabbit.oak.api.PropertyValue;
+import org.apache.jackrabbit.oak.api.ResultRow;
import org.apache.jackrabbit.oak.api.Tree;
import org.apache.jackrabbit.oak.plugins.index.IndexUtils;
import org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent;
@@ -50,7 +60,7 @@ public class MultiPropertyOrTest extends
NodeBuilder index = IndexUtils.getOrCreateOakIndex(builder);
IndexUtils.createIndexDefinition(
index, "xyz", true, false,
- ImmutableList.<String>of("x", "y", "z"), null);
+ ImmutableList.of("x", "y", "z", "w"), null);
}
})
.with(new OpenSecurityProvider())
@@ -102,4 +112,91 @@ public class MultiPropertyOrTest extends
ImmutableList.of("/a", "/b"));
setTraversalEnabled(false);
}
+
+ @Test
+ public void unionSortResultCount() throws Exception {
+ // create test data
+ Tree test = root.getTree("/").addChild("test");
+ root.commit();
+
+ List<Integer> nodes = Lists.newArrayList();
+ Random r = new Random();
+ int seed = -2;
+ for (int i = 0; i < 1000; i++) {
+ Tree a = test.addChild("a" + i);
+ a.setProperty("x", "fooa");
+ seed += 2;
+ int num = r.nextInt(100);
+ a.setProperty("z", num);
+ nodes.add(num);
+ }
+
+ seed = -1;
+ for (int i = 0; i < 1000; i++) {
+ Tree a = test.addChild("b" + i);
+ a.setProperty("y", "foob");
+ seed += 2;
+ int num = 100 + r.nextInt(100);
+ a.setProperty("z", num);
+ nodes.add(num);
+ }
+ root.commit();
+
+ // scan count scans the whole result set
+ String query =
+ "measure /jcr:root//element(*, nt:base)[(@x = 'fooa' or @y = 'foob')] order by @z";
+ assertThat(measureWithLimit(query, XPATH, 100), containsString("scanCount: 2000"));
+ }
+
+ @Test
+ public void unionSortQueries() throws Exception {
+ // create test data
+ Tree test = root.getTree("/").addChild("test");
+ root.commit();
+
+ int seed = -3;
+ for (int i = 0; i < 5; i++) {
+ Tree a = test.addChild("a" + i);
+ a.setProperty("x", "a" + i);
+ seed += 3;
+ a.setProperty("w", seed);
+ }
+
+ seed = -2;
+ for (int i = 0; i < 5; i++) {
+ Tree a = test.addChild("b" + i);
+ a.setProperty("y", "b" + i);
+ seed += 3;
+ a.setProperty("w", seed);
+ }
+ seed = -1;
+ for (int i = 0; i < 5; i++) {
+ Tree a = test.addChild("c" + i);
+ a.setProperty("z", "c" + i);
+ seed += 3;
+ a.setProperty("w", seed);
+ }
+ root.commit();
+
+ assertQuery(
+ "/jcr:root//element(*, nt:base)[(@x = 'a4' or @y = 'b3')] order by @w",
+ XPATH,
+ asList("/test/b3", "/test/a4"));
+ assertQuery(
+ "/jcr:root//element(*, nt:base)[(@x = 'a3' or @y = 'b0' or @z = 'c2')] order by @w",
+ XPATH,
+ asList("/test/b0", "/test/c2", "/test/a3"));
+ }
+
+ private String measureWithLimit(String query, String lang, int limit) throws ParseException {
+ List<? extends ResultRow> result = Lists.newArrayList(
+ qe.executeQuery(query, lang, limit, 0, Maps.<String, PropertyValue>newHashMap(),
+ NO_MAPPINGS).getRows());
+
+ String measure = "";
+ if (result.size() > 0) {
+ measure = result.get(0).toString();
+ }
+ return measure;
+ }
}
Modified: jackrabbit/oak/branches/1.2/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java?rev=1743094&r1=1743093&r2=1743094&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java (original)
+++ jackrabbit/oak/branches/1.2/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java Tue May 10 04:06:19 2016
@@ -71,6 +71,7 @@ import org.apache.jackrabbit.util.ISO860
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.FilterDirectory;
+import org.hamcrest.CoreMatchers;
import org.junit.After;
import org.junit.Rule;
import org.junit.Test;
@@ -1698,6 +1699,92 @@ public class LucenePropertyIndexTest ext
}
@Test
+ public void unionSortResultCount() throws Exception {
+ // Index Definition
+ Tree idx = createIndex("test1", of("propa", "propb", "propc"));
+ idx.setProperty(createProperty(ORDERED_PROP_NAMES, of("propc"), STRINGS));
+ useV2(idx);
+
+ // create test data
+ Tree test = root.getTree("/").addChild("test");
+ root.commit();
+
+ List<Integer> nodes = Lists.newArrayList();
+ Random r = new Random();
+ int seed = -2;
+ for (int i = 0; i < 1000; i++) {
+ Tree a = test.addChild("a" + i);
+ a.setProperty("propa", "fooa");
+ seed += 2;
+ int num = r.nextInt(100);
+ a.setProperty("propc", num);
+ nodes.add(num);
+ }
+
+ seed = -1;
+ for (int i = 0; i < 1000; i++) {
+ Tree a = test.addChild("b" + i);
+ a.setProperty("propb", "foob");
+ seed += 2;
+ int num = 100 + r.nextInt(100);
+ a.setProperty("propc", num);
+ nodes.add(num);
+ }
+ root.commit();
+
+ // scan count scans the whole result set
+ String query =
+ "measure /jcr:root//element(*, nt:base)[(@propa = 'fooa' or @propb = 'foob')] order by @propc";
+ assertThat(measureWithLimit(query, XPATH, 100), CoreMatchers.containsString("scanCount: 101"));
+ }
+
+
+ @Test
+ public void unionSortQueries() throws Exception {
+ // Index Definition
+ Tree idx = createIndex("test1", of("propa", "propb", "propc", "propd"));
+ idx.setProperty(createProperty(ORDERED_PROP_NAMES, of("propd"), STRINGS));
+ useV2(idx);
+
+ // create test data
+ Tree test = root.getTree("/").addChild("test");
+ root.commit();
+
+ int seed = -3;
+ for (int i = 0; i < 5; i++) {
+ Tree a = test.addChild("a" + i);
+ a.setProperty("propa", "a" + i);
+ seed += 3;
+ a.setProperty("propd", seed);
+ }
+
+ seed = -2;
+ for (int i = 0; i < 5; i++) {
+ Tree a = test.addChild("b" + i);
+ a.setProperty("propb", "b" + i);
+ seed += 3;
+ a.setProperty("propd", seed);
+ }
+ seed = -1;
+ for (int i = 0; i < 5; i++) {
+ Tree a = test.addChild("c" + i);
+ a.setProperty("propc", "c" + i);
+ seed += 3;
+ a.setProperty("propd", seed);
+ }
+ root.commit();
+
+ assertQuery(
+ "/jcr:root//element(*, nt:base)[(@propa = 'a4' or @propb = 'b3')] order by @propd",
+ XPATH,
+ asList("/test/b3", "/test/a4"));
+ assertQuery(
+ "/jcr:root//element(*, nt:base)[(@propa = 'a3' or @propb = 'b0' or @propc = 'c2')] order by @propd",
+ XPATH,
+ asList("/test/b0", "/test/c2", "/test/a3"));
+ }
+
+ @Test
public void aggregationAndExcludeProperty() throws Exception {
NodeTypeRegistry.register(root, IOUtils.toInputStream(TestUtil.TEST_NODE_TYPE), "test nodeType");
Tree idx = createIndex("test1", of("propa", "propb"));