You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ca...@apache.org on 2019/02/20 20:36:56 UTC
svn commit: r1853991 - in /jackrabbit/oak/branches/1.10: ./
oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/
oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/
oak-search/src/main/java/org/apache/jackrabbit...
Author: catholicon
Date: Wed Feb 20 20:36:56 2019
New Revision: 1853991
URL: http://svn.apache.org/viewvc?rev=1853991&view=rev
Log:
OAK-8046: Result items are not always correctly counted against the configured read limit if a query uses a lucene index (backport r1853969 from trunk)
Added:
jackrabbit/oak/branches/1.10/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/ReopenedLuceneIndexTest.java
- copied unchanged from r1853969, jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/ReopenedLuceneIndexTest.java
Modified:
jackrabbit/oak/branches/1.10/ (props changed)
jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
jackrabbit/oak/branches/1.10/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java
Propchange: jackrabbit/oak/branches/1.10/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 20 20:36:56 2019
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1851236,1851253,1851451,1852052,1852084,1852120,1852451,1852492-1852493,1852528,1852582,1852584,1852920,1853393,1853429,1853433,1853866,1853870,1853893
+/jackrabbit/oak/trunk:1851236,1851253,1851451,1852052,1852084,1852120,1852451,1852492-1852493,1852528,1852582,1852584,1852920,1853393,1853429,1853433,1853866,1853870,1853893,1853969
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java?rev=1853991&r1=1853990&r2=1853991&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java (original)
+++ jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java Wed Feb 20 20:36:56 2019
@@ -296,7 +296,7 @@ public class LuceneIndex implements Adva
final boolean nonFullTextConstraints = parent.isEmpty();
final int parentDepth = getDepth(parent);
QueryLimits settings = filter.getQueryLimits();
- Iterator<LuceneResultRow> itr = new AbstractIterator<LuceneResultRow>() {
+ LuceneResultRowIterator itr = new LuceneResultRowIterator() {
private final Deque<LuceneResultRow> queue = Queues.newArrayDeque();
private final Set<String> seenPaths = Sets.newHashSet();
private ScoreDoc lastDoc;
@@ -313,6 +313,11 @@ public class LuceneIndex implements Adva
return endOfData();
}
+ @Override
+ public int rewoundCount() {
+ return reloadCount;
+ }
+
private LuceneResultRow convertToRow(ScoreDoc doc, IndexSearcher searcher, String excerpt) throws IOException {
IndexReader reader = searcher.getIndexReader();
PathStoredFieldVisitor visitor = new PathStoredFieldVisitor();
@@ -480,7 +485,7 @@ public class LuceneIndex implements Adva
throw new IllegalStateException("Too many version changes");
}
lastDoc = null;
- LOG.debug("Change in index version detected {} => {}. Query would be performed without " +
+ LOG.info("Change in index version detected {} => {}. Query would be performed without " +
"offset; reload {}", currentVersion, lastSearchIndexerVersion, reloadCount);
}
this.lastSearchIndexerVersion = currentVersion;
@@ -1136,19 +1141,20 @@ public class LuceneIndex implements Adva
*/
static class LucenePathCursor implements Cursor {
- private static final int TRAVERSING_WARNING = Integer.getInteger("oak.traversing.warning", 10000);
+ private final int TRAVERSING_WARNING = Integer.getInteger("oak.traversing.warning", 10000);
private final Cursor pathCursor;
LuceneResultRow currentRow;
private final SizeEstimator sizeEstimator;
private long estimatedSize;
- LucenePathCursor(final Iterator<LuceneResultRow> it, QueryLimits settings, SizeEstimator sizeEstimator, Filter filter) {
+ LucenePathCursor(final LuceneResultRowIterator it, QueryLimits settings, SizeEstimator sizeEstimator, Filter filter) {
this.sizeEstimator = sizeEstimator;
Iterator<String> pathIterator = new Iterator<String>() {
private int readCount;
+ private int rewoundCount;
@Override
public boolean hasNext() {
@@ -1157,6 +1163,10 @@ public class LuceneIndex implements Adva
@Override
public String next() {
+ if (it.rewoundCount() > rewoundCount) {
+ readCount = 0;
+ rewoundCount = it.rewoundCount();
+ }
currentRow = it.next();
readCount++;
if (readCount % TRAVERSING_WARNING == 0) {
@@ -1227,4 +1237,8 @@ public class LuceneIndex implements Adva
return estimatedSize = sizeEstimator.getSize();
}
}
+
+ static abstract class LuceneResultRowIterator extends AbstractIterator<LuceneResultRow> {
+ abstract int rewoundCount();
+ }
}
Modified: jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java?rev=1853991&r1=1853990&r2=1853991&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (original)
+++ jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java Wed Feb 20 20:36:56 2019
@@ -243,7 +243,7 @@ public class LucenePropertyIndex extends
final Sort sort = getSort(plan);
final PlanResult pr = getPlanResult(plan);
QueryLimits settings = filter.getQueryLimits();
- Iterator<FulltextResultRow> itr = new AbstractIterator<FulltextResultRow>() {
+ LuceneResultRowIterator rItr = new LuceneResultRowIterator() {
private final Deque<FulltextResultRow> queue = Queues.newArrayDeque();
private final Set<String> seenPaths = Sets.newHashSet();
private ScoreDoc lastDoc;
@@ -252,6 +252,7 @@ public class LucenePropertyIndex extends
private IndexSearcher indexSearcher;
private int indexNodeId = -1;
private LuceneFacetProvider facetProvider = null;
+ private int rewoundCount = 0;
@Override
protected FulltextResultRow computeNext() {
@@ -262,6 +263,11 @@ public class LucenePropertyIndex extends
return endOfData();
}
+ @Override
+ public int rewoundCount() {
+ return rewoundCount;
+ }
+
private FulltextResultRow convertToRow(ScoreDoc doc, IndexSearcher searcher, Map<String, String> excerpts,
LuceneFacetProvider facetProvider,
String explanation) throws IOException {
@@ -527,7 +533,8 @@ public class LucenePropertyIndex extends
if (indexNodeId != indexNode.getIndexNodeId()){
//if already initialized then log about change
if (indexNodeId > 0){
- LOG.debug("Change in index version detected. Query would be performed without offset");
+ LOG.info("Change in index version detected. Query would be performed without offset");
+ rewoundCount++;
}
indexSearcher = indexNode.getSearcher();
@@ -542,13 +549,14 @@ public class LucenePropertyIndex extends
indexSearcher = null;
}
};
+ Iterator<FulltextResultRow> itr = rItr;
SizeEstimator sizeEstimator = getSizeEstimator(plan);
if (pr.hasPropertyIndexResult() || pr.evaluateSyncNodeTypeRestriction()) {
itr = mergePropertyIndexResult(plan, rootState, itr);
}
- return new FulltextPathCursor(itr, plan, settings, sizeEstimator);
+ return new FulltextPathCursor(itr, rItr, plan, settings, sizeEstimator);
}
private static Query addDescendantClauseIfRequired(Query query, IndexPlan plan) {
@@ -1552,4 +1560,6 @@ public class LucenePropertyIndex extends
return null;
}
}
+ static abstract class LuceneResultRowIterator extends AbstractIterator<FulltextResultRow> implements IteratorRewoundStateProvider {
+ }
}
Modified: jackrabbit/oak/branches/1.10/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java?rev=1853991&r1=1853990&r2=1853991&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.10/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java (original)
+++ jackrabbit/oak/branches/1.10/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java Wed Feb 20 20:36:56 2019
@@ -365,7 +365,7 @@ public abstract class FulltextIndex impl
protected static class FulltextPathCursor implements Cursor {
private final Logger log = LoggerFactory.getLogger(getClass());
- private static final int TRAVERSING_WARNING = Integer.getInteger("oak.traversing.warning", 10000);
+ private final int TRAVERSING_WARNING = Integer.getInteger("oak.traversing.warning", 10000);
private final Cursor pathCursor;
private final String pathPrefix;
@@ -374,12 +374,13 @@ public abstract class FulltextIndex impl
private long estimatedSize;
private final int numberOfFacets;
- public FulltextPathCursor(final Iterator<FulltextResultRow> it, final IndexPlan plan, QueryLimits settings, SizeEstimator sizeEstimator) {
+ public FulltextPathCursor(final Iterator<FulltextResultRow> it, final IteratorRewoundStateProvider iterStateProvider, final IndexPlan plan, QueryLimits settings, SizeEstimator sizeEstimator) {
pathPrefix = plan.getPathPrefix();
this.sizeEstimator = sizeEstimator;
Iterator<String> pathIterator = new Iterator<String>() {
private int readCount;
+ private int rewoundCount;
@Override
public boolean hasNext() {
@@ -388,6 +389,10 @@ public abstract class FulltextIndex impl
@Override
public String next() {
+ if (iterStateProvider.rewoundCount() > rewoundCount) {
+ readCount = 0;
+ rewoundCount = iterStateProvider.rewoundCount();
+ }
currentRow = it.next();
readCount++;
if (readCount % TRAVERSING_WARNING == 0) {
@@ -493,6 +498,10 @@ public abstract class FulltextIndex impl
}
}
+ public interface IteratorRewoundStateProvider {
+ int rewoundCount();
+ }
+
/**
* A query result facet, composed by its label and count.
*/