You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by al...@apache.org on 2015/07/01 15:00:32 UTC
svn commit: r1688637 - in /jackrabbit/oak/branches/1.2: ./
oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
Author: alexparvulescu
Date: Wed Jul 1 13:00:32 2015
New Revision: 1688637
URL: http://svn.apache.org/r1688637
Log:
OAK-2934 Certain searches cause lucene index to hit OutOfMemoryError
- merged r1688636
Modified:
jackrabbit/oak/branches/1.2/ (props changed)
jackrabbit/oak/branches/1.2/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
Propchange: jackrabbit/oak/branches/1.2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 1 13:00:32 2015
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1672350,1672468,1672537,1672603,1672642,1672644,1672834-1672835,1673351,1673410,1673414-1673415,1673436,1673644,1673662-1673664,1673669,1673695,1673738,1673787,1673791,1674046,1674065,1674075,1674107,1674228,1674780,1674880,1675054-1675055,1675319,1675332,1675354,1675357,1675382,1675555,1675566,1675593,1676198,1676237,1676407,1676458,1676539,1676670,1676693,1676703,1676725,1677579,1677581,1677609,1677611,1677774,1677788,1677797,1677804,1677806,1677939,1677991,1678173,1678211,1678323,1678758,1678938,1678954,1679144,1679165,1679191,1679232,1679235,1679503,1679958,1679961,1680182,1680222,1680232,1680236,1680461,1680633,1680643,1680747,1680805-1680806,1680903,1681282,1681767,1681918,1682218,1682235,1682437,1682494,1682555,1682855,1682904,1683059,1683089,1683213,1683249,1683259,1683278,1683323,1683687,1683700,1684174-1684175,1684186,1684376,1684442,1684561,1684570,1684601,1684618,1684868,1685023,1685370,1685552,1685589,1685840,1685999,1686097,1686162,1686229,1686234
,1686253,1686414,1686780,1686854,1686857,1686971,1687053-1687055,1687198,1687220,1687239-1687240,1687301,1687441,1687553,1688089-1688090,1688172,1688349,1688421,1688436,1688453,1688616,1688622
+/jackrabbit/oak/trunk:1672350,1672468,1672537,1672603,1672642,1672644,1672834-1672835,1673351,1673410,1673414-1673415,1673436,1673644,1673662-1673664,1673669,1673695,1673738,1673787,1673791,1674046,1674065,1674075,1674107,1674228,1674780,1674880,1675054-1675055,1675319,1675332,1675354,1675357,1675382,1675555,1675566,1675593,1676198,1676237,1676407,1676458,1676539,1676670,1676693,1676703,1676725,1677579,1677581,1677609,1677611,1677774,1677788,1677797,1677804,1677806,1677939,1677991,1678173,1678211,1678323,1678758,1678938,1678954,1679144,1679165,1679191,1679232,1679235,1679503,1679958,1679961,1680182,1680222,1680232,1680236,1680461,1680633,1680643,1680747,1680805-1680806,1680903,1681282,1681767,1681918,1682218,1682235,1682437,1682494,1682555,1682855,1682904,1683059,1683089,1683213,1683249,1683259,1683278,1683323,1683687,1683700,1684174-1684175,1684186,1684376,1684442,1684561,1684570,1684601,1684618,1684868,1685023,1685370,1685552,1685589,1685840,1685999,1686097,1686162,1686229,1686234
,1686253,1686414,1686780,1686854,1686857,1686971,1687053-1687055,1687198,1687220,1687239-1687240,1687301,1687441,1687553,1688089-1688090,1688172,1688349,1688421,1688436,1688453,1688616,1688622,1688636
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.2/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java?rev=1688637&r1=1688636&r2=1688637&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java (original)
+++ jackrabbit/oak/branches/1.2/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java Wed Jul 1 13:00:32 2015
@@ -31,7 +31,6 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newPathTerm;
import static org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneIndexHelper.skipTokenization;
import static org.apache.jackrabbit.oak.query.QueryImpl.JCR_PATH;
-import static org.apache.jackrabbit.oak.spi.query.QueryIndex.AdvanceFulltextQueryIndex;
import static org.apache.lucene.search.BooleanClause.Occur.MUST;
import static org.apache.lucene.search.BooleanClause.Occur.MUST_NOT;
import static org.apache.lucene.search.BooleanClause.Occur.SHOULD;
@@ -48,12 +47,6 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
-
-import com.google.common.collect.AbstractIterator;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Queues;
-import com.google.common.collect.Sets;
-
import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.oak.api.PropertyValue;
import org.apache.jackrabbit.oak.api.Result.SizePrecision;
@@ -77,6 +70,7 @@ import org.apache.jackrabbit.oak.spi.que
import org.apache.jackrabbit.oak.spi.query.IndexRow;
import org.apache.jackrabbit.oak.spi.query.PropertyValues;
import org.apache.jackrabbit.oak.spi.query.QueryIndex;
+import org.apache.jackrabbit.oak.spi.query.QueryIndex.AdvanceFulltextQueryIndex;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
@@ -88,15 +82,13 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.MultiPhraseQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
@@ -108,13 +100,15 @@ import org.apache.lucene.search.TotalHit
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.search.spell.SuggestWord;
import org.apache.lucene.search.suggest.Lookup;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
-import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Queues;
+import com.google.common.collect.Sets;
+
/**
* Provides a QueryIndex that does lookups against a Lucene-based index
*
@@ -911,18 +905,15 @@ public class LuceneIndex implements Adva
}
} else {
if (hasFulltextToken(tokens)) {
- MultiPhraseQuery mpq = new MultiPhraseQuery();
+ BooleanQuery bq = new BooleanQuery();
for(String token: tokens){
if (hasFulltextToken(token)) {
- Term[] terms = extractMatchingTokens(reader, fieldName, token);
- if (terms != null && terms.length > 0) {
- mpq.add(terms);
- }
+ bq.add(new WildcardQuery(newFulltextTerm(token, fieldName)), Occur.MUST);
} else {
- mpq.add(newFulltextTerm(token, fieldName));
+ bq.add(new TermQuery(newFulltextTerm(token, fieldName)), Occur.MUST);
}
}
- return mpq;
+ return bq;
} else {
PhraseQuery pq = new PhraseQuery();
for (String t : tokens) {
@@ -933,36 +924,6 @@ public class LuceneIndex implements Adva
}
}
- private static Term[] extractMatchingTokens(IndexReader reader, String fieldName, String token) {
- if (reader == null) {
- // getPlan call
- return null;
- }
-
- try {
- List<Term> terms = new ArrayList<Term>();
- Term onTerm = newFulltextTerm(token, fieldName);
- Terms t = MultiFields.getTerms(reader, onTerm.field());
-
- //No existing field with given name indexed so no possible term values
- if (t == null){
- return new Term[0];
- }
-
- Automaton a = WildcardQuery.toAutomaton(onTerm);
- CompiledAutomaton ca = new CompiledAutomaton(a);
- TermsEnum te = ca.getTermsEnum(t);
- BytesRef text;
- while ((text = te.next()) != null) {
- terms.add(newFulltextTerm(text.utf8ToString(), fieldName));
- }
- return terms.toArray(new Term[terms.size()]);
- } catch (IOException e) {
- LOG.error("Building fulltext query failed", e.getMessage());
- return null;
- }
- }
-
private static boolean hasFulltextToken(List<String> tokens) {
for (String token : tokens) {
if (hasFulltextToken(token)) {