You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2014/12/11 15:49:57 UTC
svn commit: r1644659 - in /jackrabbit/oak/trunk:
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/
oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/
oak-core/src/main/java/org/apache/jackrabbit/oak/query/fulltext/ oak-...
Author: thomasm
Date: Thu Dec 11 14:49:57 2014
New Revision: 1644659
URL: http://svn.apache.org/r1644659
Log:
OAK-2301 QueryEngine should not tokenize fulltext expression by default
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/fulltext/FullTextContains.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/FullTextSearchImpl.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/fulltext/FullTextVisitor.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java?rev=1644659&r1=1644658&r2=1644659&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java Thu Dec 11 14:49:57 2014
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.jackrabbit.oak.query.fulltext.FullTextAnd;
+import org.apache.jackrabbit.oak.query.fulltext.FullTextContains;
import org.apache.jackrabbit.oak.query.fulltext.FullTextExpression;
import org.apache.jackrabbit.oak.query.fulltext.FullTextOr;
import org.apache.jackrabbit.oak.query.fulltext.FullTextTerm;
@@ -112,6 +113,11 @@ public class AggregateIndex implements A
composite.set(false);
ft.accept(new FullTextVisitor() {
+
+ @Override
+ public boolean visit(FullTextContains contains) {
+ return contains.getBase().accept(this);
+ }
@Override
public boolean visit(FullTextTerm term) {
@@ -163,6 +169,11 @@ public class AggregateIndex implements A
}
final AtomicReference<Cursor> result = new AtomicReference<Cursor>();
constraint.accept(new FullTextVisitor() {
+
+ @Override
+ public boolean visit(FullTextContains contains) {
+ return contains.getBase().accept(this);
+ }
@Override
public boolean visit(FullTextTerm term) {
@@ -216,6 +227,11 @@ public class AggregateIndex implements A
final String path) {
constraint.accept(new FullTextVisitor() {
+
+ @Override
+ public boolean visit(FullTextContains contains) {
+ return contains.getBase().accept(this);
+ }
@Override
public boolean visit(FullTextTerm term) {
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/FullTextSearchImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/FullTextSearchImpl.java?rev=1644659&r1=1644658&r2=1644659&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/FullTextSearchImpl.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/FullTextSearchImpl.java Thu Dec 11 14:49:57 2014
@@ -30,6 +30,7 @@ import org.apache.jackrabbit.oak.api.Pro
import org.apache.jackrabbit.oak.api.Tree;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.query.fulltext.FullTextContains;
import org.apache.jackrabbit.oak.query.fulltext.FullTextExpression;
import org.apache.jackrabbit.oak.query.fulltext.FullTextParser;
import org.apache.jackrabbit.oak.query.index.FilterImpl;
@@ -138,7 +139,9 @@ public class FullTextSearchImpl extends
p = PathUtils.concat(relativePath, p);
}
String p2 = normalizePropertyName(p);
- return FullTextParser.parse(p2, v.getValue(Type.STRING));
+ String rawText = v.getValue(Type.STRING);
+ FullTextExpression e = FullTextParser.parse(p2, rawText);
+ return new FullTextContains(p2, rawText, e);
} catch (ParseException e) {
throw new IllegalArgumentException("Invalid expression: " + fullTextSearchExpression, e);
}
Added: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/fulltext/FullTextContains.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/fulltext/FullTextContains.java?rev=1644659&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/fulltext/FullTextContains.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/fulltext/FullTextContains.java Thu Dec 11 14:49:57 2014
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.query.fulltext;
+
+/**
+ * A group of full-text expressions that reflects a "contains(...)" expression,
+ * and allows to access the original (unparsed) full text term.
+ */
+public class FullTextContains extends FullTextExpression {
+
+ private final String propertyName;
+ private final String rawText;
+ private final FullTextExpression base;
+
+ public FullTextContains(String propertyName, String rawText, FullTextExpression base) {
+ this.propertyName = propertyName;
+ this.rawText = rawText;
+ this.base = base;
+ }
+
+ @Override
+ public int getPrecedence() {
+ return base.getPrecedence();
+ }
+
+ @Override
+ public boolean evaluate(String value) {
+ return base.evaluate(value);
+ }
+
+ @Override
+ FullTextExpression simplify() {
+ FullTextExpression s = base.simplify();
+ if (s == base) {
+ return this;
+ }
+ return new FullTextContains(propertyName, rawText, s);
+ }
+
+ @Override
+ public String toString() {
+ return base.toString();
+ }
+
+ @Override
+ public boolean accept(FullTextVisitor v) {
+ return v.visit(this);
+ }
+
+ public FullTextExpression getBase() {
+ return base;
+ }
+
+ public String getPropertyName() {
+ return propertyName;
+ }
+
+ public String getRawText() {
+ return rawText;
+ }
+
+}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/fulltext/FullTextVisitor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/fulltext/FullTextVisitor.java?rev=1644659&r1=1644658&r2=1644659&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/fulltext/FullTextVisitor.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/fulltext/FullTextVisitor.java Thu Dec 11 14:49:57 2014
@@ -24,6 +24,14 @@ package org.apache.jackrabbit.oak.query.
* likely visit(FullTextTerm).
*/
public interface FullTextVisitor {
+
+ /**
+ * Visit an "contains" expression.
+ *
+ * @param contains the "contains" expression
+ * @return true if visiting should continue
+ */
+ boolean visit(FullTextContains contains);
/**
* Visit an "and" expression.
@@ -55,6 +63,11 @@ public interface FullTextVisitor {
public abstract static class FullTextVisitorBase implements FullTextVisitor {
@Override
+ public boolean visit(FullTextContains contains) {
+ return contains.getBase().accept(this);
+ }
+
+ @Override
public boolean visit(FullTextAnd and) {
for (FullTextExpression e : and.list) {
if (!e.accept(this)) {
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java?rev=1644659&r1=1644658&r2=1644659&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java Thu Dec 11 14:49:57 2014
@@ -31,6 +31,7 @@ import com.google.common.collect.Iterabl
import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.IndexingRule;
+import org.apache.jackrabbit.oak.query.fulltext.FullTextContains;
import org.apache.jackrabbit.oak.query.fulltext.FullTextExpression;
import org.apache.jackrabbit.oak.query.fulltext.FullTextTerm;
import org.apache.jackrabbit.oak.query.fulltext.FullTextVisitor;
@@ -200,8 +201,19 @@ class IndexPlanner {
final AtomicBoolean relativeParentsFound = new AtomicBoolean();
ft.accept(new FullTextVisitor.FullTextVisitorBase() {
@Override
+ public boolean visit(FullTextContains contains) {
+ visitTerm(contains.getPropertyName());
+ return true;
+ }
+
+ @Override
public boolean visit(FullTextTerm term) {
- String p = term.getPropertyName();
+ visitTerm(term.getPropertyName());
+ return true;
+ }
+
+ private void visitTerm(String propertyName) {
+ String p = propertyName;
String propertyPath = null;
String nodePath = null;
if (p == null) {
@@ -229,8 +241,6 @@ class IndexPlanner {
&& !indexingRule.isIndexed(propertyPath)){
nonIndexedPaths.add(p);
}
-
- return true;
}
});
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java?rev=1644659&r1=1644658&r2=1644659&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java Thu Dec 11 14:49:57 2014
@@ -61,6 +61,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.query.QueryEngineSettings;
import org.apache.jackrabbit.oak.query.QueryImpl;
import org.apache.jackrabbit.oak.query.fulltext.FullTextAnd;
+import org.apache.jackrabbit.oak.query.fulltext.FullTextContains;
import org.apache.jackrabbit.oak.query.fulltext.FullTextExpression;
import org.apache.jackrabbit.oak.query.fulltext.FullTextOr;
import org.apache.jackrabbit.oak.query.fulltext.FullTextTerm;
@@ -701,6 +702,11 @@ public class LuceneIndex implements Adva
// (a "non-local return")
final AtomicReference<Query> result = new AtomicReference<Query>();
ft.accept(new FullTextVisitor() {
+
+ @Override
+ public boolean visit(FullTextContains contains) {
+ return contains.getBase().accept(this);
+ }
@Override
public boolean visit(FullTextOr or) {
@@ -734,19 +740,22 @@ public class LuceneIndex implements Adva
@Override
public boolean visit(FullTextTerm term) {
- String p = term.getPropertyName();
+ return visitTerm(term.getPropertyName(), term.getText(), term.getBoost(), term.isNot());
+ }
+
+ private boolean visitTerm(String propertyName, String text, String boost, boolean not) {
+ String p = propertyName;
if (p != null && p.indexOf('/') >= 0) {
p = getName(p);
}
- Query q = tokenToQuery(term.getText(), p, analyzer, reader);
+ Query q = tokenToQuery(text, p, analyzer, reader);
if (q == null) {
return false;
}
- String boost = term.getBoost();
if (boost != null) {
q.setBoost(Float.parseFloat(boost));
}
- if (term.isNot()) {
+ if (not) {
BooleanQuery bq = new BooleanQuery();
bq.add(q, MUST_NOT);
result.set(bq);
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java?rev=1644659&r1=1644658&r2=1644659&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java Thu Dec 11 14:49:57 2014
@@ -45,6 +45,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.query.QueryEngineSettings;
import org.apache.jackrabbit.oak.query.QueryImpl;
import org.apache.jackrabbit.oak.query.fulltext.FullTextAnd;
+import org.apache.jackrabbit.oak.query.fulltext.FullTextContains;
import org.apache.jackrabbit.oak.query.fulltext.FullTextExpression;
import org.apache.jackrabbit.oak.query.fulltext.FullTextOr;
import org.apache.jackrabbit.oak.query.fulltext.FullTextTerm;
@@ -99,6 +100,7 @@ import static org.apache.jackrabbit.JcrC
import static org.apache.jackrabbit.oak.api.Type.LONG;
import static org.apache.jackrabbit.oak.api.Type.STRING;
import static org.apache.jackrabbit.oak.commons.PathUtils.denotesRoot;
+import static org.apache.jackrabbit.oak.commons.PathUtils.getName;
import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath;
import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldNames.PATH;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.VERSION;
@@ -802,6 +804,14 @@ public class LucenePropertyIndex impleme
// (a "non-local return")
final AtomicReference<Query> result = new AtomicReference<Query>();
ft.accept(new FullTextVisitor() {
+
+ @Override
+ public boolean visit(FullTextContains contains) {
+ // TODO
+ // visitTerm(contains.getPropertyName(), contains.getRawText(), null, false);
+ // return true;
+ return contains.getBase().accept(this);
+ }
@Override
public boolean visit(FullTextOr or) {
@@ -835,19 +845,22 @@ public class LucenePropertyIndex impleme
@Override
public boolean visit(FullTextTerm term) {
- String p = term.getPropertyName();
+ return visitTerm(term.getPropertyName(), term.getText(), term.getBoost(), term.isNot());
+ }
+
+ private boolean visitTerm(String propertyName, String text, String boost, boolean not) {
+ String p = propertyName;
if (p != null) {
p = getLuceneFieldName(p, pr);
}
- Query q = tokenToQuery(term.getText(), p, analyzer, reader);
+ Query q = tokenToQuery(text, p, analyzer, reader);
if (q == null) {
return false;
}
- String boost = term.getBoost();
if (boost != null) {
q.setBoost(Float.parseFloat(boost));
}
- if (term.isNot()) {
+ if (not) {
BooleanQuery bq = new BooleanQuery();
bq.add(q, MUST_NOT);
result.set(bq);
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java?rev=1644659&r1=1644658&r2=1644659&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java Thu Dec 11 14:49:57 2014
@@ -92,6 +92,23 @@ public class LucenePropertyIndexTest ext
.with(new NodeTypeIndexProvider())
.createContentRepository();
}
+
+ // TODO OAK-2301
+// @Test
+// public void fulltextSearchWithCustomAnalyzer() throws Exception{
+// Tree idx = createFulltextIndex(root.getTree("/"), "test");
+// TestUtil.useV2(idx);
+//
+// Tree anl = idx.addChild(ANALYZERS).addChild(ANL_DEFAULT);
+// anl.addChild(ANL_TOKENIZER).setProperty(ANL_NAME, "whitespace");
+// anl.addChild(ANL_FILTERS).addChild("stop");
+//
+// Tree test = root.getTree("/").addChild("test");
+// test.setProperty("foo", "fox jumping");
+// root.commit();
+//
+// assertQuery("select * from [nt:base] where CONTAINS(*, 'fox was jumping')", asList("/test"));
+// }
@Test
public void indexSelection() throws Exception {