You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2015/09/08 12:55:06 UTC
svn commit: r1701768 - in /jackrabbit/oak/trunk/oak-lucene/src:
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/
test/java/org/apache/jackrabbit/oak/plugins/index/lucene/
Author: chetanm
Date: Tue Sep 8 10:55:06 2015
New Revision: 1701768
URL: http://svn.apache.org/r1701768
Log:
OAK-3367 - Boosting fields not working as expected
Query on fulltext field '.' would now be implicitly expanded across multiple all configured boosted field which contribute to fulltext field (nodeScopeIndex=true)
For boosted field analyzed would be set to true so that separate fields get created for them for boost to work
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java?rev=1701768&r1=1701767&r2=1701768&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java Tue Sep 8 10:55:06 2015
@@ -624,6 +624,7 @@ class IndexDefinition implements Aggrega
private final List<NamePattern> namePatterns;
private final List<PropertyDefinition> nullCheckEnabledProperties;
private final List<PropertyDefinition> notNullCheckEnabledProperties;
+ private final List<PropertyDefinition> boostedProps;
private final boolean indexesAllNodesOfMatchingType;
private final boolean nodeNameIndexed;
@@ -647,12 +648,15 @@ class IndexDefinition implements Aggrega
List<NamePattern> namePatterns = newArrayList();
List<PropertyDefinition> nonExistentProperties = newArrayList();
List<PropertyDefinition> existentProperties = newArrayList();
+ List<PropertyDefinition> boostedProps = newArrayList();
List<Aggregate.Include> propIncludes = newArrayList();
- this.propConfigs = collectPropConfigs(config, namePatterns, propIncludes, nonExistentProperties, existentProperties);
+ this.propConfigs = collectPropConfigs(config, namePatterns, propIncludes, nonExistentProperties,
+ existentProperties, boostedProps);
this.propAggregate = new Aggregate(nodeTypeName, propIncludes);
this.aggregate = combine(propAggregate, nodeTypeName);
this.namePatterns = ImmutableList.copyOf(namePatterns);
+ this.boostedProps = ImmutableList.copyOf(boostedProps);
this.nullCheckEnabledProperties = ImmutableList.copyOf(nonExistentProperties);
this.notNullCheckEnabledProperties = ImmutableList.copyOf(existentProperties);
this.fulltextEnabled = aggregate.hasNodeAggregates() || hasAnyFullTextEnabledProperty();
@@ -681,6 +685,7 @@ class IndexDefinition implements Aggrega
this.propAggregate = original.propAggregate;
this.nullCheckEnabledProperties = original.nullCheckEnabledProperties;
this.notNullCheckEnabledProperties = original.notNullCheckEnabledProperties;
+ this.boostedProps = original.boostedProps;
this.aggregate = combine(propAggregate, nodeTypeName);
this.fulltextEnabled = aggregate.hasNodeAggregates() || original.fulltextEnabled;
this.indexesAllNodesOfMatchingType = allMatchingNodeByTypeIndexed();
@@ -717,6 +722,10 @@ class IndexDefinition implements Aggrega
return notNullCheckEnabledProperties;
}
+ public List<PropertyDefinition> getBoostedProps() {
+ return boostedProps;
+ }
+
@Override
public String toString() {
String str = "IndexRule: "+ nodeTypeName;
@@ -815,7 +824,8 @@ class IndexDefinition implements Aggrega
private Map<String, PropertyDefinition> collectPropConfigs(NodeState config, List<NamePattern> patterns,
List<Aggregate.Include> propAggregate,
List<PropertyDefinition> nonExistentProperties,
- List<PropertyDefinition> existentProperties) {
+ List<PropertyDefinition> existentProperties,
+ List<PropertyDefinition> boostedProps) {
Map<String, PropertyDefinition> propDefns = newHashMap();
NodeState propNode = config.getChildNode(LuceneIndexConstants.PROP_NODE);
@@ -852,6 +862,14 @@ class IndexDefinition implements Aggrega
if (pd.notNullCheckEnabled){
existentProperties.add(pd);
}
+
+ //Include props with name, boosted and nodeScopeIndex
+ if (pd.nodeScopeIndex
+ && pd.boost != PropertyDefinition.DEFAULT_BOOST
+ && pd.analyzed
+ && !pd.isRegexp){
+ boostedProps.add(pd);
+ }
}
}
return ImmutableMap.copyOf(propDefns);
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java?rev=1701768&r1=1701767&r2=1701768&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java Tue Sep 8 10:55:06 2015
@@ -434,7 +434,6 @@ public class LuceneIndexEditor implement
if (pd.nodeScopeIndex) {
Field field = newFulltextField(value);
- field.setBoost(pd.boost);
fields.add(field);
}
dirty = true;
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java?rev=1701768&r1=1701767&r2=1701768&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java Tue Sep 8 10:55:06 2015
@@ -1074,7 +1074,7 @@ public class LucenePropertyIndex impleme
private boolean visitTerm(String propertyName, String text, String boost, boolean not) {
String p = getLuceneFieldName(propertyName, pr);
- Query q = tokenToQuery(text, p, analyzer);
+ Query q = tokenToQuery(text, p, pr.indexingRule, analyzer);
if (q == null) {
return false;
}
@@ -1120,6 +1120,25 @@ public class LucenePropertyIndex impleme
return p;
}
+ private static Query tokenToQuery(String text, String fieldName, IndexingRule indexingRule, Analyzer analyzer) {
+ //Expand the query on fulltext field
+ if (FieldNames.FULLTEXT.equals(fieldName) &&
+ !indexingRule.getBoostedProps().isEmpty()) {
+ BooleanQuery in = new BooleanQuery();
+ for (PropertyDefinition pd : indexingRule.getBoostedProps()) {
+ Query q = tokenToQuery(text, FieldNames.createAnalyzedFieldName(pd.name), analyzer);
+ q.setBoost(pd.boost);
+ in.add(q, BooleanClause.Occur.SHOULD);
+ }
+
+ //Add the query for actual fulltext field also. That query would
+ //not be boosted
+ in.add(tokenToQuery(text, fieldName, analyzer), BooleanClause.Occur.SHOULD);
+ return in;
+ }
+ return tokenToQuery(text, fieldName, analyzer);
+ }
+
static Query tokenToQuery(String text, String fieldName, Analyzer analyzer) {
if (analyzer == null) {
return null;
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java?rev=1701768&r1=1701767&r2=1701768&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java Tue Sep 8 10:55:06 2015
@@ -105,7 +105,13 @@ class PropertyDefinition {
this.index = getOptionalValue(defn, LuceneIndexConstants.PROP_INDEX, true);
this.stored = getOptionalValue(defn, LuceneIndexConstants.PROP_USE_IN_EXCERPT, false);
this.nodeScopeIndex = getOptionalValue(defn, LuceneIndexConstants.PROP_NODE_SCOPE_INDEX, false);
- this.analyzed = getOptionalValue(defn, LuceneIndexConstants.PROP_ANALYZED, false);
+
+ //If boost is specified then that field MUST be analyzed
+ if (defn.hasProperty(FIELD_BOOST)){
+ this.analyzed = true;
+ } else {
+ this.analyzed = getOptionalValue(defn, LuceneIndexConstants.PROP_ANALYZED, false);
+ }
//If node is not set for full text then a property definition indicates that definition is for property index
this.propertyIndex = getOptionalValue(defn, LuceneIndexConstants.PROP_PROPERTY_INDEX, false);
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java?rev=1701768&r1=1701767&r2=1701768&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java Tue Sep 8 10:55:06 2015
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-@Version("2.3.0")
+@Version("2.4.0")
@Export(optional = "provide:=true")
package org.apache.jackrabbit.oak.plugins.index.lucene;
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java?rev=1701768&r1=1701767&r2=1701768&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java Tue Sep 8 10:55:06 2015
@@ -25,12 +25,12 @@ import com.google.common.collect.Immutab
import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.oak.api.Tree;
import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.IndexingRule;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.TokenizerChain;
import org.apache.jackrabbit.oak.plugins.tree.TreeFactory;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.lucene.codecs.Codec;
-import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.IndexingRule;
import org.junit.Test;
import static com.google.common.collect.ImmutableSet.of;
@@ -625,6 +625,30 @@ public class IndexDefinitionTest {
assertFalse(idxDefn.isSuggestEnabled());
}
+ @Test
+ public void analyzedEnabledForBoostedField() throws Exception {
+ NodeBuilder rules = builder.child(INDEX_RULES);
+ rules.child("nt:folder");
+ TestUtil.child(rules, "nt:folder/properties/prop1")
+ .setProperty(LuceneIndexConstants.FIELD_BOOST, 3.0)
+ .setProperty(LuceneIndexConstants.PROP_NODE_SCOPE_INDEX, true);
+ TestUtil.child(rules, "nt:folder/properties/prop2")
+ .setProperty(LuceneIndexConstants.PROP_PROPERTY_INDEX, true)
+ .setProperty(LuceneIndexConstants.PROP_NODE_SCOPE_INDEX, true);
+
+ IndexDefinition defn = new IndexDefinition(root, builder.getNodeState());
+
+ IndexingRule rule1 = defn.getApplicableIndexingRule(newTree(newNode("nt:folder")));
+ assertNotNull(rule1);
+
+ PropertyDefinition pd = rule1.getConfig("prop1");
+ assertEquals(3.0f, pd.boost, 0);
+ assertTrue("Analyzed should be assumed to be true for boosted fields", pd.analyzed);
+ assertFalse(rule1.getConfig("prop2").analyzed);
+
+ assertEquals(1, rule1.getBoostedProps().size());
+ }
+
//TODO indexesAllNodesOfMatchingType - with nullCheckEnabled
private static IndexingRule getRule(IndexDefinition defn, String typeName){
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java?rev=1701768&r1=1701767&r2=1701768&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java Tue Sep 8 10:55:06 2015
@@ -1145,7 +1145,6 @@ public class LucenePropertyIndexTest ext
assertOrderedQuery(queryString, asList("/test/c", "/test/a", "/test/b"), XPATH, true);
}
- @Ignore("OAK-3367")
@Test
public void boostTitleOverDescription() throws Exception{
NodeTypeRegistry.register(root, IOUtils.toInputStream(TestUtil.TEST_NODE_TYPE), "test nodeType");
@@ -1187,6 +1186,13 @@ public class LucenePropertyIndexTest ext
root.commit();
String queryString = "//element(*,oak:TestNode)[jcr:contains(., 'batman')]";
+ String explain = explainXpath(queryString);
+
+ //Assert that Lucene query generated has entries for all included boosted fields
+ assertThat(explain, containsString("full:jcr:content/jcr:title:batman^4.0"));
+ assertThat(explain, containsString("full:jcr:content/jcr:description:batman^2.0"));
+ assertThat(explain, containsString(":fulltext:batman"));
+
assertOrderedQuery(queryString, asList("/test/a", "/test/b", "/test/c"), XPATH, true);
}