You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2015/03/15 12:21:16 UTC
svn commit: r1666790 - in /jackrabbit/oak/trunk/oak-lucene/src:
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/
test/java/org/apache/jackrabbit/oak/plugins/index/lucene/
Author: chetanm
Date: Sun Mar 15 11:21:16 2015
New Revision: 1666790
URL: http://svn.apache.org/r1666790
Log:
OAK-2234 - Support property existence query (for Lucene)
Exposed new property config `notNullCheckEnabled` to enable faster and accurate execution for 'prop is not null' queries
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java?rev=1666790&r1=1666789&r2=1666790&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java Sun Mar 15 11:21:16 2015
@@ -81,6 +81,12 @@ public final class FieldNames {
public static final String NULL_PROPS = ":nullProps";
/**
+ * Name of the field that contains those property names which are exist i.e. not null
+ * for the given NodeState
+ */
+ public static final String NOT_NULL_PROPS = ":notNullProps";
+
+ /**
* Used to select only the PATH field from the lucene documents
*/
public static final Set<String> PATH_SELECTOR = new HashSet<String>(
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java?rev=1666790&r1=1666789&r2=1666790&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java Sun Mar 15 11:21:16 2015
@@ -566,6 +566,7 @@ class IndexDefinition implements Aggrega
private final Map<String, PropertyDefinition> propConfigs;
private final List<NamePattern> namePatterns;
private final List<PropertyDefinition> nullCheckEnabledProperties;
+ private final List<PropertyDefinition> notNullCheckEnabledProperties;
private final boolean indexesAllNodesOfMatchingType;
final float boost;
@@ -587,13 +588,15 @@ class IndexDefinition implements Aggrega
List<NamePattern> namePatterns = newArrayList();
List<PropertyDefinition> nonExistentProperties = newArrayList();
+ List<PropertyDefinition> existentProperties = newArrayList();
List<Aggregate.Include> propIncludes = newArrayList();
- this.propConfigs = collectPropConfigs(config, namePatterns, propIncludes, nonExistentProperties);
+ this.propConfigs = collectPropConfigs(config, namePatterns, propIncludes, nonExistentProperties, existentProperties);
this.propAggregate = new Aggregate(nodeTypeName, propIncludes);
this.aggregate = combine(propAggregate, nodeTypeName);
this.namePatterns = ImmutableList.copyOf(namePatterns);
this.nullCheckEnabledProperties = ImmutableList.copyOf(nonExistentProperties);
+ this.notNullCheckEnabledProperties = ImmutableList.copyOf(existentProperties);
this.fulltextEnabled = aggregate.hasNodeAggregates() || hasAnyFullTextEnabledProperty();
this.propertyIndexEnabled = hasAnyPropertyIndexConfigured();
this.indexesAllNodesOfMatchingType = allMatchingNodeByTypeIndexed();
@@ -618,6 +621,7 @@ class IndexDefinition implements Aggrega
this.propertyIndexEnabled = original.propertyIndexEnabled;
this.propAggregate = original.propAggregate;
this.nullCheckEnabledProperties = original.nullCheckEnabledProperties;
+ this.notNullCheckEnabledProperties = original.notNullCheckEnabledProperties;
this.aggregate = combine(propAggregate, nodeTypeName);
this.fulltextEnabled = aggregate.hasNodeAggregates() || original.fulltextEnabled;
this.indexesAllNodesOfMatchingType = allMatchingNodeByTypeIndexed();
@@ -649,6 +653,10 @@ class IndexDefinition implements Aggrega
return nullCheckEnabledProperties;
}
+ public List<PropertyDefinition> getNotNullCheckEnabledProperties() {
+ return notNullCheckEnabledProperties;
+ }
+
@Override
public String toString() {
String str = "IndexRule: "+ nodeTypeName;
@@ -742,7 +750,8 @@ class IndexDefinition implements Aggrega
private Map<String, PropertyDefinition> collectPropConfigs(NodeState config, List<NamePattern> patterns,
List<Aggregate.Include> propAggregate,
- List<PropertyDefinition> nonExistentProperties) {
+ List<PropertyDefinition> nonExistentProperties,
+ List<PropertyDefinition> existentProperties) {
Map<String, PropertyDefinition> propDefns = newHashMap();
NodeState propNode = config.getChildNode(LuceneIndexConstants.PROP_NODE);
@@ -775,6 +784,10 @@ class IndexDefinition implements Aggrega
if (pd.nullCheckEnabled){
nonExistentProperties.add(pd);
}
+
+ if (pd.notNullCheckEnabled){
+ existentProperties.add(pd);
+ }
}
}
return ImmutableMap.copyOf(propDefns);
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java?rev=1666790&r1=1666789&r2=1666790&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java Sun Mar 15 11:21:16 2015
@@ -247,4 +247,10 @@ public interface LuceneIndexConstants {
* enabled for this property
*/
String PROP_NULL_CHECK_ENABLED = "nullCheckEnabled";
+
+ /**
+ * Property definition config indicating that this property would be used with
+ * 'IS NOT NULL' constraint
+ */
+ String PROP_NOT_NULL_CHECK_ENABLED = "notNullCheckEnabled";
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java?rev=1666790&r1=1666789&r2=1666790&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java Sun Mar 15 11:21:16 2015
@@ -305,6 +305,7 @@ public class LuceneIndexEditor implement
dirty |= indexAggregates(path, fields, state);
dirty |= indexNullCheckEnabledProps(path, fields, state);
+ dirty |= indexNotNullCheckEnabledProps(path, fields, state);
if (isUpdate && !dirty) {
// updated the state but had no relevant changes
@@ -521,6 +522,17 @@ public class LuceneIndexEditor implement
//~-------------------------------------------------------< NullCheck Support >
+ private boolean indexNotNullCheckEnabledProps(String path, List<Field> fields, NodeState state) {
+ boolean fieldAdded = false;
+ for (PropertyDefinition pd : indexingRule.getNotNullCheckEnabledProperties()) {
+ if (isPropertyNotNull(state, pd)) {
+ fields.add(new StringField(FieldNames.NOT_NULL_PROPS, pd.name, Field.Store.NO));
+ fieldAdded = true;
+ }
+ }
+ return fieldAdded;
+ }
+
private boolean indexNullCheckEnabledProps(String path, List<Field> fields, NodeState state) {
boolean fieldAdded = false;
for (PropertyDefinition pd : indexingRule.getNullCheckEnabledProperties()) {
@@ -548,6 +560,22 @@ public class LuceneIndexEditor implement
return !propertyNode.hasProperty(pd.nonRelativeName);
}
+ /**
+ * Determine if the property as defined by PropertyDefinition exists or not.
+ *
+ * <p>For relative property if the intermediate nodes do not exist then property is
+ * considered to be null</p>
+ *
+ * @return true if the property exists
+ */
+ private boolean isPropertyNotNull(NodeState state, PropertyDefinition pd){
+ NodeState propertyNode = getPropertyNode(state, pd);
+ if (!propertyNode.exists()){
+ return false;
+ }
+ return propertyNode.hasProperty(pd.nonRelativeName);
+ }
+
private static NodeState getPropertyNode(NodeState nodeState, PropertyDefinition pd) {
if (!pd.relative){
return nodeState;
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java?rev=1666790&r1=1666789&r2=1666790&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java Sun Mar 15 11:21:16 2015
@@ -723,6 +723,12 @@ public class LucenePropertyIndex impleme
return new TermQuery(new Term(FieldNames.NULL_PROPS, defn.name));
}
+ //If notNullCheckEnabled explicitly enabled use the simple TermQuery
+ //otherwise later fallback to range query
+ if (pr.isNotNullRestriction() && defn.notNullCheckEnabled){
+ return new TermQuery(new Term(FieldNames.NOT_NULL_PROPS, defn.name));
+ }
+
switch (propType) {
case PropertyType.DATE: {
Long first = pr.first != null ? FieldFactory.dateToLong(pr.first.getValue(Type.DATE)) : null;
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java?rev=1666790&r1=1666789&r2=1666790&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java Sun Mar 15 11:21:16 2015
@@ -76,6 +76,8 @@ class PropertyDefinition {
final boolean nullCheckEnabled;
+ final boolean notNullCheckEnabled;
+
final int includedPropertyTypes;
final boolean relative;
@@ -117,6 +119,7 @@ class PropertyDefinition {
this.useInSuggest = getOptionalValue(defn, LuceneIndexConstants.PROP_USE_IN_SUGGEST, false);
this.useInSpellcheck = getOptionalValue(defn, LuceneIndexConstants.PROP_USE_IN_SPELLCHECK, false);
this.nullCheckEnabled = getOptionalValue(defn, LuceneIndexConstants.PROP_NULL_CHECK_ENABLED, false);
+ this.notNullCheckEnabled = getOptionalValue(defn, LuceneIndexConstants.PROP_NOT_NULL_CHECK_ENABLED, false);
this.nonRelativeName = determineNonRelativeName();
this.ancestors = computeAncestors(name);
validate();
@@ -181,7 +184,8 @@ class PropertyDefinition {
", analyzed=" + analyzed +
", ordered=" + ordered +
", useInSuggest=" + useInSuggest+
- ", nullCheckEnabled=" + nullCheckEnabled+
+ ", nullCheckEnabled=" + nullCheckEnabled +
+ ", notNullCheckEnabled=" + notNullCheckEnabled +
'}';
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java?rev=1666790&r1=1666789&r2=1666790&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java Sun Mar 15 11:21:16 2015
@@ -574,6 +574,17 @@ public class IndexDefinitionTest {
assertTrue(!idxDefn.getApplicableIndexingRule(TestUtil.NT_TEST).getNullCheckEnabledProperties().isEmpty());
}
+ @Test
+ public void notNullCheckEnabledWithTestNode() throws Exception{
+ NodeBuilder rules = builder.child(INDEX_RULES);
+ TestUtil.child(rules, "oak:TestNode/properties/prop2")
+ .setProperty(LuceneIndexConstants.PROP_NAME, "foo")
+ .setProperty(LuceneIndexConstants.PROP_NOT_NULL_CHECK_ENABLED, true);
+ root = registerTestNodeType(builder).getNodeState();
+ IndexDefinition idxDefn = new IndexDefinition(root, builder.getNodeState());
+ assertTrue(!idxDefn.getApplicableIndexingRule(TestUtil.NT_TEST).getNotNullCheckEnabledProperties().isEmpty());
+ }
+
//TODO indexesAllNodesOfMatchingType - with nullCheckEnabled
private static IndexingRule getRule(IndexDefinition defn, String typeName){
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java?rev=1666790&r1=1666789&r2=1666790&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java Sun Mar 15 11:21:16 2015
@@ -260,6 +260,38 @@ public class LuceneIndexTest {
}
@Test
+ public void testPropertyExistence() throws Exception {
+ root = TestUtil.registerTestNodeType(builder).getNodeState();
+
+ NodeBuilder index = newLucenePropertyIndexDefinition(builder.child(INDEX_DEFINITIONS_NAME),
+ "lucene", ImmutableSet.of("foo"), null);
+ NodeBuilder rules = index.child(INDEX_RULES);
+ NodeBuilder propNode = rules.child(NT_TEST).child(LuceneIndexConstants.PROP_NODE);
+
+ NodeBuilder fooProp = propNode.child("foo");
+ fooProp.setProperty(LuceneIndexConstants.PROP_PROPERTY_INDEX, true);
+ fooProp.setProperty(LuceneIndexConstants.PROP_NOT_NULL_CHECK_ENABLED, true);
+
+ NodeState before = builder.getNodeState();
+ createNodeWithType(builder, "a", NT_TEST).setProperty("foo", "bar");
+ createNodeWithType(builder, "b", NT_TEST).setProperty("foo", "bar");
+ createNodeWithType(builder, "c", NT_TEST);
+
+ NodeState after = builder.getNodeState();
+
+ NodeState indexed = HOOK.processCommit(before, after,CommitInfo.EMPTY);
+
+ IndexTracker tracker = new IndexTracker();
+ tracker.update(indexed);
+ AdvancedQueryIndex queryIndex = new LucenePropertyIndex(tracker);
+
+ FilterImpl filter = createFilter(NT_TEST);
+ filter.restrictProperty("foo", Operator.NOT_EQUAL, null);
+ assertFilter(filter, queryIndex, indexed, ImmutableList.of("/a","/b"));
+ }
+
+
+ @Test
public void testRelativePropertyNonExistence() throws Exception {
root = TestUtil.registerTestNodeType(builder).getNodeState();
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java?rev=1666790&r1=1666789&r2=1666790&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java Sun Mar 15 11:21:16 2015
@@ -403,6 +403,29 @@ public class LucenePropertyIndexTest ext
}
@Test
+ public void propertyExistenceQuery2() throws Exception {
+ NodeTypeRegistry.register(root, IOUtils.toInputStream(TestUtil.TEST_NODE_TYPE), "test nodeType");
+
+ Tree idx = createIndex("test1", of("propa", "propb"));
+ Tree props = TestUtil.newRulePropTree(idx, TestUtil.NT_TEST);
+ Tree prop = props.addChild(TestUtil.unique("prop"));
+ prop.setProperty(LuceneIndexConstants.PROP_NAME, "propa");
+ prop.setProperty(LuceneIndexConstants.PROP_PROPERTY_INDEX, true);
+ prop.setProperty(LuceneIndexConstants.PROP_NOT_NULL_CHECK_ENABLED, true);
+ root.commit();
+
+ Tree test = root.getTree("/").addChild("test");
+ createNodeWithType(test, "a", "oak:TestNode").setProperty("propa", "a");
+ createNodeWithType(test, "b", "oak:TestNode").setProperty("propa", "c");
+ createNodeWithType(test, "c", "oak:TestNode").setProperty("propb", "e");
+ root.commit();
+
+ String propabQuery = "select [jcr:path] from [oak:TestNode] where [propa] is not null";
+ assertThat(explain(propabQuery), containsString("lucene:test1(/oak:index/test1) :notNullProps:propa"));
+ assertQuery(propabQuery, asList("/test/a", "/test/b"));
+ }
+
+ @Test
public void propertyNonExistenceQuery() throws Exception {
NodeTypeRegistry.register(root, IOUtils.toInputStream(TestUtil.TEST_NODE_TYPE), "test nodeType");